Coverage for pyEDAA/Reports/DocumentationCoverage/Python.py: 96%

1# ==================================================================================================================== # 

2# _____ ____ _ _ ____ _ # 

3# _ __ _ _| ____| _ \ / \ / \ | _ \ ___ _ __ ___ _ __| |_ ___ # 

4# | '_ \| | | | _| | | | |/ _ \ / _ \ | |_) / _ \ '_ \ / _ \| '__| __/ __| # 

5# | |_) | |_| | |___| |_| / ___ \ / ___ \ _| _ < __/ |_) | (_) | | | |_\__ \ # 

6# | .__/ \__, |_____|____/_/ \_\/_/ \_(_)_| \_\___| .__/ \___/|_| \__|___/ # 

7# |_| |___/ |_| # 

8# ==================================================================================================================== # 

9# Authors: # 

10# Patrick Lehmann # 

11# # 

12# License: # 

13# ==================================================================================================================== # 

14# Copyright 2026-2026 Electronic Design Automation Abstraction (EDA²) # 

15# # 

16# Licensed under the Apache License, Version 2.0 (the "License"); # 

17# you may not use this file except in compliance with the License. # 

18# You may obtain a copy of the License at # 

19# # 

20# http://www.apache.org/licenses/LICENSE-2.0 # 

21# # 

22# Unless required by applicable law or agreed to in writing, software # 

23# distributed under the License is distributed on an "AS IS" BASIS, # 

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 

25# See the License for the specific language governing permissions and # 

26# limitations under the License. # 

27# # 

28# SPDX-License-Identifier: Apache-2.0 # 

29# ==================================================================================================================== # 

30# 

31""" 

32Reader of Python source code for the documentation coverage model. 

33 

34:class:`Analyzer` reads a package's directory - every ``*.py`` file below it - into a 

35:class:`~pyEDAA.Reports.DocumentationCoverage.DocumentationReport`. Each file is read once (:class:`SourceFile`): in its 

36declared encoding, into its lines, its syntax tree and its tokens. The tokens hold the comments, which document a field 

37or variable for Sphinx' autodoc, or excuse a missing doc-string for docstr_coverage. 

38 

39.. rubric:: Items 

40 

41* A package - a directory - and a module - a file - are items; a package's doc-string is its ``__init__.py``'s. A 

42 directory without ``__init__.py`` is a namespace package, which has nothing to document. 

43* Classes, functions, methods, constructors (``__init__``), destructors (``__del__``) and properties are items, wherever 

44 they are declared, also in a function's body. 

45* A class' fields are the names its body assigns or annotates - class variables and enumeration members too -, and its 

46 ``type`` statements, also inside an ``if`` or ``try`` statement. A name assigned several times is one item. 

47* A module's variables are the same statements in the module's body, e.g. constants, type aliases and type variables. 

48* Imports, augmented assignments, assignments to an attribute (``self._x = ...``), names assigned in a function's body, 

49 and the fields of a class declared in a function's body are no items. 

50 

51.. rubric:: Documentation 

52 

53* A module, class or callable is documented by a non-empty doc-string. A comment of docstr_coverage directly above its 

54 declaration excuses a missing doc-string: ``# docstr-coverage: inherited`` and 

55 ``# docstr-coverage: excused `reason```. 

56* A field or variable is documented, where Sphinx' autodoc finds its documentation: a ``#:`` comment behind the 

57 statement - behind a closing bracket, not behind an opening one -, ``#:`` comment lines directly above it, or a string 

58 literal directly after it. The documentation must not be empty. 

59 

60.. rubric:: Ignored and excluded items 

61 

62* Ignored: a property's setter and deleter, an ``@overload`` stub, a field or variable named with a leading and a 

63 trailing underscore (``__all__``, ``_value_``), and an item of a protected or private name, if :class:`CountedItems` 

64 doesn't select these names. 

65* Excluded: an item, whose qualified name matches one of :attr:`Analyzer.ExcludePatterns`, and the items it contains; 

66 the fields or variables, if :class:`CountedItems` doesn't select them; a module, which can't be read or parsed 

67 (:exc:`~pyEDAA.Reports.DocumentationCoverage.SourceFileWarning`). 

68""" 

69from __future__ import annotations 

70 

71from ast import AnnAssign, Assign, AST, AsyncFunctionDef, Attribute, Call, ClassDef 

72from ast import Constant, Expr, FunctionDef, If, List, Module, Name, Starred, Subscript 

73from ast import Try, Tuple, expr, get_docstring, parse, stmt 

74from datetime import timedelta 

75from enum import Flag 

76from io import StringIO 

77from pathlib import Path 

78from re import Pattern, compile as re_compile 

79from re import error as RegularExpressionError 

80from sys import version_info 

81from tokenize import COMMENT, ENDMARKER, NAME, NEWLINE, NL, NUMBER, OP, STRING 

82from tokenize import TokenError, TokenInfo, generate_tokens, open as tokenize_open 

83from typing import ClassVar, Iterable, Optional as Nullable 

84 

85from pyTooling.Common import getFullyQualifiedName 

86from pyTooling.Decorators import export, readonly 

87from pyTooling.MetaClasses import ExtendedType 

88from pyTooling.Stopwatch import Stopwatch 

89from pyTooling.Warning import WarningCollector 

90 

91from pyEDAA.Reports.DocumentationCoverage import CoverageState, DocCoverageError, DocumentationReport, Item, ItemKind 

92from pyEDAA.Reports.DocumentationCoverage import SourceFileWarning 

93 

94if version_info >= (3, 12): # pragma: no cover 

95 from ast import TypeAlias 

96 

97 

98# A class with a property named like a class - ``Path`` - can't name that class in the annotation of a field: the class 

99# body's namespace, where annotations are evaluated, binds the name to the property. 

100_Path = Path 

101 

102 

103@export 

104class SourceFile(metaclass=ExtendedType, slots=True): 

105 """ 

106 A Python source file, read once: its lines, its syntax tree, and its tokens, which hold the comments. 

107 """ 

108 

109 _COMMENT: ClassVar[Pattern[str]] = re_compile(r"^\s*#: ?(.*)\r?\n?$") #: ``#:`` line. 

110 _INHERITED: ClassVar[Pattern[str]] = re_compile(r"#\s*docstr-coverage\s*:\s*inherit(ed)?\s*") #: Inherited. 

111 _EXCUSED: ClassVar[Pattern[str]] = re_compile(r"#\s*docstr-coverage\s*:\s*excuse(d)?\s* `.*`\s*") #: Excused. 

112 

113 _path: _Path #: Path to the source file. 

114 _lines: list[str] #: The source's lines, without line endings. 

115 _tree: Module #: The source's syntax tree. 

116 _tokens: list[TokenInfo] #: The source's tokens. 

117 _parsingDuration: Nullable[timedelta] #: Duration of reading and parsing the file, if measured. 

118 _positions: dict[tuple[int, int], int] #: The index of the token starting at a line and column. 

119 _lineStarts: dict[int, int] #: The index of the first token starting in a line. 

120 _closings: dict[int, int] #: The index of the closing bracket of an opening bracket. 

121 

122 def __init__( 

123 self, 

124 path: Path, 

125 lines: list[str], 

126 tree: Module, 

127 tokens: list[TokenInfo], 

128 *, 

129 parsingDuration: Nullable[timedelta] = None 

130 ) -> None: 

131 """ 

132 Initializes a read source file, and indexes its tokens. 

133 

134 :param path: Path to the source file. 

135 :param lines: The source's lines, without line endings. 

136 :param tree: The source's syntax tree. 

137 :param tokens: The source's tokens. 

138 :param parsingDuration: Optional, the duration of reading and parsing the file. Default: ``None``. 

139 :raises ValueError: If parameter ``path`` is ``None``. 

140 :raises TypeError: If parameter ``path`` isn't of type :class:`~pathlib.Path`. 

141 :raises ValueError: If parameter ``lines`` is ``None``. 

142 :raises TypeError: If parameter ``lines`` isn't of type :class:`list`. 

143 :raises ValueError: If parameter ``tree`` is ``None``. 

144 :raises TypeError: If parameter ``tree`` isn't of type :class:`ast.Module`. 

145 :raises ValueError: If parameter ``tokens`` is ``None``. 

146 :raises TypeError: If parameter ``tokens`` isn't of type :class:`list`. 

147 :raises TypeError: If parameter ``parsingDuration`` isn't of type :class:`~datetime.timedelta`. 

148 """ 

149 for parameterName, value, expectedType in ( 

150 ("path", path, Path), 

151 ("lines", lines, list), 

152 ("tree", tree, Module), 

153 ("tokens", tokens, list) 

154 ): 

155 if value is None: 

156 raise ValueError(f"Parameter '{parameterName}' is None.") 

157 elif not isinstance(value, expectedType): 

158 ex = TypeError(f"Parameter '{parameterName}' is not of type '{expectedType.__name__}'.") 

159 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.") 

160 raise ex 

161 

162 if parsingDuration is not None and not isinstance(parsingDuration, timedelta): 

163 ex = TypeError("Parameter 'parsingDuration' is not of type 'timedelta'.") 

164 ex.add_note(f"Got type '{getFullyQualifiedName(parsingDuration)}'.") 

165 raise ex 

166 

167 self._path = path 

168 self._lines = lines 

169 self._tree = tree 

170 self._tokens = tokens 

171 self._parsingDuration = parsingDuration 

172 self._positions = {} 

173 self._lineStarts = {} 

174 self._closings = {} 

175 

176 openings: list[int] = [] 

177 for index, token in enumerate(tokens): 

178 self._positions[token.start] = index 

179 self._lineStarts.setdefault(token.start[0], index) 

180 if token.type == OP: 

181 if token.string in ("(", "[", "{"): 

182 openings.append(index) 

183 elif token.string in (")", "]", "}") and len(openings) > 0: 

184 self._closings[openings.pop()] = index 

185 

186 @classmethod 

187 def Read(cls, path: Path) -> SourceFile: 

188 """ 

189 Read a source file in its declared encoding, and parse it, measuring the duration of both. 

190 

191 :param path: The source file. 

192 :returns: The read source file. 

193 :raises ValueError: If parameter ``path`` is ``None``. 

194 :raises TypeError: If parameter ``path`` isn't of type :class:`~pathlib.Path`. 

195 :raises OSError: If the file can't be read. 

196 :raises SyntaxError: If the file declares an unknown encoding, or isn't valid Python. 

197 :raises UnicodeDecodeError: If the file isn't encoded as it declares. 

198 :raises ValueError: If the source contains a null byte (Python 3.11). 

199 :raises TokenError: If the source ends inside a statement. 

200 """ 

201 if path is None: 

202 raise ValueError("Parameter 'path' is None.") 

203 elif not isinstance(path, Path): 

204 ex = TypeError("Parameter 'path' is not of type 'Path'.") 

205 ex.add_note(f"Got type '{getFullyQualifiedName(path)}'.") 

206 raise ex 

207 

208 with Stopwatch() as stopwatch: 

209 with tokenize_open(path) as file: 

210 source = file.read() 

211 

212 tree = parse(source, filename=str(path)) 

213 tokens = list(generate_tokens(StringIO(source).readline)) 

214 

215 parsingDuration = stopwatch.Duration 

216 return cls(path, source.split("\n"), tree, tokens, parsingDuration=parsingDuration) 

217 

218 @readonly 

219 def Path(self) -> Path: 

220 """ 

221 Read-only property to access the path to the source file (:attr:`_path`). 

222 

223 :returns: The path. 

224 """ 

225 return self._path 

226 

227 @readonly 

228 def Lines(self) -> list[str]: 

229 """ 

230 Read-only property to access the source's lines (:attr:`_lines`). 

231 

232 :returns: The lines, without line endings. 

233 """ 

234 return self._lines 

235 

236 @readonly 

237 def Tree(self) -> Module: 

238 """ 

239 Read-only property to access the source's syntax tree (:attr:`_tree`). 

240 

241 :returns: The syntax tree. 

242 """ 

243 return self._tree 

244 

245 @readonly 

246 def ParsingDuration(self) -> Nullable[timedelta]: 

247 """ 

248 Read-only property to access the duration of reading, tokenizing and parsing the file (:attr:`_parsingDuration`). 

249 

250 :returns: The duration, or ``None`` if it wasn't measured. 

251 """ 

252 return self._parsingDuration 

253 

254 def DocumentationState(self, node: Module | ClassDef | FunctionDef | AsyncFunctionDef) -> CoverageState: 

255 """ 

256 Return how a module, class or function is documented: by a non-empty doc-string, or excused by a comment. 

257 

258 docstr_coverage's comment, which excuses a class or function, is the first comment above its declaration, 

259 skipping blank lines and the lines of decorators. 

260 

261 :param node: The module, class or function. 

262 :returns: The documentation state of the declaration. 

263 """ 

264 docString = get_docstring(node) 

265 if docString is not None and docString.strip() != "": 

266 return CoverageState.Documented | CoverageState.Covered 

267 elif isinstance(node, Module): 

268 return CoverageState.Undocumented 

269 

270 index = self._lineStarts.get(node.lineno, 0) - 1 

271 while index >= 0: 271 ↛ 285line 271 didn't jump to line 285 because the condition on line 271 was always true

272 token = self._tokens[index] 

273 if token.type in (NL, NEWLINE) or (token.type == NAME and token.string == "class") or \ 

274 token.line.strip().startswith("@"): 

275 index -= 1 

276 elif token.type != COMMENT: 

277 break 

278 elif self._INHERITED.match(token.string) is not None: 

279 return CoverageState.Inherited | CoverageState.Covered 

280 elif self._EXCUSED.match(token.string) is not None: 

281 return CoverageState.Excused | CoverageState.Covered 

282 else: 

283 break 

284 

285 return CoverageState.Undocumented 

286 

287 def HasDocumentationComment(self, statement: stmt) -> bool: 

288 """ 

289 Return whether an assignment is documented by a non-empty ``#:`` comment, where Sphinx' autodoc finds it. 

290 

291 Like autodoc, the comment behind the statement is looked up first - a comment behind an opening bracket belongs to 

292 the value -, then the ``#:`` comment lines directly above the statement. 

293 

294 :param statement: The assignment. 

295 :returns: ``True``, if a comment documents the assignment. 

296 """ 

297 line = self._lines[statement.lineno - 1] 

298 column = len(line.encode("utf-8")[:statement.col_offset].decode("utf-8", errors="replace")) 

299 

300 if (index := self._positions.get((statement.lineno, column), None)) is not None: 300 ↛ 316line 300 didn't jump to line 316 because the condition on line 300 was always true

301 token = self._tokens[index] 

302 while not (token.type == OP and token.string == "=") and token.type not in (NEWLINE, COMMENT, ENDMARKER): 

303 index += 1 

304 token = self._tokens[index] 

305 

306 if token.type == OP and token.string == "=": 

307 index += 1 

308 token = self._tokens[index] 

309 while token.type in (OP, NAME, NUMBER, STRING) and not (token.type == OP and token.string == ";"): 

310 index = self._closings.get(index, index) + 1 

311 token = self._tokens[index] 

312 

313 if token.type == COMMENT and (match := self._COMMENT.match(token.string)) is not None: 

314 return match[1].strip() != "" 

315 

316 if line[:column].strip() != "": 316 ↛ 317line 316 didn't jump to line 317 because the condition on line 316 was never true

317 return False 

318 

319 for lineIndex in range(statement.lineno - 2, -1, -1): 319 ↛ 325line 319 didn't jump to line 325 because the loop on line 319 didn't complete

320 if (match := self._COMMENT.match(self._lines[lineIndex])) is None: 

321 return False 

322 elif match[1].strip() != "": 322 ↛ 319line 322 didn't jump to line 319 because the condition on line 322 was always true

323 return True 

324 

325 return False 

326 

327 

328@export 

329class CountedItems(Flag): 

330 """ 

331 The items an analysis counts, besides the packages, modules, classes and callables of a public name. 

332 

333 A protected name has one leading underscore; a private name two, but not two trailing ones like a dunder name. An 

334 item of a name, which isn't selected, is ignored. Fields and variables, which aren't selected, are excluded. 

335 """ 

336 

337 Fields = 1 #: The fields of a class. 

338 Variables = 2 #: The variables of a module. 

339 Protected = 4 #: Items of a protected name, e.g. ``_x``. 

340 Private = 8 #: Items of a private, name-mangled name, e.g. ``__x``. 

341 

342 Default = Fields | Variables | Protected | Private #: All items. 

343 

344 

345# A class with a property named like a class - ``CountedItems`` - can't name that class in the annotation of a field: 

346# the class body's namespace, where annotations are evaluated, binds the name to the property. 

347_CountedItems = CountedItems 

348 

349 

350@export 

351class Analyzer(metaclass=ExtendedType, slots=True): 

352 """ 

353 Reads the documentation coverage of a Python package from its source files. 

354 

355 A package's or module's lines are its file's, so their items name no lines. 

356 

357 .. rubric:: Example 

358 

359 .. code-block:: Python 

360 

361 from pathlib import Path 

362 from pyEDAA.Reports.DocumentationCoverage.Python import Analyzer, CountedItems 

363 from pyTooling.Warning import WarningCollector 

364 

365 with WarningCollector() as warnings: 

366 counted = CountedItems.Fields | CountedItems.Variables 

367 report = Analyzer("myPackage", Path("src/myPackage"), countedItems=counted).Analyze() 

368 

369 print(f"{report.Counters.Covered}/{report.Counters.Expected}: {report.Counters.Coverage:.1%}") 

370 """ 

371 

372 _packageName: str #: Name of the package. 

373 _directory: Path #: The package's directory. 

374 _countedItems: _CountedItems #: The items the analysis counts. 

375 _excludePatterns: list[Pattern[str]] #: The patterns of qualified names, whose items are excluded. 

376 

377 def __init__( 

378 self, 

379 packageName: str, 

380 directory: Path, 

381 *, 

382 countedItems: CountedItems = CountedItems.Default, 

383 excludePatterns: Nullable[Iterable[str]] = None 

384 ) -> None: 

385 """ 

386 Initializes the analysis of a package. 

387 

388 :param packageName: Name of the package, which is the name of the top-level item, e.g. ``myPackage``. 

389 :param directory: The package's directory. 

390 :param countedItems: Optional, the items the analysis counts. Default: all of them. 

391 :param excludePatterns: Optional, regular expressions matching the whole qualified name of an item to exclude, 

392 e.g. ``myPackage\\.CLI``; the items it contains are excluded too. Default: none. 

393 :raises ValueError: If parameter ``packageName`` is ``None``. 

394 :raises TypeError: If parameter ``packageName`` isn't of type :class:`str`. 

395 :raises ValueError: If parameter ``packageName`` is empty. 

396 :raises ValueError: If parameter ``directory`` is ``None``. 

397 :raises TypeError: If parameter ``directory`` isn't of type :class:`~pathlib.Path`. 

398 :raises ValueError: If parameter ``countedItems`` is ``None``. 

399 :raises TypeError: If parameter ``countedItems`` isn't of type :class:`CountedItems`. 

400 :raises TypeError: If parameter ``excludePatterns`` isn't iterable. 

401 :raises TypeError: If parameter ``excludePatterns`` contains an element not of type :class:`str`. 

402 :raises ValueError: If parameter ``excludePatterns`` contains an invalid regular expression. 

403 """ 

404 if packageName is None: 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true

405 raise ValueError("Parameter 'packageName' is None.") 

406 elif not isinstance(packageName, str): 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true

407 ex = TypeError("Parameter 'packageName' is not of type 'str'.") 

408 ex.add_note(f"Got type '{getFullyQualifiedName(packageName)}'.") 

409 raise ex 

410 elif packageName == "": 

411 raise ValueError("Parameter 'packageName' is empty.") 

412 

413 if directory is None: 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true

414 raise ValueError("Parameter 'directory' is None.") 

415 elif not isinstance(directory, Path): 

416 ex = TypeError("Parameter 'directory' is not of type 'Path'.") 

417 ex.add_note(f"Got type '{getFullyQualifiedName(directory)}'.") 

418 raise ex 

419 

420 if countedItems is None: 

421 raise ValueError("Parameter 'countedItems' is None.") 

422 elif not isinstance(countedItems, CountedItems): 

423 ex = TypeError("Parameter 'countedItems' is not of type 'CountedItems'.") 

424 ex.add_note(f"Got type '{getFullyQualifiedName(countedItems)}'.") 

425 raise ex 

426 

427 self._packageName = packageName 

428 self._directory = directory 

429 self._countedItems = countedItems 

430 self._excludePatterns = [] 

431 

432 if excludePatterns is not None: 

433 if not isinstance(excludePatterns, Iterable): 

434 ex = TypeError("Parameter 'excludePatterns' is not iterable.") 

435 ex.add_note(f"Got type '{getFullyQualifiedName(excludePatterns)}'.") 

436 raise ex 

437 

438 for excludePattern in excludePatterns: 

439 if not isinstance(excludePattern, str): 

440 ex = TypeError("Parameter 'excludePatterns' contains an element not of type 'str'.") 

441 ex.add_note(f"Got type '{getFullyQualifiedName(excludePattern)}'.") 

442 raise ex 

443 

444 try: 

445 self._excludePatterns.append(re_compile(excludePattern)) 

446 except RegularExpressionError as cause: 

447 ex = ValueError("Parameter 'excludePatterns' contains an invalid regular expression.") 

448 ex.add_note(f"Got pattern '{excludePattern}'.") 

449 raise ex from cause 

450 

451 @readonly 

452 def PackageName(self) -> str: 

453 """ 

454 Read-only property to access the name of the package (:attr:`_packageName`). 

455 

456 :returns: The package's name. 

457 """ 

458 return self._packageName 

459 

460 @readonly 

461 def Directory(self) -> Path: 

462 """ 

463 Read-only property to access the package's directory (:attr:`_directory`). 

464 

465 :returns: The directory. 

466 """ 

467 return self._directory 

468 

469 @readonly 

470 def CountedItems(self) -> CountedItems: 

471 """ 

472 Read-only property to access the items the analysis counts (:attr:`_countedItems`). 

473 

474 :returns: The counted items. 

475 """ 

476 return self._countedItems 

477 

478 @readonly 

479 def ExcludePatterns(self) -> list[Pattern[str]]: 

480 """ 

481 Read-only property to access the patterns of qualified names, whose items are excluded (:attr:`_excludePatterns`). 

482 

483 :returns: The compiled regular expressions. 

484 """ 

485 return self._excludePatterns 

486 

487 def Analyze(self) -> DocumentationReport: 

488 """ 

489 Read every Python source file of the package into a documentation coverage report, and aggregate it. 

490 

491 The report's :attr:`~pyEDAA.Reports.DocumentationCoverage.DocumentationReport.AnalysisDuration` is the 

492 duration of the analysis. A source file, which can't be read or parsed, is raised as a 

493 :exc:`~pyEDAA.Reports.DocumentationCoverage.SourceFileWarning` to the current 

494 :class:`~pyTooling.Warning.WarningCollector`; its module is excluded, and the analysis continues. 

495 

496 :returns: The report; its top-level item is the package. 

497 :raises DocCoverageError: If the package's directory doesn't exist. 

498 :raises UnhandledCriticalWarningError: If a source file can't be analyzed, and no warning 

499 collector collects the warning. |br| 

500 Add a with-statement using 

501 :class:`~pyTooling.Warning.WarningCollector`. 

502 """ 

503 if not self._directory.is_dir(): 

504 raise DocCoverageError(f"Package directory '{self._directory.as_posix()}' doesn't exist.") 

505 

506 with Stopwatch() as stopwatch: 

507 report = DocumentationReport(Path(self._packageName)) 

508 self._ReadPackage(self._directory, self._packageName, report, report) 

509 report.Aggregate() 

510 

511 report._analysisDuration = stopwatch.Duration 

512 

513 return report 

514 

515 def _ReadPackage( 

516 self, 

517 directory: Path, 

518 name: str, 

519 parent: Item | DocumentationReport, 

520 report: DocumentationReport 

521 ) -> None: 

522 """ 

523 Read a package's ``__init__.py``, its modules, then its sub-packages, each sorted by name. 

524 

525 :param directory: The package's directory. 

526 :param name: The package's name. 

527 :param parent: The package containing this one, or the report. 

528 :param report: The report. 

529 """ 

530 if (directory / "__init__.py").is_file(): 

531 package = self._ReadModule(directory / "__init__.py", name, ItemKind.Package, parent, report) 

532 else: 

533 package = Item(name, ItemKind.Package, self._State(CoverageState.Empty, name, parent), parent=parent) 

534 

535 paths = sorted(directory.iterdir(), key=Path.as_posix) 

536 for path in paths: 

537 if path.suffix == ".py" and path.name != "__init__.py" and path.is_file(): 

538 self._ReadModule(path, path.stem, ItemKind.Module, package, report) 

539 

540 for path in paths: 

541 if path.is_dir() and next(path.rglob("*.py"), None) is not None: 

542 self._ReadPackage(path, path.name, package, report) 

543 

544 def _ReadModule( 

545 self, 

546 path: Path, 

547 name: str, 

548 kind: ItemKind, 

549 parent: Item | DocumentationReport, 

550 report: DocumentationReport 

551 ) -> Item: 

552 """ 

553 Read a module, or a package's ``__init__.py``, and the items declared in it. 

554 

555 :param path: The source file. 

556 :param name: The module's or package's name. 

557 :param kind: The kind of the module's item. 

558 :param parent: The package containing the module, or the report. 

559 :param report: The report. 

560 :returns: The module's or package's item. 

561 :raises UnhandledCriticalWarningError: If the source file can't be analyzed, and no warning 

562 collector collects the warning. 

563 """ 

564 file = report.GetOrAddFile(path.relative_to(self._directory)) 

565 try: 

566 sourceFile = SourceFile.Read(path) 

567 except (OSError, SyntaxError, TokenError, ValueError) as ex: 

568 WarningCollector.Raise( 

569 SourceFileWarning(f"Source file '{file.Path.as_posix()}' can't be analyzed.", file=file), 

570 ex 

571 ) 

572 return Item(name, kind, CoverageState.Excluded, file=file, parent=parent) 

573 

574 if len(sourceFile.Tree.body) == 0: 

575 documentation = CoverageState.Empty 

576 else: 

577 documentation = sourceFile.DocumentationState(sourceFile.Tree) 

578 

579 module = Item(name, kind, self._State(documentation, name, parent), file=file, parent=parent) 

580 self._ReadBody(sourceFile, sourceFile.Tree.body, module, False) 

581 

582 return module 

583 

584 def _ReadBody(self, sourceFile: SourceFile, body: list[stmt], parent: Item, inFunction: bool) -> None: 

585 """ 

586 Read the items declared in a module's, class' or function's body, in the order of their first declaration. 

587 

588 Classes and functions are read wherever they are declared. Fields and variables are read from a module's or class' 

589 body and the ``if`` and ``try`` statements in it, but not from a function's body or a class declared in it. An 

590 assignment to a field in the class' ``__init__`` documents the field, if commented. 

591 

592 :param sourceFile: The source file. 

593 :param body: The statements of the body. 

594 :param parent: The module's, class' or function's item. 

595 :param inFunction: Whether the body is a function's, or is declared in a function's body. 

596 """ 

597 order: list[stmt | str] = [] 

598 first: dict[str, stmt] = {} 

599 documented: dict[str, bool] = {} 

600 previous: Nullable[AST] = None 

601 

602 def visit(statements: list[stmt], readAttributes: bool, selfName: str) -> None: 

603 """ 

604 Nested function for recursion. 

605 

606 The statement before an expression is tracked as Sphinx' autodoc does: a string after an assignment documents 

607 it; the condition of an ``if`` statement is the statement before its first block. 

608 

609 :param statements: The statements of a block. 

610 :param readAttributes: Whether the block's fields or variables are read. 

611 :param selfName: Empty for a module's or class' body. In the body of a class' ``__init__``: the name of 

612 its first parameter, whose attributes document the class' fields. 

613 """ 

614 nonlocal previous 

615 for statement in statements: 

616 if isinstance(statement, (ClassDef, FunctionDef, AsyncFunctionDef)): 

617 if selfName == "": 617 ↛ 652line 617 didn't jump to line 652 because the condition on line 617 was always true

618 order.append(statement) 

619 elif readAttributes and len(names := self._TargetNames(statement, False, selfName)) > 0: 

620 hasComment = sourceFile.HasDocumentationComment(statement) 

621 for name in names: 

622 if selfName == "" and name not in first: 

623 order.append(name) 

624 first[name] = statement 

625 documented[name] = False 

626 

627 if name in documented: 

628 documented[name] = documented[name] or hasComment 

629 elif isinstance(statement, Expr) and isinstance(statement.value, Constant) and \ 

630 isinstance(statement.value.value, str) and statement.value.value.strip() != "": 

631 for name in self._TargetNames(previous, True, selfName): 

632 if name in documented: 632 ↛ 631line 632 didn't jump to line 631 because the condition on line 632 was always true

633 documented[name] = True 

634 elif isinstance(statement, If): 

635 previous = statement.test 

636 visit(statement.body, readAttributes, selfName) 

637 visit(statement.orelse, readAttributes, selfName) 

638 elif isinstance(statement, Try): 

639 visit(statement.body, readAttributes, selfName) 

640 visit(statement.orelse, readAttributes, selfName) 

641 for handler in statement.handlers: 

642 visit(handler.body, False, selfName) 

643 

644 visit(statement.finalbody, False, selfName) 

645 else: 

646 for block in ("body", "orelse", "finalbody"): 

647 visit(getattr(statement, block, []), False, selfName) 

648 

649 for clause in (*getattr(statement, "handlers", []), *getattr(statement, "cases", [])): 649 ↛ 650line 649 didn't jump to line 650 because the loop on line 649 never started

650 visit(clause.body, False, selfName) 

651 

652 previous = statement 

653 

654 isContainer = parent._kind in (ItemKind.Package, ItemKind.Module, ItemKind.Class, ItemKind.Enumeration) 

655 visit(body, isContainer and not inFunction, "") 

656 

657 if parent._kind in (ItemKind.Class, ItemKind.Enumeration) and not inFunction: 

658 for entry in [entry for entry in order if isinstance(entry, (FunctionDef, AsyncFunctionDef))]: 

659 if entry.name == "__init__" and len(parameters := (*entry.args.posonlyargs, *entry.args.args)) > 0: 

660 previous = entry 

661 visit(entry.body, True, parameters[0].arg) 

662 

663 for entry in order: 

664 if isinstance(entry, ClassDef): 

665 self._ReadClass(sourceFile, entry, parent, inFunction) 

666 elif isinstance(entry, (FunctionDef, AsyncFunctionDef)): 

667 self._ReadFunction(sourceFile, entry, parent) 

668 else: 

669 self._ReadAttribute(entry, first[entry], documented[entry], parent) 

670 

671 def _ReadClass(self, sourceFile: SourceFile, statement: ClassDef, parent: Item, inFunction: bool) -> None: 

672 """ 

673 Read a class, and the items declared in its body. 

674 

675 A class deriving from a class named ``...Enum`` or ``...Flag`` is an enumeration. 

676 

677 :param sourceFile: The source file. 

678 :param statement: The class' declaration. 

679 :param parent: The item containing the class. 

680 :param inFunction: Whether the class is declared in a function's body. 

681 """ 

682 if any(self._BaseName(base).endswith(("Enum", "Flag")) for base in statement.bases): 

683 kind = ItemKind.Enumeration 

684 else: 

685 kind = ItemKind.Class 

686 

687 item = Item( 

688 statement.name, 

689 kind, 

690 self._State(sourceFile.DocumentationState(statement), statement.name, parent), 

691 file=parent._file, 

692 startLine=self._FirstLine(statement), 

693 endLine=statement.end_lineno, 

694 parent=parent 

695 ) 

696 self._ReadBody(sourceFile, statement.body, item, inFunction) 

697 

698 def _ReadFunction(self, sourceFile: SourceFile, statement: FunctionDef | AsyncFunctionDef, parent: Item) -> None: 

699 """ 

700 Read a function, method, constructor or property, and the classes and functions declared in its body. 

701 

702 A property's setter and deleter and an ``@overload`` stub are ignored. 

703 

704 :param sourceFile: The source file. 

705 :param statement: The function's declaration. 

706 :param parent: The item containing the function. 

707 """ 

708 decorators = {self._BaseName(decorator) for decorator in statement.decorator_list} 

709 isAccessor = any( 

710 isinstance(decorator, Attribute) and decorator.attr in ("setter", "deleter") 

711 for decorator in statement.decorator_list 

712 ) 

713 isMember = parent._kind in (ItemKind.Class, ItemKind.Enumeration) 

714 

715 if isAccessor or (isMember and not decorators.isdisjoint(("property", "readonly", "cached_property", "getter"))): 

716 kind = ItemKind.Property 

717 elif isMember and statement.name == "__init__": 

718 kind = ItemKind.Constructor 

719 elif isMember and statement.name == "__del__": 

720 kind = ItemKind.Destructor 

721 elif isMember: 

722 kind = ItemKind.Method 

723 else: 

724 kind = ItemKind.Function 

725 

726 item = Item( 

727 statement.name, 

728 kind, 

729 self._State( 

730 sourceFile.DocumentationState(statement), 

731 statement.name, 

732 parent, 

733 ignored=isAccessor or "overload" in decorators 

734 ), 

735 file=parent._file, 

736 startLine=self._FirstLine(statement), 

737 endLine=statement.end_lineno, 

738 parent=parent 

739 ) 

740 self._ReadBody(sourceFile, statement.body, item, True) 

741 

742 def _ReadAttribute(self, name: str, statement: stmt, documented: bool, parent: Item) -> None: 

743 """ 

744 Read a field of a class or a variable of a module, declared first by a statement. 

745 

746 A variable assigned a call of ``TypeVar``, ``ParamSpec`` or ``TypeVarTuple`` is a type variable, of ``NewType`` a 

747 type; a variable annotated ``Final`` or named in upper case is a constant. 

748 

749 :param name: The field's or variable's name. 

750 :param statement: The first statement assigning or annotating the name. 

751 :param documented: Whether a statement assigning or annotating the name documents it. 

752 :param parent: The class' or module's item. 

753 """ 

754 isField = parent._kind in (ItemKind.Class, ItemKind.Enumeration) 

755 annotationName = self._BaseName(statement.annotation) if isinstance(statement, AnnAssign) else "" 

756 callName = self._BaseName(statement.value.func) if isinstance(getattr(statement, "value", None), Call) else "" 

757 

758 if (version_info >= (3, 12) and isinstance(statement, TypeAlias)) or annotationName == "TypeAlias": 

759 kind = ItemKind.TypeAlias 

760 elif isField and parent._kind is ItemKind.Enumeration and isinstance(statement, Assign): 

761 kind = ItemKind.EnumerationMember 

762 elif isField: 

763 kind = ItemKind.Field 

764 elif callName in ("TypeVar", "ParamSpec", "TypeVarTuple"): 

765 kind = ItemKind.TypeVariable 

766 elif callName == "NewType": 

767 kind = ItemKind.Type 

768 elif annotationName == "Final" or name.isupper(): 

769 kind = ItemKind.Constant 

770 else: 

771 kind = ItemKind.Variable 

772 

773 Item( 

774 name, 

775 kind, 

776 self._State( 

777 CoverageState.Documented | CoverageState.Covered if documented else CoverageState.Undocumented, 

778 name, 

779 parent, 

780 ignored=name.startswith("_") and name.endswith("_"), 

781 excluded=(CountedItems.Fields if isField else CountedItems.Variables) not in self._countedItems 

782 ), 

783 file=parent._file, 

784 startLine=statement.lineno, 

785 endLine=statement.end_lineno, 

786 parent=parent 

787 ) 

788 

789 def _State( 

790 self, 

791 documentation: CoverageState, 

792 name: str, 

793 parent: Item | DocumentationReport, 

794 *, 

795 ignored: bool = False, 

796 excluded: bool = False 

797 ) -> CoverageState: 

798 """ 

799 Return an item's coverage state: how it is documented, and whether it is excluded or ignored. 

800 

801 An item is excluded, if its qualified name, or the qualified name of an item containing it, matches an exclude 

802 pattern. It is ignored, if its name is protected or private, and :class:`CountedItems` doesn't select such names. 

803 

804 :param documentation: How the item is documented. 

805 :param name: The item's name. 

806 :param parent: The item's parent. 

807 :param ignored: Optional, whether the reader's rules ignore the item. Default: ``False``. 

808 :param excluded: Optional, whether the options exclude the item. Default: ``False``. 

809 :returns: The coverage state. 

810 """ 

811 names = f"{parent.QualifiedName}.{name}".split(".") if isinstance(parent, Item) else name.split(".") 

812 if excluded or any( 

813 pattern.fullmatch(".".join(names[:length])) is not None 

814 for pattern in self._excludePatterns for length in range(1, len(names) + 1) 

815 ): 

816 return documentation | CoverageState.Excluded 

817 elif ignored or ( 

818 name.startswith("_") and not name.startswith("__") and CountedItems.Protected not in self._countedItems