Coverage for pyEDAA/Reports/Unittesting/__init__.py: 84%
920 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-14 02:26 +0000
1# ==================================================================================================================== #
2# _____ ____ _ _ ____ _ #
3# _ __ _ _| ____| _ \ / \ / \ | _ \ ___ _ __ ___ _ __| |_ ___ #
4# | '_ \| | | | _| | | | |/ _ \ / _ \ | |_) / _ \ '_ \ / _ \| '__| __/ __| #
5# | |_) | |_| | |___| |_| / ___ \ / ___ \ _| _ < __/ |_) | (_) | | | |_\__ \ #
6# | .__/ \__, |_____|____/_/ \_\/_/ \_(_)_| \_\___| .__/ \___/|_| \__|___/ #
7# |_| |___/ |_| #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2024-2026 Electronic Design Automation Abstraction (EDA²) #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""
32The pyEDAA.Reports.Unittesting package implements a hierarchy of test entities. These are test cases, test suites and a
33test summary provided as a class hierarchy. Test cases are the leaf elements in the hierarchy and abstract an
34individual test run. Test suites are used to group multiple test cases or other test suites. The root element is a test
35summary. When such a summary is stored in a file format like Ant + JUnit4 XML, a file format specific document is
36derived from a summary class.
38**Data Model**
40.. mermaid::
42 graph TD;
43 doc[Document]
44 sum[Summary]
45 ts1[Testsuite]
46 ts2[Testsuite]
47 ts21[Testsuite]
48 tc11[Testcase]
49 tc12[Testcase]
50 tc13[Testcase]
51 tc21[Testcase]
52 tc22[Testcase]
53 tc211[Testcase]
54 tc212[Testcase]
55 tc213[Testcase]
57 doc:::root -.-> sum:::summary
58 sum --> ts1:::suite
59 sum --> ts2:::suite
60 ts2 --> ts21:::suite
61 ts1 --> tc11:::case
62 ts1 --> tc12:::case
63 ts1 --> tc13:::case
64 ts2 --> tc21:::case
65 ts2 --> tc22:::case
66 ts21 --> tc211:::case
67 ts21 --> tc212:::case
68 ts21 --> tc213:::case
70 classDef root fill:#4dc3ff
71 classDef summary fill:#80d4ff
72 classDef suite fill:#b3e6ff
73 classDef case fill:#eeccff
74"""
75from __future__ import annotations
77from datetime import timedelta, datetime
78from enum import Flag, IntEnum
79from pathlib import Path
80from typing import Optional as Nullable, Dict, Iterable, Any, Tuple, Generator, Union, List, Generic, \
81 TypeVar, Mapping
83from pyTooling.Common import getFullyQualifiedName
84from pyTooling.Decorators import export, readonly
85from pyTooling.Exceptions import AlreadyInHierarchyError, DuplicateElementError
86from pyTooling.MetaClasses import ExtendedType, abstractmethod
87from pyTooling.Tree import Node
89from pyEDAA.Reports import ReportException
92@export
93class UnittestError(ReportException):
94 """Base-exception for all unit test related exceptions."""
97@export
98class AlreadyInTestsuiteError(UnittestError, AlreadyInHierarchyError):
99 """
100 A unit test exception raised if the element is already part of a hierarchy.
102 This exception is caused by an inconsistent data model. Elements added to the hierarchy should be part of the same
103 hierarchy should occur only once in the hierarchy.
105 .. hint::
107 This is usually caused by a non-None parent reference.
108 """
111@export
112class DuplicateTestsuiteError(UnittestError, DuplicateElementError):
113 """
114 A unit test exception raised on duplicate test suites (by name).
116 This exception is raised, if a child test suite with same name already exist in the test suite.
118 .. hint::
120 Test suite names need to be unique per parent element (test suite or test summary).
121 """
124@export
125class DuplicateTestcaseError(UnittestError, DuplicateElementError):
126 """
127 A unit test exception raised on duplicate test cases (by name).
129 This exception is raised, if a child test case with same name already exist in the test suite.
131 .. hint::
133 Test case names need to be unique per parent element (test suite).
134 """
137@export
138class TestcaseStatus(Flag):
139 """A flag enumeration describing the status of a test case."""
140 Unknown = 0 #: Testcase status is uninitialized and therefore unknown.
141 Excluded = 1 #: Testcase was permanently excluded / disabled
142 Skipped = 2 #: Testcase was temporarily skipped (e.g. based on a condition)
143 Weak = 4 #: No assertions were recorded.
144 Passed = 8 #: A passed testcase, because all assertions were successful.
145 Failed = 16 #: A failed testcase due to at least one failed assertion.
147 Mask = Excluded | Skipped | Weak | Passed | Failed
149 Inverted = 128 #: To mark inverted results
150 UnexpectedPassed = Failed | Inverted
151 ExpectedFailed = Passed | Inverted
153 Warned = 1024 #: Runtime warning
154 Errored = 2048 #: Runtime error (mostly caught exceptions)
155 Aborted = 4096 #: Uncaught runtime exception
157 SetupError = 8192 #: Preparation / compilation error
158 TearDownError = 16384 #: Cleanup error / resource release error
159 Inconsistent = 32768 #: Dataset is inconsistent
161 Flags = Warned | Errored | Aborted | SetupError | TearDownError | Inconsistent
163 # TODO: timed out ?
164 # TODO: some passed (if merged, mixed results of passed and failed)
166 def __matmul__(self, other: TestcaseStatus) -> TestcaseStatus:
167 s = self & self.Mask
168 o = other & self.Mask
169 if s is self.Excluded: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 resolved = self.Excluded if o is self.Excluded else self.Unknown
171 elif s is self.Skipped:
172 resolved = self.Unknown if (o is self.Unknown) or (o is self.Excluded) else o
173 elif s is self.Weak: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 resolved = self.Weak if o is self.Weak else self.Unknown
175 elif s is self.Passed: 175 ↛ 180line 175 didn't jump to line 180 because the condition on line 175 was always true
176 if o is self.Failed: 176 ↛ 177line 176 didn't jump to line 177 because the condition on line 176 was never true
177 resolved = self.Failed
178 else:
179 resolved = self.Passed if (o is self.Skipped) or (o is self.Passed) else self.Unknown
180 elif s is self.Failed:
181 resolved = self.Failed if (o is self.Skipped) or (o is self.Passed) or (o is self.Failed) else self.Unknown
182 else:
183 resolved = self.Unknown
185 resolved |= (self & self.Flags) | (other & self.Flags)
186 return resolved
189@export
190class TestsuiteStatus(Flag):
191 """A flag enumeration describing the status of a test suite."""
192 Unknown = 0
193 Excluded = 1 #: Testcase was permanently excluded / disabled
194 Skipped = 2 #: Testcase was temporarily skipped (e.g. based on a condition)
195 Empty = 4 #: No tests in suite
196 Passed = 8 #: Passed testcase, because all assertions succeeded
197 Failed = 16 #: Failed testcase due to failing assertions
199 Mask = Excluded | Skipped | Empty | Passed | Failed
201 Inverted = 128 #: To mark inverted results
202 UnexpectedPassed = Failed | Inverted
203 ExpectedFailed = Passed | Inverted
205 Warned = 1024 #: Runtime warning
206 Errored = 2048 #: Runtime error (mostly caught exceptions)
207 Aborted = 4096 #: Uncaught runtime exception
209 SetupError = 8192 #: Preparation / compilation error
210 TearDownError = 16384 #: Cleanup error / resource release error
212 Flags = Warned | Errored | Aborted | SetupError | TearDownError
215@export
216class TestsuiteKind(IntEnum):
217 """Enumeration describing the kind of test suite."""
218 Root = 0 #: Root element of the hierarchy.
219 Logical = 1 #: Represents a logical unit.
220 Namespace = 2 #: Represents a namespace.
221 Package = 3 #: Represents a package.
222 Module = 4 #: Represents a module.
223 Class = 5 #: Represents a class.
226@export
227class IterationScheme(Flag):
228 """
229 A flag enumeration for selecting the test suite iteration scheme.
231 When a test entity hierarchy is (recursively) iterated, this iteration scheme describes how to iterate the hierarchy
232 and what elements to return as a result.
233 """
234 Unknown = 0 #: Neutral element.
235 IncludeSelf = 1 #: Also include the element itself.
236 IncludeTestsuites = 2 #: Include test suites into the result.
237 IncludeTestcases = 4 #: Include test cases into the result.
239 Recursive = 8 #: Iterate recursively.
241 PreOrder = 16 #: Iterate in pre-order (top-down: current node, then child element left-to-right).
242 PostOrder = 32 #: Iterate in pre-order (bottom-up: child element left-to-right, then current node).
244 Default = IncludeTestsuites | Recursive | IncludeTestcases | PreOrder #: Recursively iterate all test entities in pre-order.
245 TestsuiteDefault = IncludeTestsuites | Recursive | PreOrder #: Recursively iterate only test suites in pre-order.
246 TestcaseDefault = IncludeTestcases | Recursive | PreOrder #: Recursively iterate only test cases in pre-order.
249TestsuiteType = TypeVar("TestsuiteType", bound="Testsuite")
250TestcaseAggregateReturnType = Tuple[int, int, int, int, int, int, timedelta]
251TestsuiteAggregateReturnType = Tuple[int, int, int, int, int, int, int, int, int, int, int, int, int, int, timedelta]
254@export
255class Base(metaclass=ExtendedType, slots=True):
256 """
257 Base-class for all test entities (test cases, test suites, ...).
259 It provides a reference to the parent test entity, so bidirectional referencing can be used in the test entity
260 hierarchy.
262 Every test entity has a name to identity it. It's also used in the parent's child element dictionaries to identify the
263 child. |br|
264 E.g. it's used as a test case name in the dictionary of test cases in a test suite.
266 Every test entity has fields for time tracking. If known, a start time and a test duration can be set. For more
267 details, a setup duration and teardown duration can be added. All durations are summed up in a total duration field.
269 As tests can have warnings and errors or even fail, these messages are counted and aggregated in the test entity
270 hierarchy.
272 Every test entity offers an internal dictionary for annotations. |br|
273 This feature is for example used by Ant + JUnit4's XML property fields.
275 Besides its name, a test entity can carry a title, a summary and a description written for a reader, like the
276 doc-string of a test method. Formats like JUnit XML can't express these, so they are ``None`` then.
277 """
279 _parent: Nullable[TestsuiteBase]
280 _name: str
281 _title: Nullable[str] #: Short label of the test entity, written for a reader.
282 _summary: Nullable[str] #: Summary of the test entity, e.g. the first paragraph of its doc-string.
283 _description: Nullable[str] #: Description of the test entity, e.g. its doc-string.
285 _startTime: Nullable[datetime]
286 _setupDuration: Nullable[timedelta]
287 _testDuration: Nullable[timedelta]
288 _teardownDuration: Nullable[timedelta]
289 _totalDuration: Nullable[timedelta]
291 _warningCount: int
292 _errorCount: int
293 _fatalCount: int
295 _expectedWarningCount: int
296 _expectedErrorCount: int
297 _expectedFatalCount: int
299 _dict: Dict[str, Any]
301 def __init__(
302 self,
303 name: str,
304 title: Nullable[str] = None,
305 summary: Nullable[str] = None,
306 description: Nullable[str] = None,
307 startTime: Nullable[datetime] = None,
308 setupDuration: Nullable[timedelta] = None,
309 testDuration: Nullable[timedelta] = None,
310 teardownDuration: Nullable[timedelta] = None,
311 totalDuration: Nullable[timedelta] = None,
312 warningCount: int = 0,
313 errorCount: int = 0,
314 fatalCount: int = 0,
315 expectedWarningCount: int = 0,
316 expectedErrorCount: int = 0,
317 expectedFatalCount: int = 0,
318 keyValuePairs: Nullable[Mapping[str, Any]] = None,
319 *,
320 parent: Nullable[TestsuiteBase] = None
321 ) -> None:
322 """
323 Initializes the fields of the base-class.
325 :param name: Name of the test entity.
326 :param title: Optional, short label of the test entity, written for a reader.
327 :param summary: Optional, summary of the test entity.
328 :param description: Optional, description of the test entity.
329 :param startTime: Optional, time when the test entity was started.
330 :param setupDuration: Optional, duration it took to set up the entity.
331 :param testDuration: Optional, duration of the entity's test run.
332 :param teardownDuration: Optional, duration it took to tear down the entity.
333 :param totalDuration: Optional, total duration of the entity's execution (setup + test + teardown).
334 :param warningCount: Optional, count of encountered warnings.
335 :param errorCount: Optional, count of encountered errors.
336 :param fatalCount: Optional, count of encountered fatal errors.
337 :param keyValuePairs: Optional, mapping of key-value pairs to initialize the test entity with.
338 :param parent: Optional, reference to the parent test entity.
339 :raises ValueError: When parameter 'name' is None.
340 :raises TypeError: When parameter 'name' is not a string.
341 :raises ValueError: When parameter 'name' is empty.
342 :raises TypeError: When parameter 'title' is not a string.
343 :raises TypeError: When parameter 'summary' is not a string.
344 :raises TypeError: When parameter 'description' is not a string.
345 :raises TypeError: When parameter 'setupDuration' is not a timedelta.
346 :raises TypeError: When parameter 'testDuration' is not a timedelta.
347 :raises TypeError: When parameter 'teardownDuration' is not a timedelta.
348 :raises TypeError: When parameter 'totalDuration' is not a timedelta.
349 :raises ValueError: When parameter 'totalDuration' is not consistent.
350 :raises TypeError: When parameter 'warningCount' is not an integer.
351 :raises TypeError: When parameter 'errorCount' is not an integer.
352 :raises TypeError: When parameter 'fatalCount' is not an integer.
353 :raises TypeError: When parameter 'expectedWarningCount' is not an integer.
354 :raises TypeError: When parameter 'expectedErrorCount' is not an integer.
355 :raises TypeError: When parameter 'expectedFatalCount' is not an integer.
356 :raises TypeError: When parameter 'keyValuePairs' is not a Mapping.
357 :raises TypeError: When parameter 'parent' is not a TestsuiteBase.
358 """
360 if parent is not None and not isinstance(parent, TestsuiteBase): 360 ↛ 361line 360 didn't jump to line 361 because the condition on line 360 was never true
361 ex = TypeError("Parameter 'parent' is not of type 'TestsuiteBase'.")
362 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")
363 raise ex
365 if name is None:
366 raise ValueError("Parameter 'name' is None.")
367 elif not isinstance(name, str): 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true
368 ex = TypeError("Parameter 'name' is not of type 'str'.")
369 ex.add_note(f"Got type '{getFullyQualifiedName(name)}'.")
370 raise ex
371 elif name.strip() == "": 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true
372 raise ValueError("Parameter 'name' is empty.")
374 self._parent = parent
375 self._name = name
377 for parameterName, value in (
378 ("testDuration", testDuration),
379 ("setupDuration", setupDuration),
380 ("teardownDuration", teardownDuration),
381 ("totalDuration", totalDuration)
382 ):
383 if value is not None and not isinstance(value, timedelta): 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true
384 ex = TypeError(f"Parameter '{parameterName}' is not of type 'timedelta'.")
385 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.")
386 raise ex
388 if testDuration is not None:
389 if setupDuration is not None:
390 if teardownDuration is not None: 390 ↛ 391line 390 didn't jump to line 391 because the condition on line 390 was never true
391 if totalDuration is not None:
392 if totalDuration < (setupDuration + testDuration + teardownDuration):
393 raise ValueError("Parameter 'totalDuration' can not be less than the sum of setup, test and teardown durations.")
394 else: # no total
395 totalDuration = setupDuration + testDuration + teardownDuration
396 # no teardown
397 elif totalDuration is not None: 397 ↛ 398line 397 didn't jump to line 398 because the condition on line 397 was never true
398 if totalDuration < (setupDuration + testDuration):
399 raise ValueError("Parameter 'totalDuration' can not be less than the sum of setup and test durations.")
400 # no teardown, no total
401 else:
402 totalDuration = setupDuration + testDuration
403 # no setup
404 elif teardownDuration is not None: 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true
405 if totalDuration is not None:
406 if totalDuration < (testDuration + teardownDuration):
407 raise ValueError("Parameter 'totalDuration' can not be less than the sum of test and teardown durations.")
408 else: # no setup, no total
409 totalDuration = testDuration + teardownDuration
410 # no setup, no teardown
411 elif totalDuration is not None:
412 if totalDuration < testDuration: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true
413 raise ValueError("Parameter 'totalDuration' can not be less than test durations.")
414 else: # no setup, no teardown, no total
415 totalDuration = testDuration
416 # no test
417 elif totalDuration is not None:
418 testDuration = totalDuration
419 if setupDuration is not None:
420 testDuration -= setupDuration
422 if teardownDuration is not None:
423 testDuration -= teardownDuration
425 if testDuration < timedelta():
426 raise ValueError("Parameter 'totalDuration' can not be less than the sum of setup and teardown durations.")
428 self._startTime = startTime
429 self._setupDuration = setupDuration
430 self._testDuration = testDuration
431 self._teardownDuration = teardownDuration
432 self._totalDuration = totalDuration
434 for parameterName, value in (
435 ("warningCount", warningCount),
436 ("errorCount", errorCount),
437 ("fatalCount", fatalCount),
438 ("expectedWarningCount", expectedWarningCount),
439 ("expectedErrorCount", expectedErrorCount),
440 ("expectedFatalCount", expectedFatalCount)
441 ):
442 if not isinstance(value, int): 442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 ex = TypeError(f"Parameter '{parameterName}' is not of type 'int'.")
444 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.")
445 raise ex
447 self._warningCount = warningCount
448 self._errorCount = errorCount
449 self._fatalCount = fatalCount
450 self._expectedWarningCount = expectedWarningCount
451 self._expectedErrorCount = expectedErrorCount
452 self._expectedFatalCount = expectedFatalCount
454 for parameterName, value in (("title", title), ("summary", summary), ("description", description)):
455 if value is not None and not isinstance(value, str):
456 ex = TypeError(f"Parameter '{parameterName}' is not of type 'str'.")
457 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.")
458 raise ex
460 self._title = title
461 self._summary = summary
462 self._description = description
464 if keyValuePairs is not None and not isinstance(keyValuePairs, Mapping):
465 ex = TypeError("Parameter 'keyValuePairs' is not a mapping.")
466 ex.add_note(f"Got type '{getFullyQualifiedName(keyValuePairs)}'.")
467 raise ex
469 self._dict = {} if keyValuePairs is None else dict(keyValuePairs)
471 # QUESTION: allow Parent as setter?
472 @readonly
473 def Parent(self) -> Nullable[TestsuiteBase]:
474 """
475 Read-only property to access the reference to the parent test entity.
477 :returns: Reference to the parent entity.
478 """
479 return self._parent
481 @readonly
482 def Name(self) -> str:
483 """
484 Read-only property to access the test entity's name.
486 :returns: The test entities name.
487 """
488 return self._name
490 @readonly
491 def Title(self) -> Nullable[str]:
492 """
493 Read-only property to access the test entity's title (:attr:`_title`).
495 The title is a short label written for a reader, while the name identifies the test entity.
497 :returns: The test entity's title, or ``None`` if it wasn't recorded.
498 """
499 return self._title
501 @readonly
502 def Summary(self) -> Nullable[str]:
503 """
504 Read-only property to access the test entity's summary (:attr:`_summary`).
506 :returns: The test entity's summary, or ``None`` if it wasn't recorded.
507 """
508 return self._summary
510 @readonly
511 def Description(self) -> Nullable[str]:
512 """
513 Read-only property to access the test entity's description (:attr:`_description`).
515 :returns: The test entity's description, or ``None`` if it wasn't recorded.
516 """
517 return self._description
519 @readonly
520 def StartTime(self) -> Nullable[datetime]:
521 """
522 Read-only property to access the time when the test entity was started.
524 :returns: Time when the test entity was started.
525 """
526 return self._startTime
528 @readonly
529 def SetupDuration(self) -> Nullable[timedelta]:
530 """
531 Read-only property to access the duration of the test entity's setup.
533 :returns: Duration it took to set up the entity.
534 """
535 return self._setupDuration
537 @readonly
538 def TestDuration(self) -> Nullable[timedelta]:
539 """
540 Read-only property to access the duration of a test entities run.
542 This duration is excluding setup and teardown durations. In case setup and/or teardown durations are unknown or not
543 distinguishable, assign setup and teardown durations with zero.
545 :returns: Duration of the entity's test run.
546 """
547 return self._testDuration
549 @readonly
550 def TeardownDuration(self) -> Nullable[timedelta]:
551 """
552 Read-only property to access the duration of the test entity's teardown.
554 :returns: Duration it took to tear down the entity.
555 """
556 return self._teardownDuration
558 @readonly
559 def TotalDuration(self) -> Nullable[timedelta]:
560 """
561 Read-only property to access the total duration of a test entity run.
563 this duration includes setup and teardown durations.
565 :returns: Total duration of the entity's execution (setup + test + teardown)
566 """
567 return self._totalDuration
569 @readonly
570 def WarningCount(self) -> int:
571 """
572 Read-only property to access the number of encountered warnings.
574 :returns: Count of encountered warnings.
575 """
576 return self._warningCount
578 @readonly
579 def ErrorCount(self) -> int:
580 """
581 Read-only property to access the number of encountered errors.
583 :returns: Count of encountered errors.
584 """
585 return self._errorCount
587 @readonly
588 def FatalCount(self) -> int:
589 """
590 Read-only property to access the number of encountered fatal errors.
592 :returns: Count of encountered fatal errors.
593 """
594 return self._fatalCount
596 @readonly
597 def ExpectedWarningCount(self) -> int:
598 """
599 Read-only property to access the number of expected warnings.
601 :returns: Count of expected warnings.
602 """
603 return self._expectedWarningCount
605 @readonly
606 def ExpectedErrorCount(self) -> int:
607 """
608 Read-only property to access the number of expected errors.
610 :returns: Count of expected errors.
611 """
612 return self._expectedErrorCount
614 @readonly
615 def ExpectedFatalCount(self) -> int:
616 """
617 Read-only property to access the number of expected fatal errors.
619 :returns: Count of expected fatal errors.
620 """
621 return self._expectedFatalCount
623 def __len__(self) -> int:
624 """
625 Returns the number of annotated key-value pairs.
627 :returns: Number of annotated key-value pairs.
628 """
629 return len(self._dict)
631 def __getitem__(self, key: str) -> Any:
632 """
633 Access a key-value pair by key.
635 :param key: Name if the key-value pair.
636 :returns: Value of the accessed key.
637 """
638 return self._dict[key]
640 def __setitem__(self, key: str, value: Any) -> None:
641 """
642 Set the value of a key-value pair by key.
644 If the pair doesn't exist yet, it's created.
646 :param key: Key of the key-value pair.
647 :param value: Value of the key-value pair.
648 """
649 self._dict[key] = value
651 def __delitem__(self, key: str) -> None:
652 """
653 Delete a key-value pair by key.
655 :param key: Name if the key-value pair.
656 """
657 del self._dict[key]
659 def __contains__(self, key: str) -> bool:
660 """
661 Returns True, if a key-value pairs was annotated by this key.
663 :param key: Name of the key-value pair.
664 :returns: True, if the pair was annotated.
665 """
666 return key in self._dict
668 def __iter__(self) -> Generator[Tuple[str, Any], None, None]:
669 """
670 Iterate all annotated key-value pairs.
672 :returns: A generator of key-value pair tuples (key, value).
673 """
674 yield from self._dict.items()
676 @abstractmethod
677 def Aggregate(self, strict: bool = True) -> None:
678 """
679 Aggregate all test entities in the hierarchy.
680 """
682 @abstractmethod
683 def __str__(self) -> str:
684 """
685 Formats the test entity as human-readable incl. some statistics.
686 """
689@export
690class TestcaseOutputMixin(metaclass=ExtendedType, mixin=True):
691 """
692 A mixin-class adding the output of a test case run: a message, details and the captured output streams.
694 A test case's status can be explained by a short message and details, like the message of a failed assertion and its
695 traceback. These are the explanation of a failure, an error or why a test case was skipped. |br|
696 In addition, the test case's standard output and standard error can be captured while it runs.
698 All fields are ``None``, if the information wasn't recorded.
699 """
701 _message: Nullable[str] #: Message explaining the test case's status (e.g. a failed assertion).
702 _details: Nullable[str] #: Details explaining the test case's status (e.g. a traceback).
703 _standardOutput: Nullable[str] #: Captured standard output of the test case.
704 _standardError: Nullable[str] #: Captured standard error of the test case.
706 def __init__(
707 self,
708 message: Nullable[str] = None,
709 details: Nullable[str] = None,
710 standardOutput: Nullable[str] = None,
711 standardError: Nullable[str] = None
712 ) -> None:
713 """
714 Initializes the fields of the mixin-class.
716 :param message: Optional, message explaining the test case's status.
717 :param details: Optional, details explaining the test case's status.
718 :param standardOutput: Optional, captured standard output of the test case.
719 :param standardError: Optional, captured standard error of the test case.
720 :raises TypeError: If parameter 'message' is not a string.
721 :raises TypeError: If parameter 'details' is not a string.
722 :raises TypeError: If parameter 'standardOutput' is not a string.
723 :raises TypeError: If parameter 'standardError' is not a string.
724 """
725 for parameterName, value in (
726 ("message", message), ("details", details), ("standardOutput", standardOutput), ("standardError", standardError)
727 ):
728 if value is not None and not isinstance(value, str):
729 ex = TypeError(f"Parameter '{parameterName}' is not of type 'str'.")
730 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.")
731 raise ex
733 self._message = message
734 self._details = details
735 self._standardOutput = standardOutput
736 self._standardError = standardError
738 @readonly
739 def Message(self) -> Nullable[str]:
740 """
741 Read-only property to access the message explaining the test case's status (:attr:`_message`).
743 In a JUnit XML file, it's the ``message`` attribute of a ``<failure>``, ``<error>`` or ``<skipped>`` element.
745 :returns: The message, or ``None`` if it wasn't recorded.
746 """
747 return self._message
749 @readonly
750 def Details(self) -> Nullable[str]:
751 """
752 Read-only property to access the details explaining the test case's status (:attr:`_details`).
754 In a JUnit XML file, it's the text of a ``<failure>``, ``<error>`` or ``<skipped>`` element, e.g. a traceback.
756 :returns: The details, or ``None`` if they weren't recorded.
757 """
758 return self._details
760 @readonly
761 def StandardOutput(self) -> Nullable[str]:
762 """
763 Read-only property to access the captured standard output of the test case (:attr:`_standardOutput`).
765 In a JUnit XML file, it's the text of a ``<system-out>`` element.
767 :returns: The captured standard output, or ``None`` if it wasn't recorded.
768 """
769 return self._standardOutput
771 @readonly
772 def StandardError(self) -> Nullable[str]:
773 """
774 Read-only property to access the captured standard error of the test case (:attr:`_standardError`).
776 In a JUnit XML file, it's the text of a ``<system-err>`` element.
778 :returns: The captured standard error, or ``None`` if it wasn't recorded.
779 """
780 return self._standardError
783@export
784class Testcase(Base, TestcaseOutputMixin):
785 """
786 A testcase is the leaf-entity in the test entity hierarchy representing an individual test run.
788 Test cases are grouped by test suites in the test entity hierarchy. The root of the hierarchy is a test summary.
790 Every test case has an overall status like unknown, skipped, failed or passed.
792 In addition to all features from its base-class, test cases provide additional statistics for passed and failed
793 assertions (checks) as well as a sum thereof. The message and details explaining the status, as well as the
794 captured output streams are provided by :class:`TestcaseOutputMixin`.
795 """
797 _status: TestcaseStatus
798 _assertionCount: Nullable[int]
799 _failedAssertionCount: Nullable[int]
800 _passedAssertionCount: Nullable[int]
802 def __init__(
803 self,
804 name: str,
805 title: Nullable[str] = None,
806 summary: Nullable[str] = None,
807 description: Nullable[str] = None,
808 startTime: Nullable[datetime] = None,
809 setupDuration: Nullable[timedelta] = None,
810 testDuration: Nullable[timedelta] = None,
811 teardownDuration: Nullable[timedelta] = None,
812 totalDuration: Nullable[timedelta] = None,
813 status: TestcaseStatus = TestcaseStatus.Unknown,
814 assertionCount: Nullable[int] = None,
815 failedAssertionCount: Nullable[int] = None,
816 passedAssertionCount: Nullable[int] = None,
817 warningCount: int = 0,
818 errorCount: int = 0,
819 fatalCount: int = 0,
820 expectedWarningCount: int = 0,
821 expectedErrorCount: int = 0,
822 expectedFatalCount: int = 0,
823 keyValuePairs: Nullable[Mapping[str, Any]] = None,
824 message: Nullable[str] = None,
825 details: Nullable[str] = None,
826 standardOutput: Nullable[str] = None,
827 standardError: Nullable[str] = None,
828 *,
829 parent: Nullable[Testsuite] = None
830 ) -> None:
831 """
832 Initializes the fields of a test case.
834 :param name: Name of the test entity.
835 :param title: Optional, short label of the test case, written for a reader.
836 :param summary: Optional, summary of the test case.
837 :param description: Optional, description of the test case.
838 :param startTime: Optional, time when the test entity was started.
839 :param setupDuration: Optional, duration it took to set up the entity.
840 :param testDuration: Optional, duration of the entity's test run.
841 :param teardownDuration: Optional, duration it took to tear down the entity.
842 :param totalDuration: Optional, total duration of the entity's execution (setup + test + teardown)
843 :param status: Optional, status of the test case.
844 :param assertionCount: Optional, number of assertions within the test.
845 :param failedAssertionCount: Optional, number of failed assertions within the test.
846 :param passedAssertionCount: Optional, number of passed assertions within the test.
847 :param warningCount: Optional, count of encountered warnings.
848 :param errorCount: Optional, count of encountered errors.
849 :param fatalCount: Optional, count of encountered fatal errors.
850 :param expectedWarningCount: Optional, count of expected warnings.
851 :param expectedErrorCount: Optional, count of expected errors.
852 :param expectedFatalCount: Optional, count of expected fatal errors.
853 :param keyValuePairs: Optional, mapping of key-value pairs to initialize the test case.
854 :param message: Optional, message explaining the test case's status.
855 :param details: Optional, details explaining the test case's status (e.g. a traceback).
856 :param standardOutput: Optional, captured standard output of the test case.
857 :param standardError: Optional, captured standard error of the test case.
858 :param parent: Optional, reference to the parent test suite.
859 :raises ValueError: If parameter 'assertionCount' is not consistent.
860 :raises TypeError: If parameter 'parent' is not a Testsuite.
861 """
863 if parent is not None:
864 if not isinstance(parent, Testsuite):
865 ex = TypeError("Parameter 'parent' is not of type 'Testsuite'.")
866 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")
867 raise ex
869 parent._testcases[name] = self
871 super().__init__(
872 name,
873 title, summary, description,
874 startTime,
875 setupDuration, testDuration, teardownDuration, totalDuration,
876 warningCount, errorCount, fatalCount,
877 expectedWarningCount, expectedErrorCount, expectedFatalCount,
878 keyValuePairs,
879 parent=parent
880 )
881 TestcaseOutputMixin.__init__(self, message, details, standardOutput, standardError)
883 if not isinstance(status, TestcaseStatus): 883 ↛ 884line 883 didn't jump to line 884 because the condition on line 883 was never true
884 ex = TypeError("Parameter 'status' is not of type 'TestcaseStatus'.")
885 ex.add_note(f"Got type '{getFullyQualifiedName(status)}'.")
886 raise ex
888 self._status = status
890 for parameterName, value in (
891 ("assertionCount", assertionCount),
892 ("failedAssertionCount", failedAssertionCount),
893 ("passedAssertionCount", passedAssertionCount)
894 ):
895 if value is not None and not isinstance(value, int): 895 ↛ 896line 895 didn't jump to line 896 because the condition on line 895 was never true
896 ex = TypeError(f"Parameter '{parameterName}' is not of type 'int'.")
897 ex.add_note(f"Got type '{getFullyQualifiedName(value)}'.")
898 raise ex
900 self._assertionCount = assertionCount
901 if assertionCount is not None:
902 if failedAssertionCount is not None:
903 self._failedAssertionCount = failedAssertionCount
905 if passedAssertionCount is not None:
906 if passedAssertionCount + failedAssertionCount != assertionCount:
907 raise ValueError(f"passed assertion count ({passedAssertionCount}) + failed assertion count ({failedAssertionCount} != assertion count ({assertionCount}")
909 self._passedAssertionCount = passedAssertionCount
910 else:
911 self._passedAssertionCount = assertionCount - failedAssertionCount
912 elif passedAssertionCount is not None:
913 self._passedAssertionCount = passedAssertionCount
914 self._failedAssertionCount = assertionCount - passedAssertionCount
915 else:
916 raise ValueError("Neither passed assertion count nor failed assertion count are provided.")
917 elif failedAssertionCount is not None:
918 self._failedAssertionCount = failedAssertionCount
920 if passedAssertionCount is not None:
921 self._passedAssertionCount = passedAssertionCount
922 self._assertionCount = passedAssertionCount + failedAssertionCount
923 else:
924 raise ValueError("Passed assertion count is mandatory, if failed assertion count is provided instead of assertion count.")
925 elif passedAssertionCount is not None:
926 raise ValueError("Assertion count or failed assertion count is mandatory, if passed assertion count is provided.")
927 else:
928 self._passedAssertionCount = None
929 self._failedAssertionCount = None
931 @readonly
932 def Status(self) -> TestcaseStatus:
933 """
934 Read-only property to access the status of the test case.
936 :returns: The test case's status.
937 """
938 return self._status
940 @readonly
941 def AssertionCount(self) -> int:
942 """
943 Read-only property to return the number of assertions (checks) in a test case.
945 :returns: Number of assertions.
946 """
947 if self._assertionCount is None:
948 return 0
949 return self._assertionCount
951 @readonly
952 def FailedAssertionCount(self) -> int:
953 """
954 Read-only property to access the number of failed assertions (failed checks) in a test case.
956 :returns: Number of assertions.
957 """
958 return self._failedAssertionCount
960 @readonly
961 def PassedAssertionCount(self) -> int:
962 """
963 Read-only property to access the number of passed assertions (successful checks) in a test case.
965 :returns: Number of passed assertions.
966 """
967 return self._passedAssertionCount
969 def Copy(self) -> Testcase:
970 return self.__class__(
971 self._name,
972 self._title, self._summary, self._description,
973 self._startTime,
974 self._setupDuration,
975 self._testDuration,
976 self._teardownDuration,
977 self._totalDuration,
978 self._status,
979 self._assertionCount,
980 self._failedAssertionCount,
981 self._passedAssertionCount,
982 warningCount=self._warningCount,
983 errorCount=self._errorCount,
984 fatalCount=self._fatalCount,
985 expectedWarningCount=self._expectedWarningCount,
986 expectedErrorCount=self._expectedErrorCount,
987 expectedFatalCount=self._expectedFatalCount,
988 message=self._message,
989 details=self._details,
990 standardOutput=self._standardOutput,
991 standardError=self._standardError
992 )
993 # TODO: copy key-value-pairs?
995 def Aggregate(self, strict: bool = True) -> TestcaseAggregateReturnType:
996 if self._status is TestcaseStatus.Unknown:
997 if self._assertionCount is None: 997 ↛ 998line 997 didn't jump to line 998 because the condition on line 997 was never true
998 self._status = TestcaseStatus.Passed
999 elif self._assertionCount == 0: 999 ↛ 1000line 999 didn't jump to line 1000 because the condition on line 999 was never true
1000 self._status = TestcaseStatus.Weak
1001 elif self._failedAssertionCount == 0:
1002 self._status = TestcaseStatus.Passed
1003 else:
1004 self._status = TestcaseStatus.Failed
1006 if self._warningCount - self._expectedWarningCount > 0: 1006 ↛ 1007line 1006 didn't jump to line 1007 because the condition on line 1006 was never true
1007 self._status |= TestcaseStatus.Warned
1009 if self._errorCount - self._expectedErrorCount > 0: 1009 ↛ 1010line 1009 didn't jump to line 1010 because the condition on line 1009 was never true
1010 self._status |= TestcaseStatus.Errored
1012 if self._fatalCount - self._expectedFatalCount > 0: 1012 ↛ 1013line 1012 didn't jump to line 1013 because the condition on line 1012 was never true
1013 self._status |= TestcaseStatus.Aborted
1015 if strict:
1016 self._status = self._status & ~TestcaseStatus.Passed | TestcaseStatus.Failed
1018 # TODO: check for setup errors
1019 # TODO: check for teardown errors
1021 totalDuration = timedelta() if self._totalDuration is None else self._totalDuration
1023 return self._warningCount, self._errorCount, self._fatalCount, self._expectedWarningCount, self._expectedErrorCount, self._expectedFatalCount, totalDuration
1025 def __str__(self) -> str:
1026 """
1027 Formats the test case as human-readable incl. statistics.
1029 :pycode:`f"<Testcase {}: {} - assert/pass/fail:{}/{}/{} - warn/error/fatal:{}/{}/{} - setup/test/teardown:{}/{}/{}>"`
1031 :returns: Human-readable summary of a test case object.
1032 """
1033 def formatDuration(duration: Nullable[timedelta]) -> str:
1034 """
1035 Nested function formatting a duration in seconds.
1037 :param duration: Duration to format.
1038 :returns: Duration in seconds with three decimal places, or ``None``.
1039 """
1040 return "None" if duration is None else f"{duration.total_seconds():.3f}"
1042 return (
1043 f"<Testcase {self._name}: {self._status.name} -"
1044 f" assert/pass/fail:{self._assertionCount}/{self._passedAssertionCount}/{self._failedAssertionCount} -"
1045 f" warn/error/fatal:{self._warningCount}/{self._errorCount}/{self._fatalCount} -"
1046 f" setup/test/teardown:{formatDuration(self._setupDuration)}/{formatDuration(self._testDuration)}/"
1047 f"{formatDuration(self._teardownDuration)}>"
1048 )
1051@export
1052class TestsuiteBase(Base, Generic[TestsuiteType]):
1053 """
1054 Base-class for all test suites and for test summaries.
1056 A test suite is a mid-level grouping element in the test entity hierarchy, whereas the test summary is the root
1057 element in that hierarchy. While a test suite groups other test suites and test cases, a test summary can only group
1058 test suites. Thus, a test summary contains no test cases.
1059 """
1061 _kind: TestsuiteKind
1062 _status: TestsuiteStatus
1063 _testsuites: Dict[str, TestsuiteType]
1065 _tests: int
1066 _inconsistent: int
1067 _excluded: int
1068 _skipped: int
1069 _errored: int
1070 _weak: int
1071 _failed: int
1072 _passed: int
1074 def __init__(
1075 self,
1076 name: str,
1077 title: Nullable[str] = None,
1078 summary: Nullable[str] = None,
1079 description: Nullable[str] = None,
1080 kind: TestsuiteKind = TestsuiteKind.Logical,
1081 startTime: Nullable[datetime] = None,
1082 setupDuration: Nullable[timedelta] = None,
1083 testDuration: Nullable[timedelta] = None,
1084 teardownDuration: Nullable[timedelta] = None,
1085 totalDuration: Nullable[timedelta] = None,
1086 status: TestsuiteStatus = TestsuiteStatus.Unknown,
1087 warningCount: int = 0,
1088 errorCount: int = 0,
1089 fatalCount: int = 0,
1090 testsuites: Nullable[Iterable[TestsuiteType]] = None,
1091 keyValuePairs: Nullable[Mapping[str, Any]] = None,
1092 *,
1093 parent: Nullable[Testsuite] = None
1094 ) -> None:
1095 """
1096 Initializes the based-class fields of a test suite or test summary.
1098 :param name: Name of the test entity.
1099 :param title: Optional, short label of the test entity, written for a reader.
1100 :param summary: Optional, summary of the test entity.
1101 :param description: Optional, description of the test entity.
1102 :param kind: Optional, kind of the test entity.
1103 :param startTime: Optional, time when the test entity was started.
1104 :param setupDuration: Optional, duration it took to set up the entity.
1105 :param testDuration: Optional, duration of all tests listed in the test entity.
1106 :param teardownDuration: Optional, duration it took to tear down the entity.
1107 :param totalDuration: Optional, total duration of the entity's execution (setup + test + teardown)
1108 :param status: Optional, overall status of the test entity.
1109 :param warningCount: Optional, count of encountered warnings incl. warnings from sub-elements.
1110 :param errorCount: Optional, count of encountered errors incl. errors from sub-elements.
1111 :param fatalCount: Optional, count of encountered fatal errors incl. fatal errors from sub-elements.
1112 :param testsuites: Optional, list of test suites to initialize the test entity with.
1113 :param keyValuePairs: Optional, mapping of key-value pairs to initialize the test entity with.
1114 :param parent: Optional, reference to the parent test entity.
1115 :raises TypeError: If parameter 'testsuites' is not iterable.
1116 :raises TypeError: If element in parameter 'testsuites' is not a Testsuite.
1117 :raises TypeError: If parameter 'parent' is not a TestsuiteBase.
1118 :raises AlreadyInTestsuiteError: If a test suite in parameter 'testsuites' is already part of a test entity
1119 hierarchy.
1120 :raises DuplicateTestsuiteError: If a test suite in parameter 'testsuites' is already listed (by name) in the list
1121 of test suites.
1122 """
1123 if parent is not None:
1124 if not isinstance(parent, TestsuiteBase): 1124 ↛ 1125line 1124 didn't jump to line 1125 because the condition on line 1124 was never true
1125 ex = TypeError("Parameter 'parent' is not of type 'TestsuiteBase'.")
1126 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")
1127 raise ex
1129 parent._testsuites[name] = self
1131 super().__init__(
1132 name,
1133 title, summary, description,
1134 startTime,
1135 setupDuration,
1136 testDuration,
1137 teardownDuration,
1138 totalDuration,
1139 warningCount,
1140 errorCount,
1141 fatalCount,
1142 0, 0, 0,
1143 keyValuePairs,
1144 parent=parent
1145 )
1147 self._kind = kind
1148 self._status = status
1150 self._testsuites = {}
1151 if testsuites is not None:
1152 if not isinstance(testsuites, Iterable): 1152 ↛ 1153line 1152 didn't jump to line 1153 because the condition on line 1152 was never true
1153 ex = TypeError("Parameter 'testsuites' is not iterable.")
1154 ex.add_note(f"Got type '{getFullyQualifiedName(testsuites)}'.")
1155 raise ex
1157 for testsuite in testsuites:
1158 if not isinstance(testsuite, Testsuite): 1158 ↛ 1159line 1158 didn't jump to line 1159 because the condition on line 1158 was never true
1159 ex = TypeError("Element of parameter 'testsuites' is not of type 'Testsuite'.")
1160 ex.add_note(f"Got type '{getFullyQualifiedName(testsuite)}'.")
1161 raise ex
1163 if testsuite._parent is not None: 1163 ↛ 1164line 1163 didn't jump to line 1164 because the condition on line 1163 was never true
1164 raise AlreadyInTestsuiteError(f"Testsuite '{testsuite._name}' is already part of a testsuite hierarchy.")
1166 if testsuite._name in self._testsuites:
1167 raise DuplicateTestsuiteError(f"Testsuite already contains a testsuite with same name '{testsuite._name}'.")
1169 testsuite._parent = self
1170 self._testsuites[testsuite._name] = testsuite
1172 self._tests = 0
1173 self._inconsistent = 0
1174 self._excluded = 0
1175 self._skipped = 0
1176 self._errored = 0
1177 self._weak = 0
1178 self._failed = 0
1179 self._passed = 0
1181 @readonly
1182 def Kind(self) -> TestsuiteKind:
1183 """
1184 Read-only property to access the kind of the test suite.
1186 Test suites are used to group test cases. This grouping can be due to language/framework specifics like tests
1187 grouped by a module file or namespace. Others might be just logically grouped without any relation to a programming
1188 language construct.
1190 Test summaries always return kind ``Root``.
1192 :returns: Kind of the test suite.
1193 """
1194 return self._kind
1196 @readonly
1197 def Status(self) -> TestsuiteStatus:
1198 """
1199 Read-only property to access the aggregated overall status of the test suite.
1201 :returns: Overall status of the test suite.
1202 """
1203 return self._status
1205 @readonly
1206 def Testsuites(self) -> Dict[str, TestsuiteType]:
1207 """
1208 Read-only property to access a reference to the internal dictionary of test suites.
1210 :returns: Reference to the dictionary of test suite.
1211 """
1212 return self._testsuites
1214 @readonly
1215 def TestsuiteCount(self) -> int:
1216 """
1217 Read-only property to return the number of all test suites in the test suite hierarchy.
1219 :returns: Number of test suites.
1220 """
1221 return 1 + sum(testsuite.TestsuiteCount for testsuite in self._testsuites.values())
1223 @readonly
1224 def TestcaseCount(self) -> int:
1225 """
1226 Read-only property to return the number of all test cases in the test entity hierarchy.
1228 :returns: Number of test cases.
1229 """
1230 return sum(testsuite.TestcaseCount for testsuite in self._testsuites.values())
1232 @readonly
1233 def AssertionCount(self) -> int:
1234 """
1235 Read-only property to return the number of all assertions in all test cases in the test entity hierarchy.
1237 :returns: Number of assertions in all test cases.
1238 """
1239 return sum(ts.AssertionCount for ts in self._testsuites.values())
1241 @readonly
1242 def FailedAssertionCount(self) -> int:
1243 """
1244 Read-only property to access the number of all failed assertions in all test cases in the test entity hierarchy.
1246 :returns: Number of failed assertions in all test cases.
1247 """
1248 raise NotImplementedError()
1249 # return self._assertionCount - (self._warningCount + self._errorCount + self._fatalCount)
1251 @readonly
1252 def PassedAssertionCount(self) -> int:
1253 """
1254 Read-only property to access the number of all passed assertions in all test cases in the test entity hierarchy.
1256 :returns: Number of passed assertions in all test cases.
1257 """
1258 raise NotImplementedError()
1259 # return self._assertionCount - (self._warningCount + self._errorCount + self._fatalCount)
1261 @readonly
1262 def Tests(self) -> int:
1263 """
1264 Read-only property to access the number of tests in this entity.
1266 :returns: Number of tests.
1267 """
1268 return self._tests
1270 @readonly
1271 def Inconsistent(self) -> int:
1272 """
1273 Read-only property to access the number of inconsistent tests in the test suite hierarchy.
1275 :returns: Number of inconsistent tests.
1276 """
1277 return self._inconsistent
1279 @readonly
1280 def Excluded(self) -> int:
1281 """
1282 Read-only property to access the number of excluded tests in the test suite hierarchy.
1284 :returns: Number of excluded tests.
1285 """
1286 return self._excluded
1288 @readonly
1289 def Skipped(self) -> int:
1290 """
1291 Read-only property to access the number of skipped tests in the test suite hierarchy.
1293 :returns: Number of skipped tests.
1294 """
1295 return self._skipped
1297 @readonly
1298 def Errored(self) -> int:
1299 """
1300 Read-only property to access the number of tests with errors in the test suite hierarchy.
1302 :returns: Number of errored tests.
1303 """
1304 return self._errored
1306 @readonly
1307 def Weak(self) -> int:
1308 """
1309 Read-only property to access the number of weak tests in the test suite hierarchy.
1311 :returns: Number of weak tests.
1312 """
1313 return self._weak
1315 @readonly
1316 def Failed(self) -> int:
1317 """
1318 Read-only property to access the number of failed tests in the test suite hierarchy.
1320 :returns: Number of failed tests.
1321 """
1322 return self._failed
1324 @readonly
1325 def Passed(self) -> int:
1326 """
1327 Read-only property to access the number of passed tests in the test suite hierarchy.
1329 :returns: Number of passed tests.
1330 """
1331 return self._passed
1333 @readonly
1334 def WarningCount(self) -> int:
1335 """
1336 Read-only property to access the number of warnings in this entity.
1338 :returns: Number of warnings.
1339 """
1340 raise NotImplementedError()
1341 # return self._warningCount
1343 @readonly
1344 def ErrorCount(self) -> int:
1345 """
1346 Read-only property to access the number of errors in this entity.
1348 :returns: Number of errors.
1349 """
1350 raise NotImplementedError()
1351 # return self._errorCount
1353 @readonly
1354 def FatalCount(self) -> int:
1355 """
1356 Read-only property to access the number of fatal errors in this entity.
1358 :returns: Number of fatal errors.
1359 """
1360 raise NotImplementedError()
1361 # return self._fatalCount
1363 def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType:
1364 tests = 0
1365 inconsistent = 0
1366 excluded = 0
1367 skipped = 0
1368 errored = 0
1369 weak = 0
1370 failed = 0
1371 passed = 0
1373 warningCount = 0
1374 errorCount = 0
1375 fatalCount = 0
1377 expectedWarningCount = 0
1378 expectedErrorCount = 0
1379 expectedFatalCount = 0
1381 totalDuration = timedelta()
1383 for testsuite in self._testsuites.values():
1384 t, i, ex, s, e, w, f, p, wc, ec, fc, ewc, eec, efc, td = testsuite.Aggregate(strict)
1385 tests += t
1386 inconsistent += i
1387 excluded += ex
1388 skipped += s
1389 errored += e
1390 weak += w
1391 failed += f
1392 passed += p
1394 warningCount += wc
1395 errorCount += ec
1396 fatalCount += fc
1398 expectedWarningCount += ewc
1399 expectedErrorCount += eec
1400 expectedFatalCount += efc
1402 totalDuration += td
1404 return tests, inconsistent, excluded, skipped, errored, weak, failed, passed, warningCount, errorCount, fatalCount, expectedWarningCount, expectedErrorCount, expectedFatalCount, totalDuration
1406 def AddTestsuite(self, testsuite: TestsuiteType) -> None:
1407 """
1408 Add a test suite to the list of test suites.
1410 :param testsuite: The test suite to add.
1411 :raises ValueError: If parameter 'testsuite' is None.
1412 :raises TypeError: If parameter 'testsuite' is not a Testsuite.
1413 :raises AlreadyInTestsuiteError: If parameter 'testsuite' is already part of a test entity hierarchy.
1414 :raises DuplicateTestcaseError: If parameter 'testsuite' is already listed (by name) in the list of test suites.
1415 """
1416 if testsuite is None: 1416 ↛ 1417line 1416 didn't jump to line 1417 because the condition on line 1416 was never true
1417 raise ValueError("Parameter 'testsuite' is None.")
1418 elif not isinstance(testsuite, Testsuite): 1418 ↛ 1419line 1418 didn't jump to line 1419 because the condition on line 1418 was never true
1419 ex = TypeError("Parameter 'testsuite' is not of type 'Testsuite'.")
1420 ex.add_note(f"Got type '{getFullyQualifiedName(testsuite)}'.")
1421 raise ex
1423 if testsuite._parent is not None: 1423 ↛ 1424line 1423 didn't jump to line 1424 because the condition on line 1423 was never true
1424 raise AlreadyInTestsuiteError(f"Testsuite '{testsuite._name}' is already part of a testsuite hierarchy.")
1426 if testsuite._name in self._testsuites:
1427 raise DuplicateTestsuiteError(f"Testsuite already contains a testsuite with same name '{testsuite._name}'.")
1429 testsuite._parent = self
1430 self._testsuites[testsuite._name] = testsuite
1432 def AddTestsuites(self, testsuites: Iterable[TestsuiteType]) -> None:
1433 """
1434 Add a list of test suites to the list of test suites.
1436 :param testsuites: List of test suites to add.
1437 :raises ValueError: If parameter 'testsuites' is None.
1438 :raises TypeError: If parameter 'testsuites' is not iterable.
1439 """
1440 if testsuites is None: 1440 ↛ 1441line 1440 didn't jump to line 1441 because the condition on line 1440 was never true
1441 raise ValueError("Parameter 'testsuites' is None.")
1442 elif not isinstance(testsuites, Iterable): 1442 ↛ 1443line 1442 didn't jump to line 1443 because the condition on line 1442 was never true
1443 ex = TypeError("Parameter 'testsuites' is not iterable.")
1444 ex.add_note(f"Got type '{getFullyQualifiedName(testsuites)}'.")
1445 raise ex
1447 for testsuite in testsuites:
1448 self.AddTestsuite(testsuite)
1450 @abstractmethod
1451 def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]:
1452 pass
1454 def IterateTestsuites(self, scheme: IterationScheme = IterationScheme.TestsuiteDefault) -> Generator[TestsuiteType, None, None]:
1455 return self.Iterate(scheme)
1457 def IterateTestcases(self, scheme: IterationScheme = IterationScheme.TestcaseDefault) -> Generator[Testcase, None, None]:
1458 return self.Iterate(scheme)
1460 def ToTree(self) -> Node:
1461 rootNode = Node(value=self._name)
1463 def convertTestcase(testcase: Testcase, parentNode: Node) -> None:
1464 _ = Node(value=testcase._name, parent=parentNode)
1466 def convertTestsuite(testsuite: Testsuite, parentNode: Node) -> None:
1467 testsuiteNode = Node(value=testsuite._name, parent=parentNode)
1469 for ts in testsuite._testsuites.values():
1470 convertTestsuite(ts, testsuiteNode)
1472 for tc in testsuite._testcases.values():
1473 convertTestcase(tc, testsuiteNode)
1475 for testsuite in self._testsuites.values():
1476 convertTestsuite(testsuite, rootNode)
1478 return rootNode
1481@export
1482class Testsuite(TestsuiteBase[TestsuiteType]):
1483 """
1484 A testsuite is a mid-level element in the test entity hierarchy representing a group of tests.
1486 Test suites contain test cases and optionally other test suites. Test suites can be grouped by test suites to form a
1487 hierarchy of test entities. The root of the hierarchy is a test summary.
1488 """
1490 _testcases: Dict[str, Testcase]
1491 _hostname: Nullable[str]
1493 def __init__(
1494 self,
1495 name: str,
1496 title: Nullable[str] = None,
1497 summary: Nullable[str] = None,
1498 description: Nullable[str] = None,
1499 kind: TestsuiteKind = TestsuiteKind.Logical,
1500 hostname: Nullable[str] = None,
1501 startTime: Nullable[datetime] = None,
1502 setupDuration: Nullable[timedelta] = None,
1503 testDuration: Nullable[timedelta] = None,
1504 teardownDuration: Nullable[timedelta] = None,
1505 totalDuration: Nullable[timedelta] = None,
1506 status: TestsuiteStatus = TestsuiteStatus.Unknown,
1507 warningCount: int = 0,
1508 errorCount: int = 0,
1509 fatalCount: int = 0,
1510 testsuites: Nullable[Iterable[TestsuiteType]] = None,
1511 testcases: Nullable[Iterable[Testcase]] = None,
1512 keyValuePairs: Nullable[Mapping[str, Any]] = None,
1513 *,
1514 parent: Nullable[TestsuiteType] = None
1515 ) -> None:
1516 """
1517 Initializes the fields of a test suite.
1519 :param name: Name of the test suite.
1520 :param title: Optional, short label of the test suite, written for a reader.
1521 :param summary: Optional, summary of the test suite.
1522 :param description: Optional, description of the test suite.
1523 :param kind: Optional, kind of the test suite.
1524 :param hostname: Optional, name of the host the test suite was executed on, or ``None`` if it wasn't
1525 recorded.
1526 :param startTime: Optional, time when the test suite was started.
1527 :param setupDuration: Optional, duration it took to set up the test suite.
1528 :param testDuration: Optional, duration of all tests listed in the test suite.
1529 :param teardownDuration: Optional, duration it took to tear down the test suite.
1530 :param totalDuration: Optional, total duration of the entity's execution (setup + test + teardown)
1531 :param status: Optional, overall status of the test suite.
1532 :param warningCount: Optional, count of encountered warnings incl. warnings from sub-elements.
1533 :param errorCount: Optional, count of encountered errors incl. errors from sub-elements.
1534 :param fatalCount: Optional, count of encountered fatal errors incl. fatal errors from sub-elements.
1535 :param testsuites: Optional, list of test suites to initialize the test suite with.
1536 :param testcases: Optional, list of test cases to initialize the test suite with.
1537 :param keyValuePairs: Optional, mapping of key-value pairs to initialize the test suite with.
1538 :param parent: Optional, reference to the parent test entity.
1539 :raises TypeError: If parameter 'testcases' is not iterable.
1540 :raises TypeError: If element in parameter 'testcases' is not a Testcase.
1541 :raises AlreadyInTestsuiteError: If a test case in parameter 'testcases' is already part of a test entity hierarchy.
1542 :raises DuplicateTestcaseError: If a test case in parameter 'testcases' is already listed (by name) in the list of
1543 test cases.
1544 """
1545 super().__init__(
1546 name,
1547 title, summary, description,
1548 kind,
1549 startTime,
1550 setupDuration,
1551 testDuration,
1552 teardownDuration,
1553 totalDuration,
1554 status,
1555 warningCount,
1556 errorCount,
1557 fatalCount,
1558 testsuites,
1559 keyValuePairs,
1560 parent=parent
1561 )
1563 # self._testDuration = testDuration
1564 self._hostname = hostname
1566 self._testcases = {}
1567 if testcases is not None:
1568 if not isinstance(testcases, Iterable): 1568 ↛ 1569line 1568 didn't jump to line 1569 because the condition on line 1568 was never true
1569 ex = TypeError("Parameter 'testcases' is not iterable.")
1570 ex.add_note(f"Got type '{getFullyQualifiedName(testcases)}'.")
1571 raise ex
1573 for testcase in testcases:
1574 if not isinstance(testcase, Testcase): 1574 ↛ 1575line 1574 didn't jump to line 1575 because the condition on line 1574 was never true
1575 ex = TypeError("Element of parameter 'testcases' is not of type 'Testcase'.")
1576 ex.add_note(f"Got type '{getFullyQualifiedName(testcase)}'.")
1577 raise ex
1579 if testcase._parent is not None: 1579 ↛ 1580line 1579 didn't jump to line 1580 because the condition on line 1579 was never true
1580 raise AlreadyInTestsuiteError(f"Testcase '{testcase._name}' is already part of a testsuite hierarchy.")
1582 if testcase._name in self._testcases:
1583 raise DuplicateTestcaseError(f"Testsuite already contains a testcase with same name '{testcase._name}'.")
1585 testcase._parent = self
1586 self._testcases[testcase._name] = testcase
1588 @readonly
1589 def Testcases(self) -> Dict[str, Testcase]:
1590 """
1591 Read-only property to access a reference to the internal dictionary of test cases.
1593 :returns: Reference to the dictionary of test cases.
1594 """
1595 return self._testcases
1597 @readonly
1598 def TestcaseCount(self) -> int:
1599 """
1600 Read-only property to return the number of all test cases in the test entity hierarchy.
1602 :returns: Number of test cases.
1603 """
1604 return super().TestcaseCount + len(self._testcases)
1606 @readonly
1607 def AssertionCount(self) -> int:
1608 """
1609 Read-only property to return the number of assertions in this testsuite and its testcases.
1611 :returns: Sum of the inherited assertion count and the testcases' assertion counts.
1612 """
1613 return super().AssertionCount + sum(tc.AssertionCount for tc in self._testcases.values())
1615 @readonly
1616 def Hostname(self) -> Nullable[str]:
1617 """
1618 Read-only property to access the name of the host this test suite was executed on (:attr:`_hostname`).
1620 :returns: The hostname, or ``None`` if it wasn't recorded.
1621 """
1622 return self._hostname
1624 def Copy(self) -> Testsuite:
1625 return self.__class__(
1626 self._name,
1627 self._title, self._summary, self._description,
1628 self._kind,
1629 self._hostname,
1630 self._startTime,
1631 self._setupDuration,
1632 self._testDuration,
1633 self._teardownDuration,
1634 self._totalDuration,
1635 self._status,
1636 self._warningCount,
1637 self._errorCount,
1638 self._fatalCount
1639 )
1641 def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType:
1642 (
1643 tests, inconsistent, excluded, skipped, errored, weak, failed, passed,
1644 warningCount, errorCount, fatalCount,
1645 expectedWarningCount, expectedErrorCount, expectedFatalCount,
1646 totalDuration
1647 ) = super().Aggregate(strict)
1649 for testcase in self._testcases.values():
1650 wc, ec, fc, ewc, eec, efc, td = testcase.Aggregate(strict)
1652 tests += 1
1654 warningCount += wc
1655 errorCount += ec
1656 fatalCount += fc
1658 expectedWarningCount += ewc
1659 expectedErrorCount += eec
1660 expectedFatalCount += efc
1662 totalDuration += td
1664 status = testcase._status
1665 if status is TestcaseStatus.Unknown: 1665 ↛ 1666line 1665 didn't jump to line 1666 because the condition on line 1665 was never true
1666 raise UnittestError(f"Found testcase '{testcase._name}' with state 'Unknown'.")
1667 elif TestcaseStatus.Inconsistent in status:
1668 inconsistent += 1
1669 elif status is TestcaseStatus.Excluded: 1669 ↛ 1670line 1669 didn't jump to line 1670 because the condition on line 1669 was never true
1670 excluded += 1
1671 elif status is TestcaseStatus.Skipped:
1672 skipped += 1
1673 elif status is TestcaseStatus.Errored:
1674 errored += 1
1675 elif status is TestcaseStatus.Weak: 1675 ↛ 1676line 1675 didn't jump to line 1676 because the condition on line 1675 was never true
1676 weak += 1
1677 elif status is TestcaseStatus.Passed:
1678 passed += 1
1679 elif status is TestcaseStatus.Failed:
1680 failed += 1
1681 elif status is TestcaseStatus.ExpectedFailed:
1682 passed += 1
1683 elif status is TestcaseStatus.UnexpectedPassed: 1683 ↛ 1685line 1683 didn't jump to line 1685 because the condition on line 1683 was always true
1684 failed += 1
1685 elif status & TestcaseStatus.Mask is not TestcaseStatus.Unknown:
1686 raise UnittestError(f"Found testcase '{testcase._name}' with unsupported state '{status}'.")
1687 else:
1688 raise UnittestError(f"Internal error for testcase '{testcase._name}', field '_status' is '{status}'.")
1690 self._tests = tests
1691 self._inconsistent = inconsistent
1692 self._excluded = excluded
1693 self._skipped = skipped
1694 self._errored = errored
1695 self._weak = weak
1696 self._failed = failed
1697 self._passed = passed
1699 self._warningCount = warningCount
1700 self._errorCount = errorCount
1701 self._fatalCount = fatalCount
1703 self._expectedWarningCount = expectedWarningCount
1704 self._expectedErrorCount = expectedErrorCount
1705 self._expectedFatalCount = expectedFatalCount
1707 if self._totalDuration is None:
1708 self._totalDuration = totalDuration
1710 if errored > 0:
1711 self._status = TestsuiteStatus.Errored
1712 elif failed > 0:
1713 self._status = TestsuiteStatus.Failed
1714 elif tests == 0:
1715 self._status = TestsuiteStatus.Empty
1716 elif tests - skipped == passed: 1716 ↛ 1718line 1716 didn't jump to line 1718 because the condition on line 1716 was always true
1717 self._status = TestsuiteStatus.Passed
1718 elif tests == skipped:
1719 self._status = TestsuiteStatus.Skipped
1720 else:
1721 self._status = TestsuiteStatus.Unknown
1723 return tests, inconsistent, excluded, skipped, errored, weak, failed, passed, warningCount, errorCount, fatalCount, expectedWarningCount, expectedErrorCount, expectedFatalCount, totalDuration
1725 def AddTestcase(self, testcase: Testcase) -> None:
1726 """
1727 Add a test case to the list of test cases.
1729 :param testcase: The test case to add.
1730 :raises ValueError: If parameter 'testcase' is None.
1731 :raises TypeError: If parameter 'testcase' is not a Testcase.
1732 :raises AlreadyInTestsuiteError: If parameter 'testcase' is already part of a test entity hierarchy.
1733 :raises DuplicateTestcaseError: If parameter 'testcase' is already listed (by name) in the list of test cases.
1734 """
1735 if testcase is None: 1735 ↛ 1736line 1735 didn't jump to line 1736 because the condition on line 1735 was never true
1736 raise ValueError("Parameter 'testcase' is None.")
1737 elif not isinstance(testcase, Testcase): 1737 ↛ 1738line 1737 didn't jump to line 1738 because the condition on line 1737 was never true
1738 ex = TypeError("Parameter 'testcase' is not of type 'Testcase'.")
1739 ex.add_note(f"Got type '{getFullyQualifiedName(testcase)}'.")
1740 raise ex
1742 if testcase._parent is not None:
1743 raise AlreadyInTestsuiteError(f"Testcase '{testcase._name}' is already part of a testsuite hierarchy.")
1745 if testcase._name in self._testcases:
1746 raise DuplicateTestcaseError(f"Testsuite already contains a testcase with same name '{testcase._name}'.")
1748 testcase._parent = self
1749 self._testcases[testcase._name] = testcase
1751 def AddTestcases(self, testcases: Iterable[Testcase]) -> None:
1752 """
1753 Add a list of test cases to the list of test cases.
1755 :param testcases: List of test cases to add.
1756 :raises ValueError: If parameter 'testcases' is None.
1757 :raises TypeError: If parameter 'testcases' is not iterable.
1758 """
1759 if testcases is None: 1759 ↛ 1760line 1759 didn't jump to line 1760 because the condition on line 1759 was never true
1760 raise ValueError("Parameter 'testcases' is None.")
1761 elif not isinstance(testcases, Iterable): 1761 ↛ 1762line 1761 didn't jump to line 1762 because the condition on line 1761 was never true
1762 ex = TypeError("Parameter 'testcases' is not iterable.")
1763 ex.add_note(f"Got type '{getFullyQualifiedName(testcases)}'.")
1764 raise ex
1766 for testcase in testcases:
1767 self.AddTestcase(testcase)
1769 def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]:
1770 if IterationScheme.PreOrder in scheme:
1771 if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites in scheme:
1772 yield self
1774 if IterationScheme.IncludeTestcases in scheme:
1775 for testcase in self._testcases.values():
1776 yield testcase
1778 for testsuite in self._testsuites.values():
1779 yield from testsuite.Iterate(scheme | IterationScheme.IncludeSelf)
1781 if IterationScheme.PostOrder in scheme:
1782 if IterationScheme.IncludeTestcases in scheme:
1783 for testcase in self._testcases.values():
1784 yield testcase
1786 if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites in scheme:
1787 yield self
1789 def __str__(self) -> str:
1790 return (
1791 f"<Testsuite {self._name}: {self._status.name} -"
1792 # f" assert/pass/fail:{self._assertionCount}/{self._passedAssertionCount}/{self._failedAssertionCount} -"
1793 f" warn/error/fatal:{self._warningCount}/{self._errorCount}/{self._fatalCount}>"
1794 )
1797@export
1798class TestsuiteSummary(TestsuiteBase[TestsuiteType]):
1799 """
1800 A testsuite summary is the root element in the test entity hierarchy representing a summary of all test suites and cases.
1802 The testsuite summary contains test suites, which in turn can contain test suites and test cases.
1803 """
1805 def __init__(
1806 self,
1807 name: str,
1808 title: Nullable[str] = None,
1809 summary: Nullable[str] = None,
1810 description: Nullable[str] = None,
1811 startTime: Nullable[datetime] = None,
1812 setupDuration: Nullable[timedelta] = None,
1813 testDuration: Nullable[timedelta] = None,
1814 teardownDuration: Nullable[timedelta] = None,
1815 totalDuration: Nullable[timedelta] = None,
1816 status: TestsuiteStatus = TestsuiteStatus.Unknown,
1817 warningCount: int = 0,
1818 errorCount: int = 0,
1819 fatalCount: int = 0,
1820 testsuites: Nullable[Iterable[TestsuiteType]] = None,
1821 keyValuePairs: Nullable[Mapping[str, Any]] = None,
1822 *,
1823 parent: Nullable[TestsuiteType] = None
1824 ) -> None:
1825 """
1826 Initializes the fields of a test summary.
1828 :param name: Name of the test summary.
1829 :param title: Optional, short label of the test summary, written for a reader.
1830 :param summary: Optional, summary of the test summary.
1831 :param description: Optional, description of the test summary.
1832 :param startTime: Optional, time when the test summary was started.
1833 :param setupDuration: Optional, duration it took to set up the test summary.
1834 :param testDuration: Optional, duration of all tests listed in the test summary.
1835 :param teardownDuration: Optional, duration it took to tear down the test summary.
1836 :param totalDuration: Optional, total duration of the entity's execution (setup + test + teardown)
1837 :param status: Optional, overall status of the test summary.
1838 :param warningCount: Optional, count of encountered warnings incl. warnings from sub-elements.
1839 :param errorCount: Optional, count of encountered errors incl. errors from sub-elements.
1840 :param fatalCount: Optional, count of encountered fatal errors incl. fatal errors from sub-elements.
1841 :param testsuites: Optional, list of test suites to initialize the test summary with.
1842 :param keyValuePairs: Optional, mapping of key-value pairs to initialize the test summary with.
1843 :param parent: Optional, reference to the parent test summary.
1844 """
1845 super().__init__(
1846 name,
1847 title, summary, description,
1848 TestsuiteKind.Root,
1849 startTime, setupDuration, testDuration, teardownDuration, totalDuration,
1850 status,
1851 warningCount, errorCount, fatalCount,
1852 testsuites,
1853 keyValuePairs,
1854 parent=parent
1855 )
1857 def Aggregate(self, strict: bool = True) -> TestsuiteAggregateReturnType:
1858 tests, inconsistent, excluded, skipped, errored, weak, failed, passed, warningCount, errorCount, fatalCount, expectedWarningCount, expectedErrorCount, expectedFatalCount, totalDuration = super().Aggregate(strict)
1860 self._tests = tests
1861 self._inconsistent = inconsistent
1862 self._excluded = excluded
1863 self._skipped = skipped
1864 self._errored = errored
1865 self._weak = weak
1866 self._failed = failed
1867 self._passed = passed
1869 self._warningCount = warningCount
1870 self._errorCount = errorCount
1871 self._fatalCount = fatalCount
1873 self._expectedWarningCount = expectedWarningCount
1874 self._expectedErrorCount = expectedErrorCount
1875 self._expectedFatalCount = expectedFatalCount
1877 if self._totalDuration is None:
1878 self._totalDuration = totalDuration
1880 if errored > 0:
1881 self._status = TestsuiteStatus.Errored
1882 elif failed > 0:
1883 self._status = TestsuiteStatus.Failed
1884 elif tests == 0:
1885 self._status = TestsuiteStatus.Empty
1886 elif tests - skipped == passed: 1886 ↛ 1888line 1886 didn't jump to line 1888 because the condition on line 1886 was always true
1887 self._status = TestsuiteStatus.Passed
1888 elif tests == skipped:
1889 self._status = TestsuiteStatus.Skipped
1890 elif tests == excluded:
1891 self._status = TestsuiteStatus.Excluded
1892 else:
1893 self._status = TestsuiteStatus.Unknown
1895 return (
1896 tests, inconsistent, excluded, skipped, errored, weak, failed, passed,
1897 warningCount, errorCount, fatalCount,
1898 expectedWarningCount, expectedErrorCount, expectedFatalCount,
1899 totalDuration
1900 )
1902 def Iterate(self, scheme: IterationScheme = IterationScheme.Default) -> Generator[Union[TestsuiteType, Testcase], None, None]:
1903 if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PreOrder in scheme:
1904 yield self
1906 for testsuite in self._testsuites.values():
1907 yield from testsuite.IterateTestsuites(scheme | IterationScheme.IncludeSelf)
1909 if IterationScheme.IncludeSelf | IterationScheme.IncludeTestsuites | IterationScheme.PostOrder in scheme: 1909 ↛ 1910line 1909 didn't jump to line 1910 because the condition on line 1909 was never true
1910 yield self
1912 def __str__(self) -> str:
1913 return (
1914 f"<TestsuiteSummary {self._name}: {self._status.name} -"
1915 # f" assert/pass/fail:{self._assertionCount}/{self._passedAssertionCount}/{self._failedAssertionCount} -"
1916 f" warn/error/fatal:{self._warningCount}/{self._errorCount}/{self._fatalCount}>"
1917 )
1920@export
1921class Document(metaclass=ExtendedType, mixin=True):
1922 """A mixin-class representing a unit test summary document (file)."""
1924 _path: Path #: Path to the report file.
1926 _analysisDuration: Nullable[timedelta] #: Duration of :meth:`Analyze`, or ``None`` before it ran.
1927 _modelConversion: Nullable[timedelta] #: Duration of :meth:`Convert`, or ``None`` before it ran.
1929 def __init__(self, reportFile: Path, analyzeAndConvert: bool = False) -> None:
1930 self._path = reportFile
1932 self._analysisDuration = None
1933 self._modelConversion = None
1935 if analyzeAndConvert:
1936 self.Analyze()
1937 self.Convert()
1939 @readonly
1940 def Path(self) -> Path:
1941 """
1942 Read-only property to access the path to the file of this document.
1944 :returns: The document's path to the file.
1945 """
1946 return self._path
1948 @readonly
1949 def AnalysisDuration(self) -> Nullable[timedelta]:
1950 """
1951 Read-only property to access the analysis duration (:attr:`_analysisDuration`).
1953 .. note::
1955 This includes usually the duration to validate and parse the file format, but it excludes the time to convert the
1956 content to the test entity hierarchy.
1958 :returns: Duration to analyze the document, or ``None`` if it wasn't analyzed yet.
1959 """
1960 return self._analysisDuration
1962 @readonly
1963 def ModelConversionDuration(self) -> Nullable[timedelta]:
1964 """
1965 Read-only property to access the conversion duration (:attr:`_modelConversion`).
1967 .. note::
1969 This includes usually the duration to convert the document's content to the test entity hierarchy. It might also
1970 include the duration to (re-)aggregate all states and statistics in the hierarchy.
1972 :returns: Duration to convert the document, or ``None`` if it wasn't converted yet.
1973 """
1974 return self._modelConversion
1976 @abstractmethod
1977 def Analyze(self) -> None:
1978 """Analyze and validate the document's content."""
1980 # @abstractmethod
1981 # def Write(self, path: Nullable[Path] = None, overwrite: bool = False):
1982 # pass
1984 @abstractmethod
1985 def Convert(self):
1986 """Convert the document's content to an instance of the test entity hierarchy."""
1989@export
1990class Merged(metaclass=ExtendedType, mixin=True):
1991 """A mixin-class representing a merged test entity."""
1993 _mergedCount: int
1995 def __init__(self, mergedCount: int = 1) -> None:
1996 self._mergedCount = mergedCount
1998 def _MergeStartTime(self, otherStartTime: Nullable[datetime]) -> Nullable[datetime]:
1999 """
2000 Combine this entity's start time with the start time of an entity being merged in.
2002 A merged entity started when the earliest of its parts started. An entity without a start time contributes
2003 nothing rather than erasing what is known.
2005 :param otherStartTime: The start time of the entity being merged in.
2006 :returns: The earlier of the two start times, or ``None`` if neither has one.
2007 """
2008 if otherStartTime is None:
2009 return self._startTime
2010 elif self._startTime is None or otherStartTime < self._startTime:
2011 return otherStartTime
2012 else:
2013 return self._startTime
2015 def _MergeTexts(self, other: Base) -> None:
2016 """
2017 Take over the title, summary and description of an entity being merged in, where this entity has none.
2019 :param other: The entity being merged in.
2020 """
2021 if self._title is None: 2021 ↛ 2024line 2021 didn't jump to line 2024 because the condition on line 2021 was always true
2022 self._title = other._title
2024 if self._summary is None: 2024 ↛ 2027line 2024 didn't jump to line 2027 because the condition on line 2024 was always true
2025 self._summary = other._summary
2027 if self._description is None: 2027 ↛ exitline 2027 didn't return from function '_MergeTexts' because the condition on line 2027 was always true
2028 self._description = other._description
2030 @readonly
2031 def MergedCount(self) -> int:
2032 """
2033 Read-only property to access how many source entities were merged into this one.
2035 :returns: Number of merged entities.
2036 """
2037 return self._mergedCount
2040@export
2041class Combined(metaclass=ExtendedType, mixin=True):
2042 _combinedCount: int
2044 def __init__(self, combinedCound: int = 1) -> None:
2045 self._combinedCount = combinedCound
2047 @readonly
2048 def CombinedCount(self) -> int:
2049 """
2050 Read-only property to access how many source entities were combined into this one.
2052 :returns: Number of combined entities.
2053 """
2054 return self._combinedCount
2057@export
2058class MergedTestcase(Testcase, Merged):
2059 """
2060 A test case merged from the same test case found in multiple test reports.
2062 The message, details and captured output streams (see :class:`TestcaseOutputMixin`) as well as the title, summary
2063 and description are taken from the first merged test case that has them.
2064 """
2066 _mergedTestcases: List[Testcase] #: List of test cases merged into this test case.
2068 def __init__(
2069 self,
2070 testcase: Testcase,
2071 *,
2072 parent: Nullable[Testsuite] = None
2073 ) -> None:
2074 if testcase is None: 2074 ↛ 2075line 2074 didn't jump to line 2075 because the condition on line 2074 was never true
2075 raise ValueError("Parameter 'testcase' is None.")
2077 super().__init__(
2078 testcase._name,
2079 testcase._title, testcase._summary, testcase._description,
2080 testcase._startTime,
2081 testcase._setupDuration, testcase._testDuration, testcase._teardownDuration, testcase._totalDuration,
2082 TestcaseStatus.Unknown,
2083 testcase._assertionCount, testcase._failedAssertionCount, testcase._passedAssertionCount,
2084 testcase._warningCount, testcase._errorCount, testcase._fatalCount,
2085 testcase._expectedWarningCount, testcase._expectedErrorCount, testcase._expectedFatalCount,
2086 message=testcase._message,
2087 details=testcase._details,
2088 standardOutput=testcase._standardOutput,
2089 standardError=testcase._standardError,
2090 parent=parent
2091 )
2092 Merged.__init__(self)
2094 self._mergedTestcases = [testcase]
2096 @readonly
2097 def Status(self) -> TestcaseStatus:
2098 """
2099 Read-only property to access the status merged from all source testcases.
2101 :returns: Merged status. ``TestcaseStatus.Inconsistent`` if the sources disagree.
2102 """
2103 if self._status is TestcaseStatus.Unknown: 2103 ↛ 2110line 2103 didn't jump to line 2110 because the condition on line 2103 was always true
2104 status = self._mergedTestcases[0]._status
2105 for mtc in self._mergedTestcases[1:]:
2106 status @= mtc._status
2108 self._status = status
2110 return self._status
2112 @readonly
2113 def SummedAssertionCount(self) -> int:
2114 """
2115 Read-only property to return the number of assertions across all merged testcases.
2117 :returns: Sum of the merged testcases' assertion counts.
2118 """
2119 return sum(tc._assertionCount for tc in self._mergedTestcases)
2121 @readonly
2122 def SummedPassedAssertionCount(self) -> int:
2123 """
2124 Read-only property to return the number of passed assertions across all merged testcases.
2126 :returns: Sum of the merged testcases' passed assertion counts.
2127 """
2128 return sum(tc._passedAssertionCount for tc in self._mergedTestcases)
2130 @readonly
2131 def SummedFailedAssertionCount(self) -> int:
2132 """
2133 Read-only property to return the number of failed assertions across all merged testcases.
2135 :returns: Sum of the merged testcases' failed assertion counts.
2136 """
2137 return sum(tc._failedAssertionCount for tc in self._mergedTestcases)
2139 def Aggregate(self, strict: bool = True) -> TestcaseAggregateReturnType:
2140 firstMTC = self._mergedTestcases[0]
2142 status = firstMTC._status
2143 warningCount = firstMTC._warningCount
2144 errorCount = firstMTC._errorCount
2145 fatalCount = firstMTC._fatalCount
2146 totalDuration = firstMTC._totalDuration
2148 for mtc in self._mergedTestcases[1:]:
2149 status @= mtc._status
2150 warningCount += mtc._warningCount
2151 errorCount += mtc._errorCount
2152 fatalCount += mtc._fatalCount
2154 self._status = status
2156 return warningCount, errorCount, fatalCount, self._expectedWarningCount, self._expectedErrorCount, self._expectedFatalCount, totalDuration
2158 def Merge(self, tc: Testcase) -> None:
2159 """
2160 Merge another occurrence of this test case into this merged test case.
2162 Warning, error and fatal counts are summed up. The message, details, captured output streams, title, summary and
2163 description are kept from the first merged test case that has them.
2165 :param tc: The test case to merge.
2166 """
2167 self._mergedCount += 1
2169 self._mergedTestcases.append(tc)
2171 self._warningCount += tc._warningCount
2172 self._errorCount += tc._errorCount
2173 self._fatalCount += tc._fatalCount
2175 if self._message is None:
2176 self._message = tc._message
2178 if self._details is None:
2179 self._details = tc._details
2181 if self._standardOutput is None:
2182 self._standardOutput = tc._standardOutput
2184 if self._standardError is None: 2184 ↛ 2187line 2184 didn't jump to line 2187 because the condition on line 2184 was always true
2185 self._standardError = tc._standardError
2187 self._MergeTexts(tc)
2189 def ToTestcase(self) -> Testcase:
2190 return Testcase(
2191 self._name,
2192 self._title, self._summary, self._description,
2193 self._startTime,
2194 self._setupDuration,
2195 self._testDuration,
2196 self._teardownDuration,
2197 self._totalDuration,
2198 self._status,
2199 self._assertionCount,
2200 self._failedAssertionCount,
2201 self._passedAssertionCount,
2202 self._warningCount,
2203 self._errorCount,
2204 self._fatalCount,
2205 message=self._message,
2206 details=self._details,
2207 standardOutput=self._standardOutput,
2208 standardError=self._standardError
2209 )
2212@export
2213class MergedTestsuite(Testsuite, Merged):
2214 def __init__(
2215 self,
2216 testsuite: Testsuite,
2217 addTestsuites: bool = False,
2218 addTestcases: bool = False,
2219 *,
2220 parent: Nullable[Testsuite] = None
2221 ) -> None:
2222 if testsuite is None: 2222 ↛ 2223line 2222 didn't jump to line 2223 because the condition on line 2222 was never true
2223 raise ValueError("Parameter 'testsuite' is None.")
2225 super().__init__(
2226 testsuite._name,
2227 testsuite._title, testsuite._summary, testsuite._description,
2228 testsuite._kind,
2229 testsuite._hostname,
2230 testsuite._startTime,
2231 testsuite._setupDuration, testsuite._testDuration, testsuite._teardownDuration, testsuite._totalDuration,
2232 TestsuiteStatus.Unknown,
2233 testsuite._warningCount, testsuite._errorCount, testsuite._fatalCount,
2234 parent=parent
2235 )
2236 Merged.__init__(self)
2238 if addTestsuites: 2238 ↛ 2243line 2238 didn't jump to line 2243 because the condition on line 2238 was always true
2239 for ts in testsuite._testsuites.values():
2240 mergedTestsuite = MergedTestsuite(ts, addTestsuites, addTestcases)
2241 self.AddTestsuite(mergedTestsuite)
2243 if addTestcases: 2243 ↛ exitline 2243 didn't return from function '__init__' because the condition on line 2243 was always true
2244 for tc in testsuite._testcases.values():
2245 mergedTestcase = MergedTestcase(tc)
2246 self.AddTestcase(mergedTestcase)
2248 def _MergeHostname(self, otherHostname: Nullable[str]) -> Nullable[str]:
2249 """
2250 Combine this test suite's hostname with the hostname of a test suite being merged in.
2252 A test suite without a hostname ran somewhere unrecorded, not somewhere else, so it keeps an otherwise
2253 unanimous hostname.
2255 :param otherHostname: The hostname of the test suite being merged in.
2256 :returns: The common hostname, ``"various"`` if they disagree, or ``None`` if neither has one.
2257 """
2258 if otherHostname is None:
2259 return self._hostname
2260 elif self._hostname is None or self._hostname == otherHostname:
2261 return otherHostname
2262 else:
2263 return "various"
2265 def Merge(self, testsuite: Testsuite) -> None:
2266 self._mergedCount += 1
2267 self._hostname = self._MergeHostname(testsuite._hostname)
2268 self._startTime = self._MergeStartTime(testsuite._startTime)
2269 self._MergeTexts(testsuite)
2271 for ts in testsuite._testsuites.values():
2272 if ts._name in self._testsuites: 2272 ↛ 2275line 2272 didn't jump to line 2275 because the condition on line 2272 was always true
2273 self._testsuites[ts._name].Merge(ts)
2274 else:
2275 mergedTestsuite = MergedTestsuite(ts, addTestsuites=True, addTestcases=True)
2276 self.AddTestsuite(mergedTestsuite)
2278 for tc in testsuite._testcases.values():
2279 if tc._name in self._testcases: 2279 ↛ 2282line 2279 didn't jump to line 2282 because the condition on line 2279 was always true
2280 self._testcases[tc._name].Merge(tc)
2281 else:
2282 mergedTestcase = MergedTestcase(tc)
2283 self.AddTestcase(mergedTestcase)
2285 def ToTestsuite(self) -> Testsuite:
2286 testsuite = Testsuite(
2287 self._name,
2288 self._title, self._summary, self._description,
2289 self._kind,
2290 self._hostname,
2291 self._startTime,
2292 self._setupDuration,
2293 self._testDuration,
2294 self._teardownDuration,
2295 self._totalDuration,
2296 self._status,
2297 self._warningCount,
2298 self._errorCount,
2299 self._fatalCount,
2300 testsuites=(ts.ToTestsuite() for ts in self._testsuites.values()),
2301 testcases=(tc.ToTestcase() for tc in self._testcases.values())
2302 )
2304 testsuite._tests = self._tests
2305 testsuite._excluded = self._excluded
2306 testsuite._inconsistent = self._inconsistent
2307 testsuite._skipped = self._skipped
2308 testsuite._errored = self._errored
2309 testsuite._weak = self._weak
2310 testsuite._failed = self._failed
2311 testsuite._passed = self._passed
2313 return testsuite
2316@export
2317class MergedTestsuiteSummary(TestsuiteSummary, Merged):
2318 _mergedFiles: Dict[Path, TestsuiteSummary]
2320 def __init__(self, name: str) -> None:
2321 super().__init__(name)
2322 Merged.__init__(self, mergedCount=0)
2324 self._mergedFiles = {}
2326 def Merge(self, testsuiteSummary: TestsuiteSummary) -> None:
2327 # if summary.File in self._mergedFiles:
2328 # raise
2330 # FIXME: a summary is not necessarily a file
2331 self._mergedCount += 1
2332 self._mergedFiles[testsuiteSummary._name] = testsuiteSummary
2333 self._startTime = self._MergeStartTime(testsuiteSummary._startTime)
2334 self._MergeTexts(testsuiteSummary)
2336 for testsuite in testsuiteSummary._testsuites.values():
2337 if testsuite._name in self._testsuites:
2338 self._testsuites[testsuite._name].Merge(testsuite)
2339 else:
2340 mergedTestsuite = MergedTestsuite(testsuite, addTestsuites=True, addTestcases=True)
2341 self.AddTestsuite(mergedTestsuite)
2343 def ToTestsuiteSummary(self) -> TestsuiteSummary:
2344 testsuiteSummary = TestsuiteSummary(
2345 self._name,
2346 self._title, self._summary, self._description,
2347 self._startTime,
2348 self._setupDuration,
2349 self._testDuration,
2350 self._teardownDuration,
2351 self._totalDuration,
2352 self._status,
2353 self._warningCount,
2354 self._errorCount,
2355 self._fatalCount,
2356 testsuites=(ts.ToTestsuite() for ts in self._testsuites.values())
2357 )
2359 testsuiteSummary._tests = self._tests
2360 testsuiteSummary._excluded = self._excluded
2361 testsuiteSummary._inconsistent = self._inconsistent
2362 testsuiteSummary._skipped = self._skipped
2363 testsuiteSummary._errored = self._errored
2364 testsuiteSummary._weak = self._weak
2365 testsuiteSummary._failed = self._failed
2366 testsuiteSummary._passed = self._passed
2368 return testsuiteSummary