Coverage for pyEDAA/OSVVM/TestsuiteSummary.py: 66%
212 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-27 22:24 +0000
1# ==================================================================================================================== #
2# _____ ____ _ _ ___ ______ ____ ____ __ #
3# _ __ _ _| ____| _ \ / \ / \ / _ \/ ___\ \ / /\ \ / / \/ | #
4# | '_ \| | | | _| | | | |/ _ \ / _ \ | | | \___ \\ \ / / \ \ / /| |\/| | #
5# | |_) | |_| | |___| |_| / ___ \ / ___ \ | |_| |___) |\ V / \ V / | | | | #
6# | .__/ \__, |_____|____/_/ \_\/_/ \_(_)___/|____/ \_/ \_/ |_| |_| #
7# |_| |___/ #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2021-2025 Electronic Design Automation Abstraction (EDA²) #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""Reader for OSVVM test report summary files in YAML format."""
32from datetime import timedelta, datetime
33from pathlib import Path
34from typing import Optional as Nullable, Iterator, Iterable, Mapping, Any, List
36from ruamel.yaml import YAML, CommentedMap, CommentedSeq
37from pyTooling.Decorators import export, InheritDocString, notimplemented
38from pyTooling.MetaClasses import ExtendedType
39from pyTooling.Stopwatch import Stopwatch
40from pyTooling.Versioning import CalendarVersion, SemanticVersion
42from pyEDAA.Reports.Unittesting import UnittestException, Document, TestcaseStatus, TestsuiteStatus, TestsuiteType, TestsuiteKind
43from pyEDAA.Reports.Unittesting import TestsuiteSummary as ut_TestsuiteSummary, Testsuite as ut_Testsuite
44from pyEDAA.Reports.Unittesting import Testcase as ut_Testcase
47@export
48class OsvvmException:
49 pass
52@export
53@InheritDocString(UnittestException)
54class UnittestException(UnittestException, OsvvmException):
55 """@InheritDocString(UnittestException)"""
58@export
59@InheritDocString(ut_Testcase)
60class Testcase(ut_Testcase):
61 """@InheritDocString(ut_Testcase)"""
64@export
65@InheritDocString(ut_Testsuite)
66class Testsuite(ut_Testsuite):
67 """@InheritDocString(ut_Testsuite)"""
70@export
71class BuildInformation(metaclass=ExtendedType, slots=True):
72 _startTime: datetime
73 _finishTime: datetime
74 _elapsed: timedelta
75 _simulator: str
76 _simulatorVersion: SemanticVersion
77 _osvvmVersion: CalendarVersion
78 _buildErrorCode: int
79 _analyzeErrorCount: int
80 _simulateErrorCount: int
82 def __init__(self) -> None:
83 pass
86@export
87class Settings(metaclass=ExtendedType, slots=True):
88 _baseDirectory: Path
89 _reportsSubdirectory: Path
90 _simulationLogFile: Path
91 _simulationHtmlLogFile: Path
92 _requirementsSubdirectory: Path
93 _coverageSubdirectory: Path
94 _report2CssFiles: List[Path]
95 _report2PngFile: List[Path]
97 def __init__(self) -> None:
98 pass
101@export
102@InheritDocString(ut_TestsuiteSummary)
103class TestsuiteSummary(ut_TestsuiteSummary):
104 """@InheritDocString(ut_TestsuiteSummary)"""
106 _datetime: datetime
108 def __init__(
109 self,
110 name: str,
111 startTime: Nullable[datetime] = None,
112 setupDuration: Nullable[timedelta] = None,
113 testDuration: Nullable[timedelta] = None,
114 teardownDuration: Nullable[timedelta] = None,
115 totalDuration: Nullable[timedelta] = None,
116 status: TestsuiteStatus = TestsuiteStatus.Unknown,
117 warningCount: int = 0,
118 errorCount: int = 0,
119 fatalCount: int = 0,
120 testsuites: Nullable[Iterable[TestsuiteType]] = None,
121 keyValuePairs: Nullable[Mapping[str, Any]] = None,
122 parent: Nullable[TestsuiteType] = None
123 ) -> None:
124 """
125 Initializes the fields of a test summary.
127 :param name: Name of the test summary.
128 :param startTime: Time when the test summary was started.
129 :param setupDuration: Duration it took to set up the test summary.
130 :param testDuration: Duration of all tests listed in the test summary.
131 :param teardownDuration: Duration it took to tear down the test summary.
132 :param totalDuration: Total duration of the entity's execution (setup + test + teardown)
133 :param status: Overall status of the test summary.
134 :param warningCount: Count of encountered warnings incl. warnings from sub-elements.
135 :param errorCount: Count of encountered errors incl. errors from sub-elements.
136 :param fatalCount: Count of encountered fatal errors incl. fatal errors from sub-elements.
137 :param testsuites: List of test suites to initialize the test summary with.
138 :param keyValuePairs: Mapping of key-value pairs to initialize the test summary with.
139 :param parent: Reference to the parent test summary.
140 """
141 super().__init__(
142 name,
143 startTime,
144 setupDuration,
145 testDuration,
146 teardownDuration,
147 totalDuration,
148 status,
149 warningCount,
150 errorCount,
151 fatalCount,
152 testsuites,
153 keyValuePairs,
154 parent
155 )
158@export
159class BuildSummaryDocument(TestsuiteSummary, Document):
160 _yamlDocument: Nullable[YAML]
162 def __init__(self, yamlReportFile: Path, analyzeAndConvert: bool = False) -> None:
163 super().__init__("Unprocessed OSVVM YAML file")
165 self._yamlDocument = None
167 Document.__init__(self, yamlReportFile, analyzeAndConvert)
169 def Analyze(self) -> None:
170 """
171 Analyze the YAML file, parse the content into an YAML data structure.
173 .. hint::
175 The time spend for analysis will be made available via property :data:`AnalysisDuration`..
176 """
177 if not self._path.exists(): 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 raise UnittestException(f"OSVVM YAML file '{self._path}' does not exist.") \
179 from FileNotFoundError(f"File '{self._path}' not found.")
181 with Stopwatch() as sw:
182 try:
183 yamlReader = YAML()
184 self._yamlDocument = yamlReader.load(self._path)
185 except Exception as ex:
186 raise UnittestException(f"Couldn't open '{self._path}'.") from ex
188 self._analysisDuration = sw.Duration
190 @notimplemented
191 def Write(self, path: Nullable[Path] = None, overwrite: bool = False) -> None:
192 """
193 Write the data model as XML into a file adhering to the Any JUnit dialect.
195 :param path: Optional path to the YAML file, if internal path shouldn't be used.
196 :param overwrite: If true, overwrite an existing file.
197 :raises UnittestException: If the file cannot be overwritten.
198 :raises UnittestException: If the internal YAML data structure wasn't generated.
199 :raises UnittestException: If the file cannot be opened or written.
200 """
201 if path is None:
202 path = self._path
204 if not overwrite and path.exists():
205 raise UnittestException(f"OSVVM YAML file '{path}' can not be overwritten.") \
206 from FileExistsError(f"File '{path}' already exists.")
208 # if regenerate:
209 # self.Generate(overwrite=True)
211 if self._yamlDocument is None:
212 ex = UnittestException(f"Internal YAML document tree is empty and needs to be generated before write is possible.")
213 # ex.add_note(f"Call 'BuildSummaryDocument.Generate()' or 'BuildSummaryDocument.Write(..., regenerate=True)'.")
214 raise ex
216 # with path.open("w", encoding="utf-8") as file:
217 # self._yamlDocument.writexml(file, addindent="\t", encoding="utf-8", newl="\n")
219 @staticmethod
220 def _ParseSequenceFromYAML(node: CommentedMap, fieldName: str) -> Nullable[CommentedSeq]:
221 try:
222 value = node[fieldName]
223 except KeyError as ex:
224 newEx = UnittestException(f"Sequence field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
225 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
226 raise newEx from ex
228 if value is None: 228 ↛ 229line 228 didn't jump to line 229 because the condition on line 228 was never true
229 return ()
230 elif not isinstance(value, CommentedSeq): 230 ↛ 231line 230 didn't jump to line 231 because the condition on line 230 was never true
231 line = node._yaml_line_col.data[fieldName][0] + 1
232 ex = UnittestException(f"Field '{fieldName}' is not a sequence.") # TODO: from TypeError??
233 ex.add_note(f"Found type {value.__class__.__name__} at line {line}.")
234 raise ex
236 return value
238 @staticmethod
239 def _ParseMapFromYAML(node: CommentedMap, fieldName: str) -> Nullable[CommentedMap]:
240 try:
241 value = node[fieldName]
242 except KeyError as ex:
243 newEx = UnittestException(f"Dictionary field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
244 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
245 raise newEx from ex
247 if value is None: 247 ↛ 248line 247 didn't jump to line 248 because the condition on line 247 was never true
248 return {}
249 elif not isinstance(value, CommentedMap): 249 ↛ 250line 249 didn't jump to line 250 because the condition on line 249 was never true
250 line = node._yaml_line_col.data[fieldName][0] + 1
251 ex = UnittestException(f"Field '{fieldName}' is not a list.") # TODO: from TypeError??
252 ex.add_note(f"Type mismatch found for line {line}.")
253 raise ex
254 return value
256 @staticmethod
257 def _ParseStrFieldFromYAML(node: CommentedMap, fieldName: str) -> Nullable[str]:
258 try:
259 value = node[fieldName]
260 except KeyError as ex:
261 newEx = UnittestException(f"String field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
262 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
263 raise newEx from ex
265 if not isinstance(value, str): 265 ↛ 266line 265 didn't jump to line 266 because the condition on line 265 was never true
266 raise UnittestException(f"Field '{fieldName}' is not of type str.") # TODO: from TypeError??
268 return value
270 @staticmethod
271 def _ParseIntFieldFromYAML(node: CommentedMap, fieldName: str) -> Nullable[int]:
272 try:
273 value = node[fieldName]
274 except KeyError as ex:
275 newEx = UnittestException(f"Integer field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
276 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
277 raise newEx from ex
279 if not isinstance(value, int): 279 ↛ 280line 279 didn't jump to line 280 because the condition on line 279 was never true
280 raise UnittestException(f"Field '{fieldName}' is not of type int.") # TODO: from TypeError??
282 return value
284 @staticmethod
285 def _ParseDateFieldFromYAML(node: CommentedMap, fieldName: str) -> Nullable[datetime]:
286 try:
287 value = node[fieldName]
288 except KeyError as ex:
289 newEx = UnittestException(f"Date field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
290 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
291 raise newEx from ex
293 if not isinstance(value, datetime): 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true
294 raise UnittestException(f"Field '{fieldName}' is not of type datetime.") # TODO: from TypeError??
296 return value
298 @staticmethod
299 def _ParseDurationFieldFromYAML(node: CommentedMap, fieldName: str) -> Nullable[timedelta]:
300 try:
301 value = node[fieldName]
302 except KeyError as ex:
303 newEx = UnittestException(f"Duration field '{fieldName}' not found in node starting at line {node.lc.line + 1}.")
304 newEx.add_note(f"Available fields: {', '.join(key for key in node)}")
305 raise newEx from ex
307 if not isinstance(value, float): 307 ↛ 308line 307 didn't jump to line 308 because the condition on line 307 was never true
308 raise UnittestException(f"Field '{fieldName}' is not of type float.") # TODO: from TypeError??
310 return timedelta(seconds=value)
312 def Convert(self) -> None:
313 """
314 Convert the parsed YAML data structure into a test entity hierarchy.
316 This method converts the root element.
318 .. hint::
320 The time spend for model conversion will be made available via property :data:`ModelConversionDuration`.
322 :raises UnittestException: If XML was not read and parsed before.
323 """
324 if self._yamlDocument is None: 324 ↛ 325line 324 didn't jump to line 325 because the condition on line 324 was never true
325 ex = UnittestException(f"OSVVM YAML file '{self._path}' needs to be read and analyzed by a YAML parser.")
326 ex.add_note(f"Call 'Document.Analyze()' or create document using 'Document(path, parse=True)'.")
327 raise ex
329 with Stopwatch() as sw:
330 self._name = self._yamlDocument["Name"]
331 buildInfo = self._ParseMapFromYAML(self._yamlDocument, "BuildInfo")
332 self._startTime = self._ParseDateFieldFromYAML(buildInfo, "StartTime")
333 self._totalDuration = self._ParseDurationFieldFromYAML(buildInfo, "Elapsed")
335 if "TestSuites" in self._yamlDocument:
336 for yamlTestsuite in self._ParseSequenceFromYAML(self._yamlDocument, "TestSuites"):
337 self._ConvertTestsuite(self, yamlTestsuite)
339 self.Aggregate()
341 self._modelConversion = sw.Duration
343 def _ConvertTestsuite(self, parentTestsuite: Testsuite, yamlTestsuite: CommentedMap) -> None:
344 testsuiteName = self._ParseStrFieldFromYAML(yamlTestsuite, "Name")
345 totalDuration = self._ParseDurationFieldFromYAML(yamlTestsuite, "ElapsedTime")
347 testsuite = Testsuite(
348 testsuiteName,
349 totalDuration=totalDuration,
350 parent=parentTestsuite
351 )
353 # if yamlTestsuite['TestCases'] is not None:
354 for yamlTestcase in self._ParseSequenceFromYAML(yamlTestsuite, 'TestCases'):
355 self._ConvertTestcase(testsuite, yamlTestcase)
357 def _ConvertTestcase(self, parentTestsuite: Testsuite, yamlTestcase: CommentedMap) -> None:
358 testcaseName = self._ParseStrFieldFromYAML(yamlTestcase, "TestCaseName")
359 totalDuration = self._ParseDurationFieldFromYAML(yamlTestcase, "ElapsedTime")
360 yamlStatus = self._ParseStrFieldFromYAML(yamlTestcase, "Status").lower()
361 yamlResults = self._ParseMapFromYAML(yamlTestcase, "Results")
362 assertionCount = self._ParseIntFieldFromYAML(yamlResults, "AffirmCount")
363 passedAssertionCount = self._ParseIntFieldFromYAML(yamlResults, "PassedCount")
364 totalErrors = self._ParseIntFieldFromYAML(yamlResults, "TotalErrors")
365 yamlAlertCount = self._ParseMapFromYAML(yamlResults, "AlertCount")
366 warningCount = self._ParseIntFieldFromYAML(yamlAlertCount, "Warning")
367 errorCount = self._ParseIntFieldFromYAML(yamlAlertCount, "Error")
368 fatalCount = self._ParseIntFieldFromYAML(yamlAlertCount, "Failure")
370 # FIXME: write a Parse classmethod in enum
371 if yamlStatus == "passed": 371 ↛ 373line 371 didn't jump to line 373 because the condition on line 371 was always true
372 status = TestcaseStatus.Passed
373 elif yamlStatus == "skipped":
374 status = TestcaseStatus.Skipped
375 elif yamlStatus == "failed":
376 status = TestcaseStatus.Failed
377 else:
378 status = TestcaseStatus.Unknown
380 if totalErrors == warningCount + errorCount + fatalCount:
381 if warningCount > 0: 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true
382 status |= TestcaseStatus.Warned
383 if errorCount > 0: 383 ↛ 384line 383 didn't jump to line 384 because the condition on line 383 was never true
384 status |= TestcaseStatus.Errored
385 if fatalCount > 0: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true
386 status |= TestcaseStatus.Aborted
387 # else:
388 # status |= TestcaseStatus.Inconsistent
390 _ = Testcase(
391 testcaseName,
392 totalDuration=totalDuration,
393 assertionCount=assertionCount,
394 passedAssertionCount=passedAssertionCount,
395 warningCount=warningCount,
396 status=status,
397 errorCount=errorCount,
398 fatalCount=fatalCount,
399 parent=parentTestsuite
400 )
402 def __contains__(self, key: str) -> bool:
403 return key in self._testsuites
405 def __iter__(self) -> Iterator[Testsuite]:
406 return iter(self._testsuites.values())
408 def __getitem__(self, key: str) -> Testsuite:
409 return self._testsuites[key]
411 def __len__(self) -> int:
412 return self._testsuites.__len__()