Coverage for /opt/homebrew/lib/python3.11/site-packages/_pytest/junitxml.py: 27%
367 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
1"""Report test results in JUnit-XML format, for use with Jenkins and build
2integration servers.
4Based on initial code from Ross Lawley.
6Output conforms to
7https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd
8"""
9import functools
10import os
11import platform
12import re
13import xml.etree.ElementTree as ET
14from datetime import datetime
15from typing import Callable
16from typing import Dict
17from typing import List
18from typing import Match
19from typing import Optional
20from typing import Tuple
21from typing import Union
23import pytest
24from _pytest import nodes
25from _pytest import timing
26from _pytest._code.code import ExceptionRepr
27from _pytest._code.code import ReprFileLocation
28from _pytest.config import Config
29from _pytest.config import filename_arg
30from _pytest.config.argparsing import Parser
31from _pytest.fixtures import FixtureRequest
32from _pytest.reports import TestReport
33from _pytest.stash import StashKey
34from _pytest.terminal import TerminalReporter
37xml_key = StashKey["LogXML"]()
40def bin_xml_escape(arg: object) -> str:
41 r"""Visually escape invalid XML characters.
43 For example, transforms
44 'hello\aworld\b'
45 into
46 'hello#x07world#x08'
47 Note that the #xABs are *not* XML escapes - missing the ampersand «.
48 The idea is to escape visually for the user rather than for XML itself.
49 """
51 def repl(matchobj: Match[str]) -> str:
52 i = ord(matchobj.group())
53 if i <= 0xFF:
54 return "#x%02X" % i
55 else:
56 return "#x%04X" % i
58 # The spec range of valid chars is:
59 # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
60 # For an unknown(?) reason, we disallow #x7F (DEL) as well.
61 illegal_xml_re = (
62 "[^\u0009\u000A\u000D\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]"
63 )
64 return re.sub(illegal_xml_re, repl, str(arg))
67def merge_family(left, right) -> None:
68 result = {}
69 for kl, vl in left.items():
70 for kr, vr in right.items():
71 if not isinstance(vl, list):
72 raise TypeError(type(vl))
73 result[kl] = vl + vr
74 left.update(result)
77families = {}
78families["_base"] = {"testcase": ["classname", "name"]}
79families["_base_legacy"] = {"testcase": ["file", "line", "url"]}
81# xUnit 1.x inherits legacy attributes.
82families["xunit1"] = families["_base"].copy()
83merge_family(families["xunit1"], families["_base_legacy"])
85# xUnit 2.x uses strict base attributes.
86families["xunit2"] = families["_base"]
89class _NodeReporter:
90 def __init__(self, nodeid: Union[str, TestReport], xml: "LogXML") -> None:
91 self.id = nodeid
92 self.xml = xml
93 self.add_stats = self.xml.add_stats
94 self.family = self.xml.family
95 self.duration = 0.0
96 self.properties: List[Tuple[str, str]] = []
97 self.nodes: List[ET.Element] = []
98 self.attrs: Dict[str, str] = {}
100 def append(self, node: ET.Element) -> None:
101 self.xml.add_stats(node.tag)
102 self.nodes.append(node)
104 def add_property(self, name: str, value: object) -> None:
105 self.properties.append((str(name), bin_xml_escape(value)))
107 def add_attribute(self, name: str, value: object) -> None:
108 self.attrs[str(name)] = bin_xml_escape(value)
110 def make_properties_node(self) -> Optional[ET.Element]:
111 """Return a Junit node containing custom properties, if any."""
112 if self.properties:
113 properties = ET.Element("properties")
114 for name, value in self.properties:
115 properties.append(ET.Element("property", name=name, value=value))
116 return properties
117 return None
119 def record_testreport(self, testreport: TestReport) -> None:
120 names = mangle_test_address(testreport.nodeid)
121 existing_attrs = self.attrs
122 classnames = names[:-1]
123 if self.xml.prefix:
124 classnames.insert(0, self.xml.prefix)
125 attrs: Dict[str, str] = {
126 "classname": ".".join(classnames),
127 "name": bin_xml_escape(names[-1]),
128 "file": testreport.location[0],
129 }
130 if testreport.location[1] is not None:
131 attrs["line"] = str(testreport.location[1])
132 if hasattr(testreport, "url"):
133 attrs["url"] = testreport.url
134 self.attrs = attrs
135 self.attrs.update(existing_attrs) # Restore any user-defined attributes.
137 # Preserve legacy testcase behavior.
138 if self.family == "xunit1":
139 return
141 # Filter out attributes not permitted by this test family.
142 # Including custom attributes because they are not valid here.
143 temp_attrs = {}
144 for key in self.attrs.keys():
145 if key in families[self.family]["testcase"]:
146 temp_attrs[key] = self.attrs[key]
147 self.attrs = temp_attrs
149 def to_xml(self) -> ET.Element:
150 testcase = ET.Element("testcase", self.attrs, time="%.3f" % self.duration)
151 properties = self.make_properties_node()
152 if properties is not None:
153 testcase.append(properties)
154 testcase.extend(self.nodes)
155 return testcase
157 def _add_simple(self, tag: str, message: str, data: Optional[str] = None) -> None:
158 node = ET.Element(tag, message=message)
159 node.text = bin_xml_escape(data)
160 self.append(node)
162 def write_captured_output(self, report: TestReport) -> None:
163 if not self.xml.log_passing_tests and report.passed:
164 return
166 content_out = report.capstdout
167 content_log = report.caplog
168 content_err = report.capstderr
169 if self.xml.logging == "no":
170 return
171 content_all = ""
172 if self.xml.logging in ["log", "all"]:
173 content_all = self._prepare_content(content_log, " Captured Log ")
174 if self.xml.logging in ["system-out", "out-err", "all"]:
175 content_all += self._prepare_content(content_out, " Captured Out ")
176 self._write_content(report, content_all, "system-out")
177 content_all = ""
178 if self.xml.logging in ["system-err", "out-err", "all"]:
179 content_all += self._prepare_content(content_err, " Captured Err ")
180 self._write_content(report, content_all, "system-err")
181 content_all = ""
182 if content_all:
183 self._write_content(report, content_all, "system-out")
185 def _prepare_content(self, content: str, header: str) -> str:
186 return "\n".join([header.center(80, "-"), content, ""])
188 def _write_content(self, report: TestReport, content: str, jheader: str) -> None:
189 tag = ET.Element(jheader)
190 tag.text = bin_xml_escape(content)
191 self.append(tag)
193 def append_pass(self, report: TestReport) -> None:
194 self.add_stats("passed")
196 def append_failure(self, report: TestReport) -> None:
197 # msg = str(report.longrepr.reprtraceback.extraline)
198 if hasattr(report, "wasxfail"):
199 self._add_simple("skipped", "xfail-marked test passes unexpectedly")
200 else:
201 assert report.longrepr is not None
202 reprcrash: Optional[ReprFileLocation] = getattr(
203 report.longrepr, "reprcrash", None
204 )
205 if reprcrash is not None:
206 message = reprcrash.message
207 else:
208 message = str(report.longrepr)
209 message = bin_xml_escape(message)
210 self._add_simple("failure", message, str(report.longrepr))
212 def append_collect_error(self, report: TestReport) -> None:
213 # msg = str(report.longrepr.reprtraceback.extraline)
214 assert report.longrepr is not None
215 self._add_simple("error", "collection failure", str(report.longrepr))
217 def append_collect_skipped(self, report: TestReport) -> None:
218 self._add_simple("skipped", "collection skipped", str(report.longrepr))
220 def append_error(self, report: TestReport) -> None:
221 assert report.longrepr is not None
222 reprcrash: Optional[ReprFileLocation] = getattr(
223 report.longrepr, "reprcrash", None
224 )
225 if reprcrash is not None:
226 reason = reprcrash.message
227 else:
228 reason = str(report.longrepr)
230 if report.when == "teardown":
231 msg = f'failed on teardown with "{reason}"'
232 else:
233 msg = f'failed on setup with "{reason}"'
234 self._add_simple("error", bin_xml_escape(msg), str(report.longrepr))
236 def append_skipped(self, report: TestReport) -> None:
237 if hasattr(report, "wasxfail"):
238 xfailreason = report.wasxfail
239 if xfailreason.startswith("reason: "):
240 xfailreason = xfailreason[8:]
241 xfailreason = bin_xml_escape(xfailreason)
242 skipped = ET.Element("skipped", type="pytest.xfail", message=xfailreason)
243 self.append(skipped)
244 else:
245 assert isinstance(report.longrepr, tuple)
246 filename, lineno, skipreason = report.longrepr
247 if skipreason.startswith("Skipped: "):
248 skipreason = skipreason[9:]
249 details = f"{filename}:{lineno}: {skipreason}"
251 skipped = ET.Element("skipped", type="pytest.skip", message=skipreason)
252 skipped.text = bin_xml_escape(details)
253 self.append(skipped)
254 self.write_captured_output(report)
256 def finalize(self) -> None:
257 data = self.to_xml()
258 self.__dict__.clear()
259 # Type ignored because mypy doesn't like overriding a method.
260 # Also the return value doesn't match...
261 self.to_xml = lambda: data # type: ignore[assignment]
264def _warn_incompatibility_with_xunit2(
265 request: FixtureRequest, fixture_name: str
266) -> None:
267 """Emit a PytestWarning about the given fixture being incompatible with newer xunit revisions."""
268 from _pytest.warning_types import PytestWarning
270 xml = request.config.stash.get(xml_key, None)
271 if xml is not None and xml.family not in ("xunit1", "legacy"):
272 request.node.warn(
273 PytestWarning(
274 "{fixture_name} is incompatible with junit_family '{family}' (use 'legacy' or 'xunit1')".format(
275 fixture_name=fixture_name, family=xml.family
276 )
277 )
278 )
281@pytest.fixture
282def record_property(request: FixtureRequest) -> Callable[[str, object], None]:
283 """Add extra properties to the calling test.
285 User properties become part of the test report and are available to the
286 configured reporters, like JUnit XML.
288 The fixture is callable with ``name, value``. The value is automatically
289 XML-encoded.
291 Example::
293 def test_function(record_property):
294 record_property("example_key", 1)
295 """
296 _warn_incompatibility_with_xunit2(request, "record_property")
298 def append_property(name: str, value: object) -> None:
299 request.node.user_properties.append((name, value))
301 return append_property
304@pytest.fixture
305def record_xml_attribute(request: FixtureRequest) -> Callable[[str, object], None]:
306 """Add extra xml attributes to the tag for the calling test.
308 The fixture is callable with ``name, value``. The value is
309 automatically XML-encoded.
310 """
311 from _pytest.warning_types import PytestExperimentalApiWarning
313 request.node.warn(
314 PytestExperimentalApiWarning("record_xml_attribute is an experimental feature")
315 )
317 _warn_incompatibility_with_xunit2(request, "record_xml_attribute")
319 # Declare noop
320 def add_attr_noop(name: str, value: object) -> None:
321 pass
323 attr_func = add_attr_noop
325 xml = request.config.stash.get(xml_key, None)
326 if xml is not None:
327 node_reporter = xml.node_reporter(request.node.nodeid)
328 attr_func = node_reporter.add_attribute
330 return attr_func
333def _check_record_param_type(param: str, v: str) -> None:
334 """Used by record_testsuite_property to check that the given parameter name is of the proper
335 type."""
336 __tracebackhide__ = True
337 if not isinstance(v, str):
338 msg = "{param} parameter needs to be a string, but {g} given" # type: ignore[unreachable]
339 raise TypeError(msg.format(param=param, g=type(v).__name__))
342@pytest.fixture(scope="session")
343def record_testsuite_property(request: FixtureRequest) -> Callable[[str, object], None]:
344 """Record a new ``<property>`` tag as child of the root ``<testsuite>``.
346 This is suitable to writing global information regarding the entire test
347 suite, and is compatible with ``xunit2`` JUnit family.
349 This is a ``session``-scoped fixture which is called with ``(name, value)``. Example:
351 .. code-block:: python
353 def test_foo(record_testsuite_property):
354 record_testsuite_property("ARCH", "PPC")
355 record_testsuite_property("STORAGE_TYPE", "CEPH")
357 :param name:
358 The property name.
359 :param value:
360 The property value. Will be converted to a string.
362 .. warning::
364 Currently this fixture **does not work** with the
365 `pytest-xdist <https://github.com/pytest-dev/pytest-xdist>`__ plugin. See
366 :issue:`7767` for details.
367 """
369 __tracebackhide__ = True
371 def record_func(name: str, value: object) -> None:
372 """No-op function in case --junitxml was not passed in the command-line."""
373 __tracebackhide__ = True
374 _check_record_param_type("name", name)
376 xml = request.config.stash.get(xml_key, None)
377 if xml is not None:
378 record_func = xml.add_global_property # noqa
379 return record_func
382def pytest_addoption(parser: Parser) -> None:
383 group = parser.getgroup("terminal reporting")
384 group.addoption(
385 "--junitxml",
386 "--junit-xml",
387 action="store",
388 dest="xmlpath",
389 metavar="path",
390 type=functools.partial(filename_arg, optname="--junitxml"),
391 default=None,
392 help="Create junit-xml style report file at given path",
393 )
394 group.addoption(
395 "--junitprefix",
396 "--junit-prefix",
397 action="store",
398 metavar="str",
399 default=None,
400 help="Prepend prefix to classnames in junit-xml output",
401 )
402 parser.addini(
403 "junit_suite_name", "Test suite name for JUnit report", default="pytest"
404 )
405 parser.addini(
406 "junit_logging",
407 "Write captured log messages to JUnit report: "
408 "one of no|log|system-out|system-err|out-err|all",
409 default="no",
410 )
411 parser.addini(
412 "junit_log_passing_tests",
413 "Capture log information for passing tests to JUnit report: ",
414 type="bool",
415 default=True,
416 )
417 parser.addini(
418 "junit_duration_report",
419 "Duration time to report: one of total|call",
420 default="total",
421 ) # choices=['total', 'call'])
422 parser.addini(
423 "junit_family",
424 "Emit XML for schema: one of legacy|xunit1|xunit2",
425 default="xunit2",
426 )
429def pytest_configure(config: Config) -> None:
430 xmlpath = config.option.xmlpath
431 # Prevent opening xmllog on worker nodes (xdist).
432 if xmlpath and not hasattr(config, "workerinput"):
433 junit_family = config.getini("junit_family")
434 config.stash[xml_key] = LogXML(
435 xmlpath,
436 config.option.junitprefix,
437 config.getini("junit_suite_name"),
438 config.getini("junit_logging"),
439 config.getini("junit_duration_report"),
440 junit_family,
441 config.getini("junit_log_passing_tests"),
442 )
443 config.pluginmanager.register(config.stash[xml_key])
446def pytest_unconfigure(config: Config) -> None:
447 xml = config.stash.get(xml_key, None)
448 if xml:
449 del config.stash[xml_key]
450 config.pluginmanager.unregister(xml)
453def mangle_test_address(address: str) -> List[str]:
454 path, possible_open_bracket, params = address.partition("[")
455 names = path.split("::")
456 # Convert file path to dotted path.
457 names[0] = names[0].replace(nodes.SEP, ".")
458 names[0] = re.sub(r"\.py$", "", names[0])
459 # Put any params back.
460 names[-1] += possible_open_bracket + params
461 return names
464class LogXML:
465 def __init__(
466 self,
467 logfile,
468 prefix: Optional[str],
469 suite_name: str = "pytest",
470 logging: str = "no",
471 report_duration: str = "total",
472 family="xunit1",
473 log_passing_tests: bool = True,
474 ) -> None:
475 logfile = os.path.expanduser(os.path.expandvars(logfile))
476 self.logfile = os.path.normpath(os.path.abspath(logfile))
477 self.prefix = prefix
478 self.suite_name = suite_name
479 self.logging = logging
480 self.log_passing_tests = log_passing_tests
481 self.report_duration = report_duration
482 self.family = family
483 self.stats: Dict[str, int] = dict.fromkeys(
484 ["error", "passed", "failure", "skipped"], 0
485 )
486 self.node_reporters: Dict[
487 Tuple[Union[str, TestReport], object], _NodeReporter
488 ] = {}
489 self.node_reporters_ordered: List[_NodeReporter] = []
490 self.global_properties: List[Tuple[str, str]] = []
492 # List of reports that failed on call but teardown is pending.
493 self.open_reports: List[TestReport] = []
494 self.cnt_double_fail_tests = 0
496 # Replaces convenience family with real family.
497 if self.family == "legacy":
498 self.family = "xunit1"
500 def finalize(self, report: TestReport) -> None:
501 nodeid = getattr(report, "nodeid", report)
502 # Local hack to handle xdist report order.
503 workernode = getattr(report, "node", None)
504 reporter = self.node_reporters.pop((nodeid, workernode))
505 if reporter is not None:
506 reporter.finalize()
508 def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporter:
509 nodeid: Union[str, TestReport] = getattr(report, "nodeid", report)
510 # Local hack to handle xdist report order.
511 workernode = getattr(report, "node", None)
513 key = nodeid, workernode
515 if key in self.node_reporters:
516 # TODO: breaks for --dist=each
517 return self.node_reporters[key]
519 reporter = _NodeReporter(nodeid, self)
521 self.node_reporters[key] = reporter
522 self.node_reporters_ordered.append(reporter)
524 return reporter
526 def add_stats(self, key: str) -> None:
527 if key in self.stats:
528 self.stats[key] += 1
530 def _opentestcase(self, report: TestReport) -> _NodeReporter:
531 reporter = self.node_reporter(report)
532 reporter.record_testreport(report)
533 return reporter
535 def pytest_runtest_logreport(self, report: TestReport) -> None:
536 """Handle a setup/call/teardown report, generating the appropriate
537 XML tags as necessary.
539 Note: due to plugins like xdist, this hook may be called in interlaced
540 order with reports from other nodes. For example:
542 Usual call order:
543 -> setup node1
544 -> call node1
545 -> teardown node1
546 -> setup node2
547 -> call node2
548 -> teardown node2
550 Possible call order in xdist:
551 -> setup node1
552 -> call node1
553 -> setup node2
554 -> call node2
555 -> teardown node2
556 -> teardown node1
557 """
558 close_report = None
559 if report.passed:
560 if report.when == "call": # ignore setup/teardown
561 reporter = self._opentestcase(report)
562 reporter.append_pass(report)
563 elif report.failed:
564 if report.when == "teardown":
565 # The following vars are needed when xdist plugin is used.
566 report_wid = getattr(report, "worker_id", None)
567 report_ii = getattr(report, "item_index", None)
568 close_report = next(
569 (
570 rep
571 for rep in self.open_reports
572 if (
573 rep.nodeid == report.nodeid
574 and getattr(rep, "item_index", None) == report_ii
575 and getattr(rep, "worker_id", None) == report_wid
576 )
577 ),
578 None,
579 )
580 if close_report:
581 # We need to open new testcase in case we have failure in
582 # call and error in teardown in order to follow junit
583 # schema.
584 self.finalize(close_report)
585 self.cnt_double_fail_tests += 1
586 reporter = self._opentestcase(report)
587 if report.when == "call":
588 reporter.append_failure(report)
589 self.open_reports.append(report)
590 if not self.log_passing_tests:
591 reporter.write_captured_output(report)
592 else:
593 reporter.append_error(report)
594 elif report.skipped:
595 reporter = self._opentestcase(report)
596 reporter.append_skipped(report)
597 self.update_testcase_duration(report)
598 if report.when == "teardown":
599 reporter = self._opentestcase(report)
600 reporter.write_captured_output(report)
602 for propname, propvalue in report.user_properties:
603 reporter.add_property(propname, str(propvalue))
605 self.finalize(report)
606 report_wid = getattr(report, "worker_id", None)
607 report_ii = getattr(report, "item_index", None)
608 close_report = next(
609 (
610 rep
611 for rep in self.open_reports
612 if (
613 rep.nodeid == report.nodeid
614 and getattr(rep, "item_index", None) == report_ii
615 and getattr(rep, "worker_id", None) == report_wid
616 )
617 ),
618 None,
619 )
620 if close_report:
621 self.open_reports.remove(close_report)
623 def update_testcase_duration(self, report: TestReport) -> None:
624 """Accumulate total duration for nodeid from given report and update
625 the Junit.testcase with the new total if already created."""
626 if self.report_duration == "total" or report.when == self.report_duration:
627 reporter = self.node_reporter(report)
628 reporter.duration += getattr(report, "duration", 0.0)
630 def pytest_collectreport(self, report: TestReport) -> None:
631 if not report.passed:
632 reporter = self._opentestcase(report)
633 if report.failed:
634 reporter.append_collect_error(report)
635 else:
636 reporter.append_collect_skipped(report)
638 def pytest_internalerror(self, excrepr: ExceptionRepr) -> None:
639 reporter = self.node_reporter("internal")
640 reporter.attrs.update(classname="pytest", name="internal")
641 reporter._add_simple("error", "internal error", str(excrepr))
643 def pytest_sessionstart(self) -> None:
644 self.suite_start_time = timing.time()
646 def pytest_sessionfinish(self) -> None:
647 dirname = os.path.dirname(os.path.abspath(self.logfile))
648 # exist_ok avoids filesystem race conditions between checking path existence and requesting creation
649 os.makedirs(dirname, exist_ok=True)
651 with open(self.logfile, "w", encoding="utf-8") as logfile:
652 suite_stop_time = timing.time()
653 suite_time_delta = suite_stop_time - self.suite_start_time
655 numtests = (
656 self.stats["passed"]
657 + self.stats["failure"]
658 + self.stats["skipped"]
659 + self.stats["error"]
660 - self.cnt_double_fail_tests
661 )
662 logfile.write('<?xml version="1.0" encoding="utf-8"?>')
664 suite_node = ET.Element(
665 "testsuite",
666 name=self.suite_name,
667 errors=str(self.stats["error"]),
668 failures=str(self.stats["failure"]),
669 skipped=str(self.stats["skipped"]),
670 tests=str(numtests),
671 time="%.3f" % suite_time_delta,
672 timestamp=datetime.fromtimestamp(self.suite_start_time).isoformat(),
673 hostname=platform.node(),
674 )
675 global_properties = self._get_global_properties_node()
676 if global_properties is not None:
677 suite_node.append(global_properties)
678 for node_reporter in self.node_reporters_ordered:
679 suite_node.append(node_reporter.to_xml())
680 testsuites = ET.Element("testsuites")
681 testsuites.append(suite_node)
682 logfile.write(ET.tostring(testsuites, encoding="unicode"))
684 def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:
685 terminalreporter.write_sep("-", f"generated xml file: {self.logfile}")
687 def add_global_property(self, name: str, value: object) -> None:
688 __tracebackhide__ = True
689 _check_record_param_type("name", name)
690 self.global_properties.append((name, bin_xml_escape(value)))
692 def _get_global_properties_node(self) -> Optional[ET.Element]:
693 """Return a Junit node containing custom properties, if any."""
694 if self.global_properties:
695 properties = ET.Element("properties")
696 for name, value in self.global_properties:
697 properties.append(ET.Element("property", name=name, value=value))
698 return properties
699 return None