gh-158895: Teach libregrtest to output github job summaries

Lib/test/libregrtest/main.py

@@ -15,6 +15,7 @@
 from .findtests import findtests, split_test_packages, list_cases, collect_cases
 from .logger import Logger
 from .pgo import setup_pgo_tests
+from .reporter import get_reporter
 from .result import TestResult
 from .results import TestResults, EXITCODE_INTERRUPTED
 from .runtests import RunTests, HuntRefleak
@@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
 
         # Logger
         self.logger = Logger(self.results, self.quiet, self.pgo)
+        self.reporter = get_reporter()
 
         # Actions
         self.want_header: bool = ns.header
@@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests:
             forever=False,
             fail_fast=False,
             match_tests_dict=match_tests_dict,
-            output_on_failure=False)
+            output_on_failure=False,
+            final_run=True)
         self.logger.set_tests(runtests)
+        self.reporter.run_started(runtests)
 
         msg = f"Re-running {len(tests)} failed tests in verbose mode"
         if not self.single_process:
@@ -395,6 +399,7 @@ def run_test(
             result = run_single_test(test_name, runtests)
 
         self.results.accumulate_result(result, runtests)
+        self.reporter.test_result(result)
 
         return result
 
@@ -459,7 +464,8 @@ def get_state(self) -> str:
 
     def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None:
         from .run_workers import RunWorkers
-        RunWorkers(num_workers, runtests, self.logger, self.results).run()
+        RunWorkers(num_workers, runtests, self.logger, self.results,
+                   self.reporter).run()
 
     def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
         if self.next_single_filename:
@@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
         if self.junit_filename:
             self.results.write_junit(self.junit_filename)
 
+        self.reporter.summary(self.results, self.get_state())
+
     def display_summary(self) -> None:
         if self.first_runtests is None:
             raise ValueError(
@@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests:
             hunt_refleak=self.hunt_refleak,
             test_dir=self.test_dir,
             use_junit=(self.junit_filename is not None),
+            final_run=not (self.want_rerun and not self.python_cmd),
             coverage=self.coverage,
             memory_limit=self.memory_limit,
             gc_threshold=self.gc_threshold,
@@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
             runtests = runtests.copy(tests=case_ids, case_groups=case_groups)
         self.first_runtests = runtests
         self.logger.set_tests(runtests)
+        self.reporter.run_started(runtests)
 
         if (runtests.hunt_refleak is not None) and (not self.num_workers):
             # gh-109739: WindowsLoadTracker thread interferes with refleak check

Lib/test/libregrtest/reporter.py

@@ -0,0 +1,142 @@
+import html
+import os
+import pathlib
+import re
+from _colorize import decolor  # type: ignore[import-not-found]
+from typing import TextIO
+
+from .result import TestResult
+lazy from .results import TestResults
+from .runtests import RunTests
+from .utils import count
+
+
+class Reporter:
+    """Report test failures, for example to a CI service.
+
+    The default reporter does nothing.
+    """
+
+    def run_started(self, runtests: RunTests) -> None:
+        """Start a run of tests.
+
+        Called in the main process at the start of each run (the first run
+        and the re-run), and in the process running the tests before each
+        test file.
+        """
+
+    def test_case_failed(self, test_case: str, traceback: str,
+                         stream: TextIO) -> None:
+        """Report a failed test case.
+
+        Called just after unittest printed the failure, or at the end of the
+        test file in quiet mode.
+        """
+
+    def test_result(self, result: TestResult) -> None:
+        """Report the result of a test file, in the main process."""
+
+    def summary(self, results: TestResults, state: str) -> None:
+        """Report the results of all runs, in the main process."""
+
+    @staticmethod
+    def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]:
+        """Parse the traceback of a failure into (filename, line, exception).
+
+        The location is the last frame in a test file, or else the first frame
+        (the test).
+        """
+        frames = re.findall(r'^ *File "(.*)", line (\d+)', traceback,
+                            re.MULTILINE)
+        test_frames = [frame for frame in frames
+                       if pathlib.Path(frame[0]).name.startswith('test_')]
+        filename, line = (test_frames[-1] if test_frames
+                          else frames[0] if frames
+                          else (None, None))
+        exception = re.split(r'^  File .*\n(?:    .*\n)*', traceback,
+                             flags=re.MULTILINE)[-1].strip()
+        return filename, line, exception
+
+
+def get_reporter() -> Reporter:
+    if GitHubActionsReporter.enabled():
+        return GitHubActionsReporter()
+    return Reporter()
+
+
+class GitHubActionsReporter(Reporter):
+    """Annotate failures in the GitHub Actions job log and list them in the
+    job summary."""
+
+    # Escape the data and the properties of workflow commands
+    ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"})
+    ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"})
+
+    @staticmethod
+    def enabled() -> bool:
+        return "GITHUB_STEP_SUMMARY" in os.environ
+
+    def run_started(self, runtests: RunTests) -> None:
+        self.runtests = runtests
+
+    def test_case_failed(self, test_case: str, traceback: str,
+                         stream: TextIO) -> None:
+        # Only report the failures which decide the result
+        if not self.runtests.final_run:
+            return
+        # The job log has the full traceback: only keep the exception
+        filename, line, exception = self.parse_traceback(decolor(traceback))
+        self.annotate(test_case, exception, filename, line, stream)
+
+    def test_result(self, result: TestResult) -> None:
+        # Failed test cases are annotated by test_case_failed()
+        if (not self.runtests.final_run
+                or not result.is_failed(self.runtests.fail_env_changed)
+                or result.failed_test_cases):
+            return
+        message = "\n".join([str(result), *(result.env_changed_reasons or ())])
+        self.annotate(result.test_name, decolor(message),
+                      result.find_filename(self.runtests.test_dir))
+
+    def annotate(self, title: str, message: str, filename: str | None,
+                 line: str | None = None, stream: TextIO | None = None) -> None:
+        if filename:
+            # The stdlib can be outside of the repository checkout
+            path = pathlib.Path(filename)
+            stdlib_dir = pathlib.Path(os.__file__).parent
+            if path.is_relative_to(stdlib_dir):
+                filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix()
+            # Without a line, the annotation is not shown on the file
+            line = line or "1"
+        props = {"file": filename, "line": line, "title": title}
+        props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}"
+                              for key, value in props.items() if value)
+        print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}",
+              file=stream, flush=True)
+
+    def summary(self, results: TestResults, state: str) -> None:
+        # Test files which failed in the last run (the re-run, if any)
+        failed = [result for result in results.rerun_results
+                  if result.is_failed(self.runtests.fail_env_changed)]
+        if not failed:
+            return
+        ncases = sum(len(result.failed_test_cases) for result in failed)
+        blocks = [f"## {decolor(state)}: "
+                  f"{count(len(failed), 'test file')} and "
+                  f"{count(ncases, 'test case')} failed"]
+        for result in failed:
+            blocks.append(f"### {decolor(str(result))}")
+            if result.env_changed_reasons:
+                blocks.append("\n".join(f"- {reason}"
+                                        for reason in result.env_changed_reasons))
+            for name, traceback in result.failed_test_cases:
+                # Expand short tracebacks if there are only a few
+                is_open = ncases <= 5 and traceback.count("\n") < 30
+                blocks += [f"<details{' open' if is_open else ''}>"
+                           f"<summary>{html.escape(name)}</summary>",
+                           f"```pytb\n{decolor(traceback).rstrip()}\n```",
+                           "</details>"]
+        # Markdown blocks are separated by an empty line
+        with open(os.environ["GITHUB_STEP_SUMMARY"], "a",
+                  encoding="utf-8") as fp:
+            fp.write("".join(f"{block}\n\n" for block in blocks))

Lib/test/libregrtest/result.py

@@ -1,10 +1,12 @@
 import dataclasses
 import json
+import pathlib
 from _colorize import get_colors  # type: ignore[import-not-found]
 from typing import Any
 
+from .findtests import findtestdir
 from .utils import (
-    StrJSON, TestName, FilterTuple,
+    StrJSON, StrPath, TestName, FilterTuple,
     format_duration, normalize_test_name, print_warning)
 
 
@@ -103,6 +105,9 @@ class TestResult:
     # short descriptions of how the test altered the execution environment
     env_changed_reasons: list[str] | None = None
 
+    # test file, set once the test module is imported
+    filename: str | None = None
+
     def is_failed(self, fail_env_changed: bool) -> bool:
         if self.state == State.ENV_CHANGED:
             return fail_env_changed
@@ -178,6 +183,23 @@ def __str__(self) -> str:
     def has_meaningful_duration(self):
         return State.has_meaningful_duration(self.state)
 
+    @property
+    def failed_test_cases(self) -> list[tuple[str, str]]:
+        return (self.errors or []) + (self.failures or [])
+
+    def find_filename(self, test_dir: StrPath | None) -> str | None:
+        """Get the filename of the test, or None if it is not found."""
+        if self.filename:
+            return self.filename
+        # The worker crashed or timed out before importing the test:
+        # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py
+        name = self.test_name.removeprefix("test.")
+        path = pathlib.Path(findtestdir(test_dir), *name.split("."))
+        for filename in (path.with_suffix(".py"), path / "__init__.py"):
+            if filename.exists():
+                return str(filename)
+        return None
+
     def set_env_changed(self, *reasons):
         if self.state is None or self.state == State.PASSED:
             self.state = State.ENV_CHANGED

Lib/test/libregrtest/run_workers.py

@@ -16,6 +16,7 @@
 from test.support import os_helper, MS_WINDOWS
 
 from .logger import Logger
+from .reporter import Reporter
 from .result import TestResult, State
 from .results import TestResults
 from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType
@@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None:
 
 class RunWorkers:
     def __init__(self, num_workers: int, runtests: RunTests,
-                 logger: Logger, results: TestResults) -> None:
+                 logger: Logger, results: TestResults,
+                 reporter: Reporter) -> None:
         self.num_workers = num_workers
         self.runtests = runtests
         self.logger = logger
         self.log = logger.log
         self.display_progress = logger.display_progress
         self.results: TestResults = results
+        self.reporter = reporter
         self.live_worker_count = 0
 
         self.output: queue.Queue[QueueContent] = queue.Queue()
@@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
+        # Report after the output: env changed warnings, crash traceback
+        self.reporter.test_result(result)
 
         return result
 

Lib/test/libregrtest/runtests.py

@@ -93,6 +93,8 @@ class RunTests:
     hunt_refleak: HuntRefleak | None
     test_dir: StrPath | None
     use_junit: bool
+    # Failed tests are not re-run after this run
+    final_run: bool
     coverage: bool
     memory_limit: str | None
     gc_threshold: int | None

Lib/test/libregrtest/setup.py

@@ -11,7 +11,9 @@
 from _colorize import can_colorize  # type: ignore[import-not-found]
 
 from .filter import set_match_tests
+from .reporter import get_reporter
 from .runtests import RunTests
+lazy from .testresult import RegressionTestResult
 from .utils import (
     setup_unraisable_hook, setup_threading_excepthook,
     adjust_rlimit_nofile)
@@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None:
 
     set_match_tests(runtests.match_tests)
 
+    reporter = get_reporter()
+    reporter.run_started(runtests)
+    RegressionTestResult.reporter = reporter
     if runtests.use_junit:
         support.junit_xml_list = []
-        from .testresult import RegressionTestResult
         RegressionTestResult.USE_XML = True
     else:
         support.junit_xml_list = None

Lib/test/libregrtest/single.py

@@ -153,6 +153,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None:
     test_name = result.test_name
     module_name = abs_module_name(test_name, runtests.test_dir)
     test_mod = importlib.import_module(module_name)
+    result.filename = test_mod.__file__
 
     if hasattr(test_mod, "test_main"):
         # https://github.com/python/cpython/issues/89392

Lib/test/libregrtest/testresult.py

@@ -9,10 +9,12 @@
 import traceback
 import unittest
 from test import support
+from test.libregrtest.reporter import Reporter
 from test.libregrtest.utils import sanitize_xml
 
 class RegressionTestResult(unittest.TextTestResult):
     USE_XML = False
+    reporter = Reporter()
 
     def __init__(self, stream, descriptions, verbosity):
         super().__init__(stream=stream, descriptions=descriptions,
@@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test):
         self._add_result(test, outcome='UNEXPECTED_SUCCESS')
         super().addUnexpectedSuccess(test)
 
+    def printErrorList(self, flavour, errors):
+        for test, err in errors:
+            super().printErrorList(flavour, [(test, err)])
+            self.report_failure(test, err)
+
+    def report_failure(self, test, err):
+        self.reporter.test_case_failed(str(test), err, self.stream)
+
     def get_xml_element(self):
         if not self.USE_XML:
             raise ValueError("USE_XML is false")
@@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False):
 
     def run(self, test):
         test(self.result)
+        # Failures are not printed in quiet mode
+        for test, err in self.result.errors + self.result.failures:
+            self.result.report_failure(test, err)
         return self.result
 
 def get_test_runner_class(verbosity, buffer=False):

Lib/test/test_bool.py

@@ -4,6 +4,7 @@
 from test.support import os_helper
 
 import os
+import sys
 
 class BoolTest(unittest.TestCase):
 
@@ -25,6 +26,11 @@
         self.assertIs(eval(repr(True)), True)
 
     def test_str(self):
+        if sys.platform == "linux":
+            # XXX: Intentional failure to test GitHub Actions annotations
+            self.assertEqual(
+                str(True),
+                'Yes')
         self.assertEqual(str(False), 'False')
         self.assertEqual(str(True), 'True')