From 3eeb4db946ff089d9e222423e9dd11d27183b44b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 22:39:22 -0700 Subject: [PATCH 01/37] gh-158895: Teach librertest to output github job summaries Goal is to make it faster to triage individual test failures while keeping the full existing regrtest fidelity. Fairly minimal integration to hopefully save some human eyeball time. Setup so only the test summary process should output the summary, subprocesses have the env var which will cause output unset. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++- Lib/test/libregrtest/worker.py | 2 ++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d32..acaebf96e9ce53 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,5 @@ +import contextlib +import io import os import random import re @@ -6,7 +8,7 @@ import sysconfig import time import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -480,6 +482,25 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.write_github_summary() + + def write_github_summary(self) -> None: + filename = os.environ.get("GITHUB_STEP_SUMMARY") + if not filename: + return + with contextlib.redirect_stdout(io.StringIO()) as summary: + self.display_summary() + lines = ["## Test results", + f"```\n{decolor(summary.getvalue()).strip()}\n```"] + # Tests which failed in the last run (the re-run, if any) + for result in self.results.rerun_results: + lines.append(f"### {decolor(str(result))}") + for name, traceback in (result.errors or []) + (result.failures or []): + lines.append(f"
{name}\n\n" + f"```\n{traceback}\n```\n
") + with open(filename, "a", encoding="utf-8") as fp: + fp.write("\n\n".join(lines) + "\n") + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 4e69ab9d8fad05..24ae95ed23914f 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,6 +28,8 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) + # Only the main process writes the GitHub Actions job summary + env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 00000000000000..ebeca7174e5903 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,2 @@ +Individual test failures and an overall summary is now written by +:mod:`test.regrtest` to the Github Action Summary. From 8fcfa5d0f2f8a68424594d4e22824cf1817af2cf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:09:38 -0700 Subject: [PATCH 02/37] Only output errors, too many jobs. Try coloring a bit more --- Lib/test/libregrtest/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index acaebf96e9ce53..40fcfe5f5c577f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -486,7 +486,10 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") - if not filename: + # Only write a summary on failure: a CI run has many test jobs + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + if not filename or not exitcode: return with contextlib.redirect_stdout(io.StringIO()) as summary: self.display_summary() @@ -497,7 +500,7 @@ def write_github_summary(self) -> None: lines.append(f"### {decolor(str(result))}") for name, traceback in (result.errors or []) + (result.failures or []): lines.append(f"
{name}\n\n" - f"```\n{traceback}\n```\n
") + f"```pytb\n{decolor(traceback)}\n```\n") with open(filename, "a", encoding="utf-8") as fp: fp.write("\n\n".join(lines) + "\n") From e6e09bd77e81ba8def2a1d2b399a11521d995e90 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:10:30 -0700 Subject: [PATCH 03/37] Fail CI on linux for experimenting with output formatting --- Lib/test/test_os/test_os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 81b3043eb7e75b..04321264e6f121 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -4080,7 +4080,8 @@ def test_memfd_create(self): self.assertFalse(os.get_inheritable(fd)) with open(fd, "wb", closefd=False) as f: f.write(b'memfd_create') - self.assertEqual(f.tell(), 12) + # XXX: Intentional failure to test GitHub Actions job summaries + self.assertEqual(f.tell(), 13) fd2 = os.memfd_create("Hi") self.addCleanup(os.close, fd2) From 54d123fd515a812147582e8dd1a8264ac1e13cb2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:42:57 -0700 Subject: [PATCH 04/37] Reformat failures to be failure first, easier visibility. Simplify formatting code some --- Lib/test/libregrtest/main.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 40fcfe5f5c577f..42dbf166837a6f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,3 @@ -import contextlib -import io import os import random import re @@ -491,18 +489,31 @@ def write_github_summary(self) -> None: self.fail_rerun) if not filename or not exitcode: return - with contextlib.redirect_stdout(io.StringIO()) as summary: - self.display_summary() - lines = ["## Test results", - f"```\n{decolor(summary.getvalue()).strip()}\n```"] # Tests which failed in the last run (the re-run, if any) - for result in self.results.rerun_results: - lines.append(f"### {decolor(str(result))}") - for name, traceback in (result.errors or []) + (result.failures or []): - lines.append(f"
{name}\n\n" - f"```pytb\n{decolor(traceback)}\n```\n
") + failed = self.results.rerun_results + cases = [(result.errors or []) + (result.failures or []) + for result in failed] + ncase = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: - fp.write("\n\n".join(lines) + "\n") + def write(text: str) -> None: + # Separate Markdown blocks with an empty line + print(text, end="\n\n", file=fp) + + write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncase, 'test case')} failed") + for result, result_cases in zip(failed, cases): + write(f"### {decolor(str(result))}") + if result.env_changed_reasons: + write("\n".join(f"- {reason}" + for reason in result.env_changed_reasons)) + for name, traceback in result_cases: + # Expand short tracebacks when there are only a few + is_open = ncase <= 5 and traceback.count("\n") < 30 + write(f"" + f"{name}") + write(f"```pytb\n{decolor(traceback).rstrip()}\n```") + write("") def display_summary(self) -> None: if self.first_runtests is None: From 3468742f5b333d261a0544c1c807283a2111842d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:44:54 -0700 Subject: [PATCH 05/37] add error annotations --- Lib/test/libregrtest/main.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6f..f191dd123ed896 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,6 +30,15 @@ get_process_memory_usage, EXIT_TIMEOUT) +def github_error(title: str, message: str) -> None: + # Emit a GitHub Actions error annotation (workflow command) + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + title = escape(title).replace(":", "%3A").replace(",", "%2C") + print(f"::error title={title}::{escape(message)}", flush=True) + + class Regrtest: """Execute a test suite. @@ -504,10 +513,14 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") + if not result_cases: + github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: + github_error(f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" From ff7b75a45b6302ee2ab25d99265262882ddee709 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:14:59 -0700 Subject: [PATCH 06/37] Add groups more of the time --- Lib/test/libregrtest/main.py | 13 ------------- Lib/test/libregrtest/results.py | 20 ++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 26 ++++++++++++++++++++++++++ Lib/test/libregrtest/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f191dd123ed896..42dbf166837a6f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,15 +30,6 @@ get_process_memory_usage, EXIT_TIMEOUT) -def github_error(title: str, message: str) -> None: - # Emit a GitHub Actions error annotation (workflow command) - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - title = escape(title).replace(":", "%3A").replace(",", "%2C") - print(f"::error title={title}::{escape(message)}", flush=True) - - class Regrtest: """Execute a test suite. @@ -513,14 +504,10 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") - if not result_cases: - github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - github_error(f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3475f645729182..9015cc1fe5f8a1 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,13 +1,14 @@ +import os import sys import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration) + printlist, count, format_duration, github_annotation) # Python uses exit code 1 when an exception is not caught @@ -19,6 +20,16 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 +def annotate_github(result: TestResult, level: str) -> None: + cases = (result.errors or []) + (result.failures or []) + for name, traceback in cases: + github_annotation(level, f"{result.test_name}: {name}", + decolor(traceback)) + if not cases: + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + github_annotation(level, result.test_name, decolor(message)) + + class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -128,6 +139,11 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True + if (os.environ.get("GITHUB_STEP_SUMMARY") + and result.is_failed(fail_env_changed)): + # A failure is only an error if it fails again when re-run + annotate_github(result, "error" if rerun else "warning") + if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..69ff5edb2607da 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,6 +10,7 @@ import threading import time import traceback +from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -519,6 +520,11 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] + # On GitHub Actions, fold the first run's log into groups: ranges of + # passing tests, and one group per failed test. Re-runs aren't folded. + self.github_groups = (not runtests.rerun + and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -611,7 +617,16 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result + failed = result.is_failed(self.runtests.fail_env_changed) + if self.github_groups and failed: + # Keep the failure annotation out of collapsed groups + self.github_group(None) self.results.accumulate_result(result, self.runtests) + if self.github_groups: + if failed: + self.github_group(decolor(str(result))) + elif not self.in_github_group: + self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -624,9 +639,19 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.github_groups and failed: + self.github_group(None) return result + def github_group(self, title: str | None) -> None: + """End the current GitHub Actions log group, start a new one if title.""" + if self.in_github_group: + print("::endgroup::", flush=True) + self.in_github_group = title is not None + if title is not None: + print(f"::group::{title}", flush=True) + def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -659,6 +684,7 @@ def run(self) -> None: print() self.results.interrupted = True finally: + self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 83e0575619c99a..c31022ea044ece 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -140,6 +140,29 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def github_annotation(level: str, title: str, message: str) -> None: + """Emit a GitHub Actions annotation (workflow command). + + Point it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + props = f"title={escape_property(title)}" + frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) + # Source checkout which contains Lib/ + srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep + if frames and frames[-1][0].startswith(srcdir): + filename, line = frames[-1] + filename = filename.removeprefix(srcdir).replace(os.sep, "/") + props = f"file={escape_property(filename)},line={line},{props}" + print(f"::{level} {props}::{escape(message)}", flush=True) + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 946c801af026df3da4514eaa4cbca2e9bb9b18bf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:50:16 -0700 Subject: [PATCH 07/37] drop groups, error only on final failure --- Lib/test/libregrtest/main.py | 6 +++++- Lib/test/libregrtest/results.py | 12 ++++++------ Lib/test/libregrtest/run_workers.py | 26 -------------------------- Lib/test/libregrtest/utils.py | 11 ++++++----- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6f..1120dacf3b5a6a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED +from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -491,6 +491,10 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now + for result in failed: + annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 9015cc1fe5f8a1..a860ad0f9c8549 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -20,14 +20,14 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult, level: str) -> None: +def annotate_github(result: TestResult) -> None: cases = (result.errors or []) + (result.failures or []) for name, traceback in cases: - github_annotation(level, f"{result.test_name}: {name}", + github_annotation("error", f"{result.test_name}: {name}", decolor(traceback)) if not cases: message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation(level, result.test_name, decolor(message)) + github_annotation("error", result.test_name, decolor(message)) class TestResults: @@ -139,10 +139,10 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") + if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun and result.is_failed(fail_env_changed)): - # A failure is only an error if it fails again when re-run - annotate_github(result, "error" if rerun else "warning") + # Only annotate tests which fail again when re-run + annotate_github(result) if result.has_meaningful_duration() and not rerun: if result.duration is None: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 69ff5edb2607da..c6db78e0882ba3 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,7 +10,6 @@ import threading import time import traceback -from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -520,11 +519,6 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] - # On GitHub Actions, fold the first run's log into groups: ranges of - # passing tests, and one group per failed test. Re-runs aren't folded. - self.github_groups = (not runtests.rerun - and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) - self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -617,16 +611,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result - failed = result.is_failed(self.runtests.fail_env_changed) - if self.github_groups and failed: - # Keep the failure annotation out of collapsed groups - self.github_group(None) self.results.accumulate_result(result, self.runtests) - if self.github_groups: - if failed: - self.github_group(decolor(str(result))) - elif not self.in_github_group: - self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -639,19 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.github_groups and failed: - self.github_group(None) return result - def github_group(self, title: str | None) -> None: - """End the current GitHub Actions log group, start a new one if title.""" - if self.in_github_group: - print("::endgroup::", flush=True) - self.in_github_group = title is not None - if title is not None: - print(f"::group::{title}", flush=True) - def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -684,7 +659,6 @@ def run(self) -> None: print() self.results.interrupted = True finally: - self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index c31022ea044ece..0f21c7981c5141 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -154,12 +154,13 @@ def escape_property(text: str) -> str: props = f"title={escape_property(title)}" frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Source checkout which contains Lib/ - srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep - if frames and frames[-1][0].startswith(srcdir): + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if frames and frames[-1][0].startswith(stdlib_dir): filename, line = frames[-1] - filename = filename.removeprefix(srcdir).replace(os.sep, "/") - props = f"file={escape_property(filename)},line={line},{props}" + filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props = f"file=Lib/{escape_property(filename)},line={line},{props}" print(f"::{level} {props}::{escape(message)}", flush=True) From 14394da14e30c9bda7aaef14c950d692c1415fcc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:27:47 -0700 Subject: [PATCH 08/37] More structured error blocks --- Lib/test/libregrtest/utils.py | 55 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 0f21c7981c5141..ee241844098c40 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,5 +1,6 @@ import contextlib import faulthandler +import linecache import locale import math import os.path @@ -140,6 +141,39 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def traceback_location(traceback: str) -> dict[str, str | int] | None: + """Location of the last frame of a traceback, relative to the repository. + + Return a dict with file and line, plus col and endColumn when the + traceback underlines the failing expression (~~~^^^). + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if match := re.match(r' File "(.+)", line (\d+)', lines[index]): + break + else: + return None + filename, lineno = match.group(1), int(match.group(2)) + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return None + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} + + # The frame is followed by the dedented source line (" code") and, + # for a single-line expression, by a line of ~ and ^ markers. + markers = lines[index + 2] if index + 2 < len(lines) else "" + source = linecache.getline(filename, lineno) + if markers.strip() and not markers.strip(" ~^") and source: + indent = len(source) - len(source.lstrip()) + start = len(markers) - len(markers.lstrip()) + location["col"] = indent + start - 4 + 1 + location["endColumn"] = indent + len(markers.rstrip()) - 4 + return location + + def github_annotation(level: str, title: str, message: str) -> None: """Emit a GitHub Actions annotation (workflow command). @@ -152,16 +186,17 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - props = f"title={escape_property(title)}" - frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if frames and frames[-1][0].startswith(stdlib_dir): - filename, line = frames[-1] - filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props = f"file=Lib/{escape_property(filename)},line={line},{props}" - print(f"::{level} {props}::{escape(message)}", flush=True) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::{level} {props_text}::{escape(message)}", flush=True) def regrtest_unraisable_hook(unraisable) -> None: From 63598766247dd7c00eb73a02990113730b936e8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:39:56 -0700 Subject: [PATCH 09/37] Fail more tests --- Lib/test/test_bool.py | 6 ++++++ Lib/test/test_json/test_decode.py | 4 ++++ Lib/test/test_math.py | 5 +++++ Lib/test/test_textwrap.py | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index dcdf7bdce03b80..2214e7408aa560 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -4,6 +4,7 @@ from test.support import os_helper import os +import sys class BoolTest(unittest.TestCase): @@ -25,6 +26,11 @@ def test_repr(self): self.assertIs(eval(repr(True)), True) def test_str(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.assertEqual( + str(True), + 'Yes') self.assertEqual(str(False), 'False') self.assertEqual(str(True), 'True') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 7bff16ddf2155e..d01368254f6701 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import sys import unittest.mock from io import StringIO from collections import OrderedDict @@ -9,6 +10,9 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.loads('{"bad": }') self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a1250c668223e7..40dc0d9acc75ff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -513,6 +513,11 @@ def testFabs(self): def testFloor(self): self.assertRaises(TypeError, math.floor) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + for x in (0.5, 1.5): + with self.subTest(x=x): + self.assertEqual(math.floor(x), x) self.assertEqual(int, type(math.floor(0.5))) self.assertEqual(math.floor(0.5), 0) self.assertEqual(math.floor(1.0), 1) diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb5..c1d670c978c970 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,8 @@ # $Id$ # +import os +import sys import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -66,6 +68,9 @@ def test_simple(self): self.check_wrap(text, 80, [text]) def test_empty_string(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + os.environ["GHA_ANNOTATION_TEST"] = "1" # Check that wrapping the empty string returns an empty list. self.check_wrap("", 6, []) self.check_wrap("", 6, [], drop_whitespace=False) From 041e57792d78978a97ff19dde573506b894a9e2a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:44:04 -0700 Subject: [PATCH 10/37] Adjust and simplify a little --- Lib/test/libregrtest/main.py | 15 +++++++++------ Lib/test/libregrtest/results.py | 20 ++------------------ 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1120dacf3b5a6a..dc6e8ebc232fcc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github +from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, EXIT_TIMEOUT) + get_process_memory_usage, github_annotation, EXIT_TIMEOUT) class Regrtest: @@ -491,10 +491,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now - for result in failed: - annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -511,7 +507,14 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) + if not result_cases: + message = "\n".join([str(result), + *(result.env_changed_reasons or ())]) + github_annotation("error", result.test_name, + decolor(message)) for name, traceback in result_cases: + github_annotation("error", f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a860ad0f9c8549..3475f645729182 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,14 +1,13 @@ -import os import sys import trace -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration, github_annotation) + printlist, count, format_duration) # Python uses exit code 1 when an exception is not caught @@ -20,16 +19,6 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult) -> None: - cases = (result.errors or []) + (result.failures or []) - for name, traceback in cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) - if not cases: - message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, decolor(message)) - - class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -139,11 +128,6 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun - and result.is_failed(fail_env_changed)): - # Only annotate tests which fail again when re-run - annotate_github(result) - if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") From ba449b42c06f60b25d70a5a2d2f12d559bc16e7f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:01:32 -0700 Subject: [PATCH 11/37] Annotate inline with test output --- Lib/test/libregrtest/main.py | 16 ++++++------ Lib/test/libregrtest/result.py | 39 +++++++++++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 2 ++ Lib/test/libregrtest/utils.py | 25 ------------------ 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dc6e8ebc232fcc..9df71bbcf009b9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, github_annotation, EXIT_TIMEOUT) + get_process_memory_usage, EXIT_TIMEOUT) class Regrtest: @@ -395,6 +395,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + if runtests.rerun and result.is_failed(runtests.fail_env_changed): + result.print_github_annotations() return result @@ -491,6 +493,11 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now. + # Otherwise, they were annotated when they failed again. + for result in failed: + result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -507,14 +514,7 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) - if not result_cases: - message = "\n".join([str(result), - *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, - decolor(message)) for name, traceback in result_cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c..e806de29139a98 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,11 +1,12 @@ import dataclasses import json -from _colorize import get_colors # type: ignore[import-not-found] +import os +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning) + format_duration, normalize_test_name, print_warning, traceback_location) @dataclasses.dataclass(slots=True) @@ -178,6 +179,40 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + annotations = [(f"{self.test_name}: {name}", traceback) + for name, traceback in (self.errors or []) + + (self.failures or [])] + if not annotations: + # No test case details: crash, timeout, env changed, etc. + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + annotations = [(self.test_name, message)] + for title, message in annotations: + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::error {props_text}::{escape(message)}", flush=True) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..d88787fb30613e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,6 +624,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + result.print_github_annotations() return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ee241844098c40..e39a995662bc91 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -174,31 +174,6 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(level: str, title: str, message: str) -> None: - """Emit a GitHub Actions annotation (workflow command). - - Point it at the last frame of the traceback in message, if any. - """ - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::{level} {props_text}::{escape(message)}", flush=True) - - def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 59b1608fc0b40afcad22e75576d990ea8e4c64b4 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:18:53 -0700 Subject: [PATCH 12/37] Move annotations before test output --- Lib/test/libregrtest/result.py | 4 +--- Lib/test/libregrtest/run_workers.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index e806de29139a98..f2fe90cab9a871 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -191,9 +191,7 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - annotations = [(f"{self.test_name}: {name}", traceback) - for name, traceback in (self.errors or []) - + (self.failures or [])] + annotations = (self.errors or []) + (self.failures or []) if not annotations: # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index d88787fb30613e..2000b0b469e254 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,6 +613,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + # Annotate before the test output + result.print_github_annotations() # Display worker stdout if not self.runtests.output_on_failure: @@ -624,8 +627,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - result.print_github_annotations() return result From 75c44b00d81e346f3ec56338a4ae6402dc9cad45 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:05:54 -0700 Subject: [PATCH 13/37] Have regrtest workers print annotations in better place --- Lib/test/libregrtest/main.py | 9 +++-- Lib/test/libregrtest/result.py | 53 ++++++++++++++--------------- Lib/test/libregrtest/run_workers.py | 5 ++- Lib/test/libregrtest/runtests.py | 2 ++ Lib/test/libregrtest/setup.py | 3 ++ Lib/test/libregrtest/testresult.py | 22 +++++++++++- Lib/test/libregrtest/utils.py | 23 +++++++++++++ 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9df71bbcf009b9..2df99a0fa19b12 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -284,7 +284,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + # Annotate tests which fail again + github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -395,8 +397,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.rerun and result.is_failed(runtests.fail_env_changed): - result.print_github_annotations() + if runtests.github_annotations: + result.print_github_crash_annotation(runtests.fail_env_changed) return result @@ -559,6 +561,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + github_annotations=False, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index f2fe90cab9a871..6ed97a2148b5b4 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,12 +1,12 @@ import dataclasses import json import os -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning, traceback_location) + format_duration, normalize_test_name, print_warning, github_annotation) @dataclasses.dataclass(slots=True) @@ -179,37 +179,34 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") + def get_github_annotations(self) -> list[tuple[str, str]]: + """(test case name, message) of each failed test case. + Without test case details (crash, timeout, env changed, etc.), return + the test name and the test result. + """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return [] annotations = (self.errors or []) + (self.failures or []) if not annotations: - # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) annotations = [(self.test_name, message)] - for title, message in annotations: - message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::error {props_text}::{escape(message)}", flush=True) + return annotations + + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + for title, message in self.get_github_annotations(): + print(github_annotation(title, message), flush=True) + + def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + """Annotate a failed test without test case failures. + + Test case failures are annotated by the test runner, where they are + reported (see RegressionTestResult.printErrorList()). + """ + if (self.is_failed(fail_env_changed) + and not self.errors and not self.failures): + self.print_github_annotations() def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 2000b0b469e254..45bf9d9327fa14 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,9 +613,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - # Annotate before the test output - result.print_github_annotations() + if self.runtests.github_annotations: + result.print_github_crash_annotation(self.runtests.fail_env_changed) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a60..a116b6e4166fc6 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Print a GitHub Actions error annotation before each failure report + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..7531e69d0988e6 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -119,6 +119,9 @@ def setup_tests(runtests: RunTests) -> None: else: support.junit_xml_list = None + from .testresult import RegressionTestResult + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations + if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb..cfd024a000e8a3 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,25 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + if not self.GITHUB_ANNOTATIONS: + super().printErrorList(flavour, errors) + return + stream = self.stream + for test, err in errors: + # Precede each failure report with a GitHub Actions annotation + # which has the report as message, so that the annotation links + # to the report in the job log. + self.stream = unittest.runner._WritelnDecorator(io.StringIO()) + try: + super().printErrorList(flavour, [(test, err)]) + report = self.stream.getvalue() + finally: + self.stream = stream + stream.writeln(github_annotation(str(test), report)) + stream.write(report) + stream.flush() + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e39a995662bc91..d4218fffbf64ae 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -14,6 +14,7 @@ import tempfile import textwrap import types +from _colorize import decolor # type: ignore[import-not-found] from collections.abc import Callable _winapi: types.ModuleType | None try: @@ -174,6 +175,28 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location +def github_annotation(title: str, message: str) -> str: + """Format a GitHub Actions error annotation. + + Locate it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + return f"::error {props_text}::{escape(message.rstrip())}" + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 1222c34f70a4e66c4ad7c321a7ce7e16df51898e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:37:38 -0700 Subject: [PATCH 14/37] Full annotations from worker --- Lib/test/libregrtest/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d4218fffbf64ae..4953a024d35598 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -187,14 +187,22 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) + message = decolor(message).rstrip() props: dict[str, str | int] = {} + header = title if location := traceback_location(message): props |= location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + header = f"{position}: {title}" props["title"] = title + # The job log only shows the message, not the properties: start the + # message with the location and the title + message = f"{header}\n{message}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) - return f"::error {props_text}::{escape(message.rstrip())}" + return f"::error {props_text}::{escape(message)}" def regrtest_unraisable_hook(unraisable) -> None: From 287df00e91602e457d2bbb839fa9c4a7ad42f3fa Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:53:04 -0700 Subject: [PATCH 15/37] Remove redundant printing --- Lib/test/libregrtest/testresult.py | 19 ++++++------------- Lib/test/libregrtest/utils.py | 9 ++++++--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index cfd024a000e8a3..6ad6c2acdc272d 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -135,20 +135,13 @@ def printErrorList(self, flavour, errors): if not self.GITHUB_ANNOTATIONS: super().printErrorList(flavour, errors) return - stream = self.stream for test, err in errors: - # Precede each failure report with a GitHub Actions annotation - # which has the report as message, so that the annotation links - # to the report in the job log. - self.stream = unittest.runner._WritelnDecorator(io.StringIO()) - try: - super().printErrorList(flavour, [(test, err)]) - report = self.stream.getvalue() - finally: - self.stream = stream - stream.writeln(github_annotation(str(test), report)) - stream.write(report) - stream.flush() + # Precede each failure report with a GitHub Actions annotation, + # so that the annotation links to the report in the job log. The + # report follows: don't repeat it in the annotation. + self.stream.writeln(github_annotation(str(test), err, + include_message=False)) + super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 4953a024d35598..8e04ba960be45f 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,10 +175,13 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, *, + include_message: bool = True) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. + Locate it at the last frame of the traceback in message, if any. If + include_message is false, the annotation only has the location and the + title. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -199,7 +202,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" + message = f"{header}\n{message}" if include_message else header props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 53024ed83c9815984b7f1caba450f018aebb4985 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:32:22 -0700 Subject: [PATCH 16/37] Remove special summary case, more full output everywhere --- Lib/test/libregrtest/main.py | 11 ++++------- Lib/test/libregrtest/result.py | 28 +++++----------------------- Lib/test/libregrtest/testresult.py | 5 +++++ 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2df99a0fa19b12..58e011ac752ec8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,7 +285,6 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False, - # Annotate tests which fail again github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) @@ -495,11 +494,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now. - # Otherwise, they were annotated when they failed again. - for result in failed: - result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -561,7 +555,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=False, + # Only annotate failures of the last run: the re-run, if any + github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not (self.want_rerun + and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6ed97a2148b5b4..1bb9f3a9a48dd3 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,6 +1,5 @@ import dataclasses import json -import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any @@ -179,34 +178,17 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def get_github_annotations(self) -> list[tuple[str, str]]: - """(test case name, message) of each failed test case. - - Without test case details (crash, timeout, env changed, etc.), return - the test name and the test result. - """ - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return [] - annotations = (self.errors or []) + (self.failures or []) - if not annotations: - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - annotations = [(self.test_name, message)] - return annotations - - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - for title, message in self.get_github_annotations(): - print(github_annotation(title, message), flush=True) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: """Annotate a failed test without test case failures. - Test case failures are annotated by the test runner, where they are - reported (see RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Test case failures are + annotated by the test runner, where they are reported (see + RegressionTestResult.printErrorList()). """ if (self.is_failed(fail_env_changed) and not self.errors and not self.failures): - self.print_github_annotations() + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print(github_annotation(self.test_name, message), flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 6ad6c2acdc272d..dabd0f5870c01e 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -154,11 +154,16 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): + # Wrap the stream as TextTestRunner does, for printErrors() + stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) + if self.result.GITHUB_ANNOTATIONS: + # Report annotated failures, as in verbose mode + self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): From c2d1d79dbc2f8247041baf525c0a45ec9f79ba32 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:58:11 -0700 Subject: [PATCH 17/37] Better message in the annotations --- Lib/test/libregrtest/testresult.py | 6 ++---- Lib/test/libregrtest/utils.py | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index dabd0f5870c01e..36ee03fc8ed9f7 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -137,10 +137,8 @@ def printErrorList(self, flavour, errors): return for test, err in errors: # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log. The - # report follows: don't repeat it in the annotation. - self.stream.writeln(github_annotation(str(test), err, - include_message=False)) + # so that the annotation links to the report in the job log + self.stream.writeln(github_annotation(str(test), err)) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 8e04ba960be45f..95e69454b90a5e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,13 +175,30 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str, *, - include_message: bool = True) -> str: +def traceback_exception(traceback: str) -> str: + """The exception which ends a traceback: type, message and notes. + + Return traceback unchanged if it has no frame. + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if lines[index].startswith(' File "'): + break + else: + return traceback + # Skip the frame's indented source line and ~^ markers + index += 1 + while index < len(lines) and lines[index].startswith(" "): + index += 1 + return "\n".join(lines[index:]) + + +def github_annotation(title: str, message: str) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. If - include_message is false, the annotation only has the location and the - title. + message is a traceback or a failure description. Locate the annotation at + the last frame of the traceback, if any, and only keep the exception which + ends the traceback: the job log has the full traceback. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -202,7 +219,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" if include_message else header + message = f"{header}\n{traceback_exception(message).strip()}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From c3cca61c6863cac97df8f0dfc87d7c618f79c80f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 14:25:57 -0700 Subject: [PATCH 18/37] Tweak message --- Lib/test/libregrtest/utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 95e69454b90a5e..489acc56a2491f 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -207,19 +207,12 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message).rstrip() + message = decolor(message) props: dict[str, str | int] = {} - header = title if location := traceback_location(message): props |= location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - header = f"{position}: {title}" props["title"] = title - # The job log only shows the message, not the properties: start the - # message with the location and the title - message = f"{header}\n{traceback_exception(message).strip()}" + message = traceback_exception(message).strip() props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From b125f6fe7b184cc95ab363a6696552dbebb8b650 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 15:14:34 -0700 Subject: [PATCH 19/37] Move to simpler regex traceback info extraction --- Lib/test/libregrtest/testresult.py | 6 ++- Lib/test/libregrtest/utils.py | 80 +++++++----------------------- 2 files changed, 24 insertions(+), 62 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 36ee03fc8ed9f7..7011120936242f 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -138,7 +138,11 @@ def printErrorList(self, flavour, errors): for test, err in errors: # Precede each failure report with a GitHub Actions annotation, # so that the annotation links to the report in the job log - self.stream.writeln(github_annotation(str(test), err)) + # Locate the annotation in the test file + case = getattr(test, "test_case", test) # subTest() + module = sys.modules.get(type(case).__module__) + self.stream.writeln(github_annotation( + str(test), err, getattr(module, "__file__", None))) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 489acc56a2491f..1eb31fd3bc7b54 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,6 +1,5 @@ import contextlib import faulthandler -import linecache import locale import math import os.path @@ -142,63 +141,13 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -def traceback_location(traceback: str) -> dict[str, str | int] | None: - """Location of the last frame of a traceback, relative to the repository. - - Return a dict with file and line, plus col and endColumn when the - traceback underlines the failing expression (~~~^^^). - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if match := re.match(r' File "(.+)", line (\d+)', lines[index]): - break - else: - return None - filename, lineno = match.group(1), int(match.group(2)) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): - return None - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} - - # The frame is followed by the dedented source line (" code") and, - # for a single-line expression, by a line of ~ and ^ markers. - markers = lines[index + 2] if index + 2 < len(lines) else "" - source = linecache.getline(filename, lineno) - if markers.strip() and not markers.strip(" ~^") and source: - indent = len(source) - len(source.lstrip()) - start = len(markers) - len(markers.lstrip()) - location["col"] = indent + start - 4 + 1 - location["endColumn"] = indent + len(markers.rstrip()) - 4 - return location - - -def traceback_exception(traceback: str) -> str: - """The exception which ends a traceback: type, message and notes. - - Return traceback unchanged if it has no frame. - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if lines[index].startswith(' File "'): - break - else: - return traceback - # Skip the frame's indented source line and ~^ markers - index += 1 - while index < len(lines) and lines[index].startswith(" "): - index += 1 - return "\n".join(lines[index:]) - - -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, + filename: str | None = None) -> str: """Format a GitHub Actions error annotation. - message is a traceback or a failure description. Locate the annotation at - the last frame of the traceback, if any, and only keep the exception which - ends the traceback: the job log has the full traceback. + message is a traceback or a failure description. Only keep the exception + which ends the traceback: the job log has the full traceback. Locate the + annotation at the last frame of the traceback in filename, if any. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -208,12 +157,21 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location + props: dict[str, str] = {} + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if filename and filename.startswith(stdlib_dir): + lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE) + if lines: + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props |= {"file": f"Lib/{relpath}", "line": lines[-1]} props["title"] = title - message = traceback_exception(message).strip() - props_text = ",".join(f"{key}={escape_property(str(value))}" + # Strip the frames: each " File" line and its indented source lines + message = re.split(r'^ File .*\n(?: .*\n)*', message, + flags=re.MULTILINE)[-1].strip() + props_text = ",".join(f"{key}={escape_property(value)}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 3f320dd94958495a1d8e2edb2d958465a3e40e3b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 20:16:37 -0700 Subject: [PATCH 20/37] lots of review changes --- Lib/test/libregrtest/main.py | 31 ++-- Lib/test/libregrtest/result.py | 20 +- Lib/test/libregrtest/run_workers.py | 2 +- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/libregrtest/setup.py | 4 +- Lib/test/libregrtest/testresult.py | 32 ++-- Lib/test/libregrtest/utils.py | 39 ++-- Lib/test/test_regrtest.py | 173 ++++++++++++++++++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 5 +- 9 files changed, 253 insertions(+), 55 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 58e011ac752ec8..ccac65aed1e25e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import html import os import random import re @@ -397,7 +398,7 @@ def run_test( self.results.accumulate_result(result, runtests) if runtests.github_annotations: - result.print_github_crash_annotation(runtests.fail_env_changed) + result.print_github_annotation(runtests) return result @@ -487,16 +488,16 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") + # Tests which failed in the last run (the re-run, if any) + failed = self.results.rerun_results # Only write a summary on failure: a CI run has many test jobs exitcode = self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - if not filename or not exitcode: + if not filename or not failed or not exitcode: return - # Tests which failed in the last run (the re-run, if any) - failed = self.results.rerun_results cases = [(result.errors or []) + (result.failures or []) for result in failed] - ncase = sum(map(len, cases)) + ncases = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: def write(text: str) -> None: # Separate Markdown blocks with an empty line @@ -504,17 +505,18 @@ def write(text: str) -> None: write(f"## {decolor(self.get_state())}: " f"{count(len(failed), 'test file')} and " - f"{count(ncase, 'test case')} failed") + f"{count(ncases, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - # Expand short tracebacks when there are only a few - is_open = ncase <= 5 and traceback.count("\n") < 30 + # Expand the tracebacks if they fit on a screen: up to 5 + # test cases with a traceback of less than 30 lines + is_open = ncases <= 5 and traceback.count("\n") < 30 write(f"" - f"{name}") + f"{html.escape(name)}") write(f"```pytb\n{decolor(traceback).rstrip()}\n```") write("") @@ -538,6 +540,12 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: + # Annotate test failures in the GitHub Actions job log of the last + # run (the re-run, if any), if it reports failures + will_rerun = self.want_rerun and not self.python_cmd + github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not will_rerun + and bool(self.verbose or self.output_on_failure)) return RunTests( tests, fail_fast=self.fail_fast, @@ -555,10 +563,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate failures of the last run: the re-run, if any - github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not (self.want_rerun - and not self.python_cmd)), + github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 1bb9f3a9a48dd3..6cf816fad1438a 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,10 +1,12 @@ import dataclasses +import importlib.util import json from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, + StrJSON, TestName, FilterTuple, abs_module_name, format_duration, normalize_test_name, print_warning, github_annotation) @@ -178,17 +180,21 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + def print_github_annotation(self, runtests: RunTests) -> None: """Annotate a failed test without test case failures. - For example: crash, timeout, env changed. Test case failures are - annotated by the test runner, where they are reported (see - RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Locate the annotation in the + test file. Test case failures are annotated by the test runner, where + they are reported (see RegressionTestResult.printErrorList()). """ - if (self.is_failed(fail_env_changed) + if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print(github_annotation(self.test_name, message), flush=True) + spec = importlib.util.find_spec( + abs_module_name(self.test_name, runtests.test_dir)) + filename = spec.origin if spec is not None else None + print(github_annotation(self.test_name, message, filename), + flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 45bf9d9327fa14..9bc220e6db5209 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -614,7 +614,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) if self.runtests.github_annotations: - result.print_github_crash_annotation(self.runtests.fail_env_changed) + result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index a116b6e4166fc6..06b1ea34a8140d 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,7 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Print a GitHub Actions error annotation before each failure report + # Annotate test failures in the GitHub Actions job log github_annotations: bool coverage: bool memory_limit: str | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 7531e69d0988e6..21734461d155cc 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,14 +112,12 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + from .testresult import RegressionTestResult if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - - from .testresult import RegressionTestResult RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 7011120936242f..625a5ea6816aa2 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,6 +4,7 @@ import functools import io +import re import sys import time import traceback @@ -132,19 +133,25 @@ def addUnexpectedSuccess(self, test): super().addUnexpectedSuccess(test) def printErrorList(self, flavour, errors): - if not self.GITHUB_ANNOTATIONS: - super().printErrorList(flavour, errors) - return for test, err in errors: - # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log - # Locate the annotation in the test file - case = getattr(test, "test_case", test) # subTest() - module = sys.modules.get(type(case).__module__) - self.stream.writeln(github_annotation( - str(test), err, getattr(module, "__file__", None))) + if self.GITHUB_ANNOTATIONS: + # Write the annotation just before the failure report, so + # that it links to the report in the job log + self.stream.writeln(github_annotation(str(test), err, + self._test_file(test))) super().printErrorList(flavour, [(test, err)]) + @staticmethod + def _test_file(test): + # Test id: "module.Class.method", "module.function" (doctest), or + # "setUpClass (module.Class)" (error in a class or module fixture) + name = getattr(test, 'test_case', test).id() # subTest() + if match := re.fullmatch(r'\w+ \((.+)\)', name): + name = match[1] + while name and name not in sys.modules: + name = name.rpartition('.')[0] + return getattr(sys.modules.get(name), '__file__', None) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -156,16 +163,11 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): - # Wrap the stream as TextTestRunner does, for printErrors() - stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) - if self.result.GITHUB_ANNOTATIONS: - # Report annotated failures, as in verbose mode - self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 1eb31fd3bc7b54..ec83a0de0d5bc4 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -141,14 +141,30 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def repository_path(filename: str) -> str: + """Path of a stdlib file relative to the repository. + + Map the stdlib directory (Lib/ in a source checkout or an installed + lib/python3.X/) to Lib/. Return other paths unchanged. + """ + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return filename + return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + + def github_annotation(title: str, message: str, - filename: str | None = None) -> str: + filename: str | None) -> str: """Format a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation at the last frame of the traceback in filename, if any. + annotation in filename, the test file, at the last frame of the traceback + in this file, if any. """ + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") + def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") .replace("\n", "%0A")) @@ -157,18 +173,15 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str] = {} - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if filename and filename.startswith(stdlib_dir): - lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', - message, re.MULTILINE) - if lines: - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props |= {"file": f"Lib/{relpath}", "line": lines[-1]} + props = {"file": repository_path(filename)} + # Line of the last traceback frame (' File "filename", line 123') or + # failed doctest example (same, but not indented) in filename + if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE): + props["line"] = lines[-1] props["title"] = title - # Strip the frames: each " File" line and its indented source lines + # Only keep the exception: strip the traceback frames, each one a + # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() props_text = ",".join(f"{key}={escape_property(value)}" diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e4..ed5533bfe971ed 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -856,6 +856,8 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) + # Don't annotate the GitHub Actions job of the test suite + env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1639,6 +1641,177 @@ def test_env_changed(self): success=True), stats=2) + def create_marked_test(self, name, code): + # Create a test: return its name and the line number of each of its + # "# marker" comments + code = textwrap.dedent(code) + name = self.create_test(name, code) + markers = {match[1]: str(lineno) + for lineno, text in enumerate(code.splitlines(), 1) + if (match := re.search(r'# (\w+)$', text))} + return name, markers + + def run_tests_github(self, *args, exitcode=0): + # Run tests as in GitHub Actions: return the output and the job + # summary (None if regrtest didn't write it) + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + self.addCleanup(os_helper.unlink, filename) + os_helper.unlink(filename) + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + env['GITHUB_STEP_SUMMARY'] = filename + output = self.run_tests(*args, env=env, exitcode=exitcode) + try: + with open(filename, encoding='utf-8') as fp: + return output, fp.read() + except FileNotFoundError: + return output, None + + @staticmethod + def parse_github_annotations(output): + # Return the (title, file, line) of each "::error" annotation + def unescape(text): + return (text.replace('%3A', ':').replace('%2C', ',') + .replace('%0A', '\n').replace('%0D', '\r') + .replace('%25', '%')) + + annotations = [] + for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): + props = dict(prop.split('=', 1) for prop in match[1].split(',')) + props = {key: unescape(value) for key, value in props.items()} + annotations.append((props['title'], props['file'], + props.get('line'))) + return annotations + + def test_github_annotations(self): + # Every test failure is annotated once in the GitHub Actions job log, + # located in the test file, at the failing line of the test file if + # there is a traceback. The job summary lists the failures. + cases, lines = self.create_marked_test('github_cases', """ + import doctest, json, sys, unittest + + def load_tests(loader, tests, pattern): + tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) + return tests + + def doctest_fail(): + ''' + >>> 1 + 1 # doctest + 3 + ''' + + class Tests(unittest.TestCase): + def test_error(self): + json.loads("{") # error + + def test_fail(self): + self.assertEqual(1, 2) # fail + + def test_subtest(self): + with self.subTest(x=1.5): + self.fail("subtest") # subtest + + class SetUpClassTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + raise ValueError("setUpClass") # setUpClass + + def test_never(self): + pass + """) + setup_module, setup_module_lines = self.create_marked_test( + 'github_setup_module', """ + import unittest + + def setUpModule(): + raise ValueError("setUpModule") # setUpModule + + class Tests(unittest.TestCase): + def test_never(self): + pass + """) + env_changed, _ = self.create_marked_test('github_env_changed', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_env_changed(self): + os.environ["REGRTEST_GITHUB_ENV_CHANGED"] = "1" + """) + crash, _ = self.create_marked_test('github_crash', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_crash(self): + os._exit(1) + """) + + def path(name): + return os.path.join(self.tmptestdir, f'{name}.py') + + # Test cases: (title, file, line) + test_cases = [ + (f'test_error ({cases}.Tests.test_error)', + path(cases), lines['error']), + (f'test_fail ({cases}.Tests.test_fail)', + path(cases), lines['fail']), + (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', + path(cases), lines['subtest']), + # Doctest examples are subtests: "[0]" is the example index + (f'doctest_fail ({cases}) [0]', + path(cases), lines['doctest']), + (f'setUpClass ({cases}.SetUpClassTests)', + path(cases), lines['setUpClass']), + (f'setUpModule ({setup_module})', + path(setup_module), setup_module_lines['setUpModule']), + ] + # Test files: heading in the job summary + test_files = { + cases: f'### {cases} failed (2 errors, 3 failures)', + setup_module: f'### {setup_module} failed (1 error)', + env_changed: f'### {env_changed} failed (env changed)', + crash: f'### {crash} worker non-zero exit code', + } + + for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + tests = list(test_files) + if '-j0' in args: + # A crash kills the main process + tests.remove(crash) + with self.subTest(args=args): + output, summary = self.run_tests_github( + '--fail-env-changed', *args, *tests, + exitcode=EXITCODE_BAD_TEST) + + self.assertCountEqual( + self.parse_github_annotations(output), + [*test_cases, + *((name, path(name), None) + for name in (env_changed, crash) if name in tests)], + output) + + # The job summary lists the failed tests in completion order + summary_lines = summary.splitlines() + self.assertEqual(summary_lines[0], + f'## FAILURE: {len(tests)} test files and ' + f'{len(test_cases)} test cases failed') + self.assertCountEqual( + [line for line in summary_lines + if line.startswith('### ')], + [test_files[name] for name in tests]) + self.assertIn('- os.environ was modified', summary_lines) + self.assertCountEqual( + re.findall(r'(.*)', summary), + [title for title, _, _ in test_cases]) + self.assertEqual(summary.count('```pytb\n'), + len(test_cases)) + + def test_github_summary_success(self): + # No annotation and no job summary when all tests pass + testname = self.create_test() + output, summary = self.run_tests_github(testname) + self.assertNotIn('::error', output) + self.assertIsNone(summary) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst index ebeca7174e5903..a1c9e648e5c744 100644 --- a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -1,2 +1,3 @@ -Individual test failures and an overall summary is now written by -:mod:`test.regrtest` to the Github Action Summary. +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary. From 9b1ce485f33570cc756c8ca3424a0d040982de39 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 21:20:44 -0700 Subject: [PATCH 21/37] Add comment --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ed5533bfe971ed..25d9c036439613 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,7 +1772,10 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + # Only failures of the last run are annotated: with --rerun and -W, + # failures of the first run are reported, but not annotated + for args in (['-j2', '-W'], ['-j2', '--rerun'], + ['-j2', '-W', '--rerun'], ['-j0', '-W']): tests = list(test_files) if '-j0' in args: # A crash kills the main process From b5610603c83b2ccd5f209b3a61998bd159ff31d1 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 09:56:03 -0700 Subject: [PATCH 22/37] Annotations after failure, better test_regrtest coverage --- Lib/test/libregrtest/testresult.py | 4 +-- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 625a5ea6816aa2..45a363b155cb3d 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -134,12 +134,12 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: + super().printErrorList(flavour, [(test, err)]) if self.GITHUB_ANNOTATIONS: - # Write the annotation just before the failure report, so + # Write the annotation just after the failure report, so # that it links to the report in the job log self.stream.writeln(github_annotation(str(test), err, self._test_file(test))) - super().printErrorList(flavour, [(test, err)]) @staticmethod def _test_file(test): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 25d9c036439613..0cecacc4b076d2 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,25 +1772,44 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - # Only failures of the last run are annotated: with --rerun and -W, - # failures of the first run are reported, but not annotated - for args in (['-j2', '-W'], ['-j2', '--rerun'], - ['-j2', '-W', '--rerun'], ['-j0', '-W']): - tests = list(test_files) - if '-j0' in args: - # A crash kills the main process - tests.remove(crash) - with self.subTest(args=args): + # Representative command lines of the CI jobs + command_lines = { + # "make ci", Windows, macOS, installed Python; the JIT jobs use + # the equivalent "-j0 --verbose2 --verbose3". Only failures of + # the re-run are annotated, not the reported first failures. + 'fast-ci': ['--fast-ci', '-j2'], + # Sanitizers and Hypothesis: env changed is not a failure + 'parallel': ['-j2', '-W'], + # iOS and WASI run tests in the main process: a crash kills it + 'single-process': ['--fast-ci', '--single-process'], + # Profile task of PGO and BOLT builds: no annotations + 'pgo': ['--pgo'], + } + for name, args in command_lines.items(): + with self.subTest(name): + tests = list(test_files) + if '--single-process' in args or '--pgo' in args: + tests.remove(crash) output, summary = self.run_tests_github( - '--fail-env-changed', *args, *tests, - exitcode=EXITCODE_BAD_TEST) - - self.assertCountEqual( - self.parse_github_annotations(output), - [*test_cases, - *((name, path(name), None) - for name in (env_changed, crash) if name in tests)], - output) + *args, *tests, exitcode=EXITCODE_BAD_TEST) + + annotations = [] + if '--pgo' not in args: + annotations += test_cases + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), + None)) + if crash in tests: + annotations.append((crash, path(crash), None)) + self.assertCountEqual(self.parse_github_annotations(output), + annotations, output) + # Each test case is annotated right after its failure report + for title, _, _ in annotations[:len(test_cases)]: + title = re.escape(title) + self.assertRegex( + output, + rf'(?m)^(?:ERROR|FAIL): {title}\n' + rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() From e506d7d940bbcb119a0d9aec20405197a6adf7f5 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 10:31:23 -0700 Subject: [PATCH 23/37] annotate env changed better. pgo runs --- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/run_workers.py | 5 +++-- Lib/test/libregrtest/save_env.py | 19 +++++++++++++++- Lib/test/test_regrtest.py | 35 ++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ccac65aed1e25e..290bc628f2e735 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -541,11 +541,12 @@ def display_summary(self) -> None: def create_run_tests(self, tests: TestTuple) -> RunTests: # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures + # run (the re-run, if any), if it reports failures: -v, -W or --pgo will_rerun = self.want_rerun and not self.python_cmd github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) and not will_rerun - and bool(self.verbose or self.output_on_failure)) + and bool(self.verbose or self.output_on_failure + or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 9bc220e6db5209..4a83f838987046 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,8 +613,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: @@ -626,6 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Annotate after the output: env changed warnings, crash traceback + if self.runtests.github_annotations: + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 4149210fa17310..6a902eaf91f5f4 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,6 +139,18 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) + @staticmethod + def describe_os_environ(original, current): + # Only list the keys: values can be secrets + before, after = original[2], current[2] + changes = ( + ('added', after.keys() - before.keys()), + ('removed', before.keys() - after.keys()), + ('changed', {key for key in before.keys() & after.keys() + if before[key] != after[key]}), + ) + return '; '.join(f'{change} {", ".join(sorted(keys))}' + for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -347,7 +359,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - support.set_environment_altered(f"{name} was modified") + reason = f"{name} was modified" + if name == 'os.environ': + delta = self.describe_os_environ(original, current) + if delta: + reason = f"{reason}: {delta}" + support.set_environment_altered(reason) restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0cecacc4b076d2..a85304262a1d7e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,6 +38,7 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats +from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -1782,7 +1783,7 @@ def path(name): 'parallel': ['-j2', '-W'], # iOS and WASI run tests in the main process: a crash kills it 'single-process': ['--fast-ci', '--single-process'], - # Profile task of PGO and BOLT builds: no annotations + # Profile task of PGO and BOLT builds: failures stop the build 'pgo': ['--pgo'], } for name, args in command_lines.items(): @@ -1793,14 +1794,11 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = [] - if '--pgo' not in args: - annotations += test_cases - if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), - None)) - if crash in tests: - annotations.append((crash, path(crash), None)) + annotations = list(test_cases) + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), None)) + if crash in tests: + annotations.append((crash, path(crash), None)) self.assertCountEqual(self.parse_github_annotations(output), annotations, output) # Each test case is annotated right after its failure report @@ -1810,6 +1808,13 @@ def path(name): output, rf'(?m)^(?:ERROR|FAIL): {title}\n' rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') + # The env changed annotation comes right after the warnings + if '--fast-ci' in args: + self.assertRegex( + output, + rf'(?m)^Warning -- os\.environ was modified by ' + rf'{env_changed}\n(?:Warning -- .*\n)*' + rf'::error .*title={env_changed}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() @@ -1820,7 +1825,9 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified', summary_lines) + self.assertIn('- os.environ was modified: ' + 'added REGRTEST_GITHUB_ENV_CHANGED', + summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2950,6 +2957,14 @@ def worker(): class TestUtils(unittest.TestCase): + def test_describe_os_environ(self): + describe = saved_test_environment.describe_os_environ + before = {'A': '1', 'B': '2', 'C': '3'} + after = {'A': '1', 'B': 'secret', 'D': '4'} + self.assertEqual(describe((0, None, before), (0, None, after)), + 'added D; removed C; changed B') + self.assertEqual(describe((0, None, before), (1, None, before)), '') + def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From 2ca705b3556086a24fa82cff579a7ec021e41931 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:20:39 -0700 Subject: [PATCH 24/37] simplify, rely on env var everywhere. Stop using findspec --- Lib/test/libregrtest/main.py | 28 ++++++++----------- Lib/test/libregrtest/result.py | 21 ++++++++------ Lib/test/libregrtest/run_workers.py | 3 +- Lib/test/libregrtest/runtests.py | 2 -- Lib/test/libregrtest/save_env.py | 19 +------------ Lib/test/libregrtest/setup.py | 1 - Lib/test/libregrtest/testresult.py | 12 ++++---- Lib/test/libregrtest/utils.py | 27 +++++++++--------- Lib/test/libregrtest/worker.py | 2 -- .../test_live_collector_ui.py | 6 +++- Lib/test/test_regrtest.py | 18 +++--------- 11 files changed, 55 insertions(+), 84 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 290bc628f2e735..20b94eff8aa3fd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,8 +285,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False, - github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + output_on_failure=False) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -397,8 +396,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.github_annotations: - result.print_github_annotation(runtests) + result.print_github_annotation(runtests) return result @@ -540,13 +538,6 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: - # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures: -v, -W or --pgo - will_rerun = self.want_rerun and not self.python_cmd - github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not will_rerun - and bool(self.verbose or self.output_on_failure - or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, @@ -564,7 +555,6 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -627,10 +617,16 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if use_load_tracker: self.logger.start_load_tracker() try: - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - else: - self.run_tests_sequentially(runtests) + with os_helper.EnvironmentVarGuard() as env: + # In GitHub Actions, only annotate failures of the last run: + # not the first run if failed tests will be re-run (with + # --python, they are not: see rerun_failed_tests()) + if self.want_rerun and not self.python_cmd: + env.unset("GITHUB_STEP_SUMMARY") + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) coverage = self.results.get_coverage_results() self.display_result(runtests) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6cf816fad1438a..e1d05777598f2e 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,13 +1,15 @@ import dataclasses -import importlib.util import json +import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, abs_module_name, - format_duration, normalize_test_name, print_warning, github_annotation) + StrJSON, TestName, FilterTuple, + format_duration, normalize_test_name, print_warning, + print_github_annotation) @dataclasses.dataclass(slots=True) @@ -190,11 +192,14 @@ def print_github_annotation(self, runtests: RunTests) -> None: if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - spec = importlib.util.find_spec( - abs_module_name(self.test_name, runtests.test_dir)) - filename = spec.origin if spec is not None else None - print(github_annotation(self.test_name, message, filename), - flush=True) + # Test file: "test_x" is test_x.py or test_x/__init__.py, + # "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = os.path.join(findtestdir(runtests.test_dir), *name.split(".")) + filename = path + ".py" + if not os.path.exists(filename): + filename = os.path.join(path, "__init__.py") + print_github_annotation(self.test_name, message, filename) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 4a83f838987046..995543d676876c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -625,8 +625,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: if stdout: print(stdout, flush=True) # Annotate after the output: env changed warnings, crash traceback - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 06b1ea34a8140d..d86b082fb34a60 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,8 +93,6 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Annotate test failures in the GitHub Actions job log - github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 6a902eaf91f5f4..4149210fa17310 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,18 +139,6 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) - @staticmethod - def describe_os_environ(original, current): - # Only list the keys: values can be secrets - before, after = original[2], current[2] - changes = ( - ('added', after.keys() - before.keys()), - ('removed', before.keys() - after.keys()), - ('changed', {key for key in before.keys() & after.keys() - if before[key] != after[key]}), - ) - return '; '.join(f'{change} {", ".join(sorted(keys))}' - for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -359,12 +347,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - reason = f"{name} was modified" - if name == 'os.environ': - delta = self.describe_os_environ(original, current) - if delta: - reason = f"{reason}: {delta}" - support.set_environment_altered(reason) + support.set_environment_altered(f"{name} was modified") restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 21734461d155cc..0c7fad58fd03bf 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -118,7 +118,6 @@ def setup_tests(runtests: RunTests) -> None: RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 45a363b155cb3d..9c453a9ef58516 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -10,11 +10,10 @@ import traceback import unittest from test import support -from test.libregrtest.utils import github_annotation, sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -135,11 +134,10 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: super().printErrorList(flavour, [(test, err)]) - if self.GITHUB_ANNOTATIONS: - # Write the annotation just after the failure report, so - # that it links to the report in the job log - self.stream.writeln(github_annotation(str(test), err, - self._test_file(test))) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + print_github_annotation(str(test), err, self._test_file(test), + file=self.stream) @staticmethod def _test_file(test): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ec83a0de0d5bc4..60921d5e018a12 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -153,25 +153,25 @@ def repository_path(filename: str) -> str: return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") -def github_annotation(title: str, message: str, - filename: str | None) -> str: - """Format a GitHub Actions error annotation. +# Escape the data and the properties of GitHub Actions workflow commands +GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) +GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", + ",": "%2C"}) + + +def print_github_annotation(title: str, message: str, filename: str | None, + file=None) -> None: + """Print a GitHub Actions error annotation, if run in GitHub Actions. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the annotation in filename, the test file, at the last frame of the traceback in this file, if any. """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return if not filename: raise ValueError(f"missing test file of annotation {title!r}") - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) props = {"file": repository_path(filename)} # Line of the last traceback frame (' File "filename", line 123') or @@ -184,9 +184,10 @@ def escape_property(text: str) -> str: # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() - props_text = ",".join(f"{key}={escape_property(value)}" + props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items()) - return f"::error {props_text}::{escape(message)}" + print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", + file=file, flush=True) def regrtest_unraisable_hook(unraisable) -> None: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 24ae95ed23914f..4e69ab9d8fad05 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,8 +28,6 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) - # Only the main process writes the GitHub Actions job summary - env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index ae4ee969f34d8b..57598715f8b288 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -11,7 +11,8 @@ import time import unittest from unittest import mock -from test.support import requires, requires_remote_subprocess_debugging +from test.support import ( + os_helper, requires, requires_remote_subprocess_debugging) from test.support.import_helper import import_module # Only run these tests if curses is available @@ -856,6 +857,9 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" + # Don't write to the GitHub Actions job summary of the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') args = [ "profiling.sampling.cli", "run", "--live", "-m", "test", diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a85304262a1d7e..0de0e76665baf5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,7 +38,6 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats -from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -625,6 +624,9 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) + # Don't annotate the GitHub Actions job running the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) @@ -857,8 +859,6 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) - # Don't annotate the GitHub Actions job of the test suite - env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1825,9 +1825,7 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified: ' - 'added REGRTEST_GITHUB_ENV_CHANGED', - summary_lines) + self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2957,14 +2955,6 @@ def worker(): class TestUtils(unittest.TestCase): - def test_describe_os_environ(self): - describe = saved_test_environment.describe_os_environ - before = {'A': '1', 'B': '2', 'C': '3'} - after = {'A': '1', 'B': 'secret', 'D': '4'} - self.assertEqual(describe((0, None, before), (0, None, after)), - 'added D; removed C; changed B') - self.assertEqual(describe((0, None, before), (1, None, before)), '') - def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From fe082633b9111239bdc45f6da806111c7ede82dc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:45:15 -0700 Subject: [PATCH 25/37] Simplify filename extraction, use pathlib --- Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/testresult.py | 27 ++++---- Lib/test/libregrtest/utils.py | 16 +++-- Lib/test/test_regrtest.py | 101 ++++++++++++++--------------- 4 files changed, 73 insertions(+), 73 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 0c7fad58fd03bf..8909225d0265bf 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,8 +112,8 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - from .testresult import RegressionTestResult if runtests.use_junit: + from .testresult import RegressionTestResult support.junit_xml_list = [] RegressionTestResult.USE_XML = True else: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 9c453a9ef58516..bdce67226fff3b 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,7 +4,6 @@ import functools import io -import re import sys import time import traceback @@ -12,6 +11,16 @@ from test import support from test.libregrtest.utils import print_github_annotation, sanitize_xml +def test_to_filename(test): + test = getattr(test, 'test_case', test) # subTest() + if dt_test := getattr(test, '_dt_test', None): # doctest.DocTestCase + return dt_test.filename + module = type(test).__module__ + # Fixture errors (setUpClass, setUpModule) are not supported + if module.startswith('unittest.'): + return None + return getattr(sys.modules.get(module), '__file__', None) + class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -136,19 +145,9 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - print_github_annotation(str(test), err, self._test_file(test), - file=self.stream) - - @staticmethod - def _test_file(test): - # Test id: "module.Class.method", "module.function" (doctest), or - # "setUpClass (module.Class)" (error in a class or module fixture) - name = getattr(test, 'test_case', test).id() # subTest() - if match := re.fullmatch(r'\w+ \((.+)\)', name): - name = match[1] - while name and name not in sys.modules: - name = name.rpartition('.')[0] - return getattr(sys.modules.get(name), '__file__', None) + if filename := test_to_filename(test): + print_github_annotation(str(test), err, filename, + file=self.stream) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 60921d5e018a12..50de4ae9ab6d67 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -3,6 +3,7 @@ import locale import math import os.path +import pathlib import platform import random import re @@ -141,16 +142,21 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/ +STDLIB_DIR = pathlib.Path(os.__file__).parent +# Stdlib directory relative to the repository +REPOSITORY_LIB_DIR = pathlib.Path("Lib") + + def repository_path(filename: str) -> str: """Path of a stdlib file relative to the repository. - Map the stdlib directory (Lib/ in a source checkout or an installed - lib/python3.X/) to Lib/. Return other paths unchanged. + Map STDLIB_DIR to Lib/. Return other paths unchanged. """ - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): + path = pathlib.Path(filename) + if not path.is_relative_to(STDLIB_DIR): return filename - return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix() # Escape the data and the properties of GitHub Actions workflow commands diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0de0e76665baf5..fa569207bee336 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1668,22 +1668,6 @@ def run_tests_github(self, *args, exitcode=0): except FileNotFoundError: return output, None - @staticmethod - def parse_github_annotations(output): - # Return the (title, file, line) of each "::error" annotation - def unescape(text): - return (text.replace('%3A', ':').replace('%2C', ',') - .replace('%0A', '\n').replace('%0D', '\r') - .replace('%25', '%')) - - annotations = [] - for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): - props = dict(prop.split('=', 1) for prop in match[1].split(',')) - props = {key: unescape(value) for key, value in props.items()} - annotations.append((props['title'], props['file'], - props.get('line'))) - return annotations - def test_github_annotations(self): # Every test failure is annotated once in the GitHub Actions job log, # located in the test file, at the failing line of the test file if @@ -1715,17 +1699,17 @@ def test_subtest(self): class SetUpClassTests(unittest.TestCase): @classmethod def setUpClass(cls): - raise ValueError("setUpClass") # setUpClass + raise ValueError("setUpClass") def test_never(self): pass """) - setup_module, setup_module_lines = self.create_marked_test( + setup_module, _ = self.create_marked_test( 'github_setup_module', """ import unittest def setUpModule(): - raise ValueError("setUpModule") # setUpModule + raise ValueError("setUpModule") class Tests(unittest.TestCase): def test_never(self): @@ -1746,24 +1730,23 @@ def test_crash(self): os._exit(1) """) - def path(name): - return os.path.join(self.tmptestdir, f'{name}.py') - - # Test cases: (title, file, line) + # Annotated test cases: (title, file, line) test_cases = [ (f'test_error ({cases}.Tests.test_error)', - path(cases), lines['error']), + f'{cases}.py', lines['error']), (f'test_fail ({cases}.Tests.test_fail)', - path(cases), lines['fail']), + f'{cases}.py', lines['fail']), (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', - path(cases), lines['subtest']), + f'{cases}.py', lines['subtest']), # Doctest examples are subtests: "[0]" is the example index (f'doctest_fail ({cases}) [0]', - path(cases), lines['doctest']), - (f'setUpClass ({cases}.SetUpClassTests)', - path(cases), lines['setUpClass']), - (f'setUpModule ({setup_module})', - path(setup_module), setup_module_lines['setUpModule']), + f'{cases}.py', lines['doctest']), + ] + # Test cases only listed in the job summary: fixture errors + # (setUpClass, setUpModule) are not annotated + failed_titles = [title for title, _, _ in test_cases] + [ + f'setUpClass ({cases}.SetUpClassTests)', + f'setUpModule ({setup_module})', ] # Test files: heading in the job summary test_files = { @@ -1794,33 +1777,45 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = list(test_cases) + expected = list(test_cases) if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), None)) + expected.append((env_changed, f'{env_changed}.py', None)) if crash in tests: - annotations.append((crash, path(crash), None)) - self.assertCountEqual(self.parse_github_annotations(output), - annotations, output) - # Each test case is annotated right after its failure report - for title, _, _ in annotations[:len(test_cases)]: - title = re.escape(title) - self.assertRegex( - output, - rf'(?m)^(?:ERROR|FAIL): {title}\n' - rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') - # The env changed annotation comes right after the warnings - if '--fast-ci' in args: - self.assertRegex( - output, - rf'(?m)^Warning -- os\.environ was modified by ' - rf'{env_changed}\n(?:Warning -- .*\n)*' - rf'::error .*title={env_changed}::') + expected.append((crash, f'{crash}.py', None)) + + # "::error file=...,line=...,title=...::message" + annotations = [] + output_lines = output.splitlines() + for i, line in enumerate(output_lines): + if not line.startswith('::error '): + continue + _, props, message = line.split('::', 2) + props = props.removeprefix('error ').split(',') + props = dict(prop.split('=', 1) for prop in props) + title = props['title'] + annotations.append((title, os.path.basename(props['file']), + props.get('line'))) + before = output_lines[:i] + if title == env_changed: + # Right after the env changed warnings + self.assertStartsWith(before[-1], 'Warning -- ') + elif title != crash: + # Right after the failure report of the test, which + # ends with the exception and an empty line + report = next( + text for text in reversed(before) + if text.startswith(('ERROR: ', 'FAIL: '))) + self.assertEqual(report.split(': ', 1)[1], title) + message = message.replace('%0A', '\n').splitlines() + self.assertEqual(before[-len(message) - 1:], + message + ['']) + self.assertCountEqual(annotations, expected, output) # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() self.assertEqual(summary_lines[0], f'## FAILURE: {len(tests)} test files and ' - f'{len(test_cases)} test cases failed') + f'{len(failed_titles)} test cases failed') self.assertCountEqual( [line for line in summary_lines if line.startswith('### ')], @@ -1828,9 +1823,9 @@ def path(name): self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), - [title for title, _, _ in test_cases]) + failed_titles) self.assertEqual(summary.count('```pytb\n'), - len(test_cases)) + len(failed_titles)) def test_github_summary_success(self): # No annotation and no job summary when all tests pass From b5247ede1141cf61367a8a98daecaca4a5b26062 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 8 Oct 2026 13:31:15 -0700 Subject: [PATCH 26/37] Simplify test --- Lib/test/test_regrtest.py | 207 +++++++++++++------------------------- 1 file changed, 71 insertions(+), 136 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index fa569207bee336..2775a80e77ef8b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1642,38 +1642,33 @@ def test_env_changed(self): success=True), stats=2) - def create_marked_test(self, name, code): - # Create a test: return its name and the line number of each of its - # "# marker" comments - code = textwrap.dedent(code) - name = self.create_test(name, code) - markers = {match[1]: str(lineno) - for lineno, text in enumerate(code.splitlines(), 1) - if (match := re.search(r'# (\w+)$', text))} - return name, markers - def run_tests_github(self, *args, exitcode=0): - # Run tests as in GitHub Actions: return the output and the job - # summary (None if regrtest didn't write it) + # Run tests as in GitHub Actions: return the output, the annotations + # as (file, line, title) tuples and the job summary ('' if not written) filename = os.path.join(self.tmptestdir, 'github_step_summary.md') - self.addCleanup(os_helper.unlink, filename) os_helper.unlink(filename) - env = dict(os.environ) + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) env.pop('SOURCE_DATE_EPOCH', None) - env['GITHUB_STEP_SUMMARY'] = filename output = self.run_tests(*args, env=env, exitcode=exitcode) + annotations = re.findall(r'^::error file=[^,]*?([^/,]+)' + r'(?:,line=(\d+))?,title=([^:]*)::', + output, re.MULTILINE) + # Every annotation has a file and a title + self.assertEqual(len(annotations), + len(re.findall('^::error', output, re.MULTILINE)), + output) try: with open(filename, encoding='utf-8') as fp: - return output, fp.read() + return output, annotations, fp.read() except FileNotFoundError: - return output, None + return output, annotations, '' def test_github_annotations(self): - # Every test failure is annotated once in the GitHub Actions job log, - # located in the test file, at the failing line of the test file if - # there is a traceback. The job summary lists the failures. - cases, lines = self.create_marked_test('github_cases', """ - import doctest, json, sys, unittest + # Every failure is annotated in the GitHub Actions job log, located + # in its test file, at the failing line ("# fail") for test cases. + # The job summary lists the failures. + code = textwrap.dedent(""" + import doctest, sys, unittest def load_tests(loader, tests, pattern): tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) @@ -1681,158 +1676,98 @@ def load_tests(loader, tests, pattern): def doctest_fail(): ''' - >>> 1 + 1 # doctest + >>> 1 + 1 # fail 3 ''' class Tests(unittest.TestCase): def test_error(self): - json.loads("{") # error + {}['key'] # fail def test_fail(self): self.assertEqual(1, 2) # fail def test_subtest(self): - with self.subTest(x=1.5): - self.fail("subtest") # subtest + with self.subTest(x=1): + self.fail() # fail class SetUpClassTests(unittest.TestCase): + # Fixture errors are only listed in the job summary @classmethod def setUpClass(cls): - raise ValueError("setUpClass") + raise ValueError def test_never(self): pass """) - setup_module, _ = self.create_marked_test( - 'github_setup_module', """ - import unittest - - def setUpModule(): - raise ValueError("setUpModule") - - class Tests(unittest.TestCase): - def test_never(self): - pass - """) - env_changed, _ = self.create_marked_test('github_env_changed', """ + cases = self.create_test('github_cases', code) + fail_lines = {str(lineno) + for lineno, line in enumerate(code.splitlines(), 1) + if line.endswith('# fail')} + env_changed = self.create_test('github_env_changed', textwrap.dedent(""" import os, unittest class Tests(unittest.TestCase): def test_env_changed(self): - os.environ["REGRTEST_GITHUB_ENV_CHANGED"] = "1" - """) - crash, _ = self.create_marked_test('github_crash', """ + os.environ['REGRTEST_GITHUB_ENV_CHANGED'] = '1' + """)) + crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest class Tests(unittest.TestCase): def test_crash(self): os._exit(1) - """) - - # Annotated test cases: (title, file, line) - test_cases = [ - (f'test_error ({cases}.Tests.test_error)', - f'{cases}.py', lines['error']), - (f'test_fail ({cases}.Tests.test_fail)', - f'{cases}.py', lines['fail']), - (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', - f'{cases}.py', lines['subtest']), - # Doctest examples are subtests: "[0]" is the example index - (f'doctest_fail ({cases}) [0]', - f'{cases}.py', lines['doctest']), + """)) + case_titles = [ + f'test_error ({cases}.Tests.test_error)', + f'test_fail ({cases}.Tests.test_fail)', + f'test_subtest ({cases}.Tests.test_subtest) (x=1)', + f'doctest_fail ({cases}) [0]', ] - # Test cases only listed in the job summary: fixture errors - # (setUpClass, setUpModule) are not annotated - failed_titles = [title for title, _, _ in test_cases] + [ - f'setUpClass ({cases}.SetUpClassTests)', - f'setUpModule ({setup_module})', - ] - # Test files: heading in the job summary - test_files = { - cases: f'### {cases} failed (2 errors, 3 failures)', - setup_module: f'### {setup_module} failed (1 error)', - env_changed: f'### {env_changed} failed (env changed)', - crash: f'### {crash} worker non-zero exit code', - } # Representative command lines of the CI jobs - command_lines = { - # "make ci", Windows, macOS, installed Python; the JIT jobs use - # the equivalent "-j0 --verbose2 --verbose3". Only failures of - # the re-run are annotated, not the reported first failures. - 'fast-ci': ['--fast-ci', '-j2'], - # Sanitizers and Hypothesis: env changed is not a failure - 'parallel': ['-j2', '-W'], - # iOS and WASI run tests in the main process: a crash kills it - 'single-process': ['--fast-ci', '--single-process'], - # Profile task of PGO and BOLT builds: failures stop the build - 'pgo': ['--pgo'], - } - for name, args in command_lines.items(): - with self.subTest(name): - tests = list(test_files) - if '--single-process' in args or '--pgo' in args: - tests.remove(crash) - output, summary = self.run_tests_github( + for args in ( + # "make ci": only the failures of the re-run are annotated + ['--fast-ci', '-j2'], + # Sanitizers: env changed is not a failure + ['-j2', '-W'], + # iOS and WASI: a crash would kill the main process + ['--fast-ci', '--single-process'], + # PGO profile task + ['--pgo'], + ): + with self.subTest(args=args): + in_process = '--single-process' in args or '--pgo' in args + tests = [cases, env_changed] + ([] if in_process else [crash]) + output, annotations, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - expected = list(test_cases) + expected = [(cases, title) for title in case_titles] if '--fast-ci' in args: - expected.append((env_changed, f'{env_changed}.py', None)) + expected.append((env_changed, env_changed)) if crash in tests: - expected.append((crash, f'{crash}.py', None)) - - # "::error file=...,line=...,title=...::message" - annotations = [] - output_lines = output.splitlines() - for i, line in enumerate(output_lines): - if not line.startswith('::error '): - continue - _, props, message = line.split('::', 2) - props = props.removeprefix('error ').split(',') - props = dict(prop.split('=', 1) for prop in props) - title = props['title'] - annotations.append((title, os.path.basename(props['file']), - props.get('line'))) - before = output_lines[:i] - if title == env_changed: - # Right after the env changed warnings - self.assertStartsWith(before[-1], 'Warning -- ') - elif title != crash: - # Right after the failure report of the test, which - # ends with the exception and an empty line - report = next( - text for text in reversed(before) - if text.startswith(('ERROR: ', 'FAIL: '))) - self.assertEqual(report.split(': ', 1)[1], title) - message = message.replace('%0A', '\n').splitlines() - self.assertEqual(before[-len(message) - 1:], - message + ['']) - self.assertCountEqual(annotations, expected, output) - - # The job summary lists the failed tests in completion order - summary_lines = summary.splitlines() - self.assertEqual(summary_lines[0], - f'## FAILURE: {len(tests)} test files and ' - f'{len(failed_titles)} test cases failed') + expected.append((crash, crash)) self.assertCountEqual( - [line for line in summary_lines - if line.startswith('### ')], - [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified', summary_lines) - self.assertCountEqual( - re.findall(r'(.*)', summary), - failed_titles) - self.assertEqual(summary.count('```pytb\n'), - len(failed_titles)) + [(file, title) for file, _, title in annotations], + [(f'{name}.py', title) for name, title in expected], + output) + for file, line, title in annotations: + # Test files without a failed test case have no line + self.assertIn(line, fail_lines if file == f'{cases}.py' + else {''}, title) + + self.assertIn(f'## FAILURE: {len(tests)} test files and ' + f'5 test cases failed', summary) + for name in tests: + self.assertIn(f'### {name} ', summary) + self.assertIn('- os.environ was modified', summary) + self.assertEqual(summary.count(''), 5) def test_github_summary_success(self): # No annotation and no job summary when all tests pass - testname = self.create_test() - output, summary = self.run_tests_github(testname) - self.assertNotIn('::error', output) - self.assertIsNone(summary) + output, annotations, summary = self.run_tests_github(self.create_test()) + self.assertEqual(annotations, []) + self.assertEqual(summary, '') def test_rerun_fail(self): # FAILURE then FAILURE From 9a94bf16399bdc49f341b8c638e2f6e1bd1bfd40 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:39:03 -0700 Subject: [PATCH 27/37] Simplify GitHub Actions annotations Locate annotations at the last traceback frame in a test file instead of mapping each test to its file. Pass annotations to workers in RunTests, like use_junit, so tests which run regrtest don't inherit GITHUB_STEP_SUMMARY. Shrink the tests. Co-Authored-By: Claude Opus 5.5 --- Lib/test/libregrtest/main.py | 30 ++-- Lib/test/libregrtest/result.py | 3 +- Lib/test/libregrtest/runtests.py | 1 + Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/single.py | 4 +- Lib/test/libregrtest/testresult.py | 16 +-- Lib/test/libregrtest/utils.py | 50 +++---- .../test_live_collector_ui.py | 6 +- Lib/test/test_regrtest.py | 131 ++++-------------- 9 files changed, 74 insertions(+), 169 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 20b94eff8aa3fd..b6bf35e057f01d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,7 +285,8 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + github_annotations=bool(self.github_summary)) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -485,18 +486,17 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: self.write_github_summary() def write_github_summary(self) -> None: - filename = os.environ.get("GITHUB_STEP_SUMMARY") # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results # Only write a summary on failure: a CI run has many test jobs exitcode = self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - if not filename or not failed or not exitcode: + if not self.github_summary or not failed or not exitcode: return cases = [(result.errors or []) + (result.failures or []) for result in failed] ncases = sum(map(len, cases)) - with open(filename, "a", encoding="utf-8") as fp: + with open(self.github_summary, "a", encoding="utf-8") as fp: def write(text: str) -> None: # Separate Markdown blocks with an empty line print(text, end="\n\n", file=fp) @@ -555,6 +555,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + # Only annotate failures of the last run: not the first run if + # failed tests will be re-run (with --python, they are not) + github_annotations=(bool(self.github_summary) and + not (self.want_rerun and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -617,16 +621,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if use_load_tracker: self.logger.start_load_tracker() try: - with os_helper.EnvironmentVarGuard() as env: - # In GitHub Actions, only annotate failures of the last run: - # not the first run if failed tests will be re-run (with - # --python, they are not: see rerun_failed_tests()) - if self.want_rerun and not self.python_cmd: - env.unset("GITHUB_STEP_SUMMARY") - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - else: - self.run_tests_sequentially(runtests) + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) coverage = self.results.get_coverage_results() self.display_result(runtests) @@ -808,6 +806,10 @@ def _init(self): strip_py_suffix(self.cmdline_args) + # Pass GitHub Actions annotations to workers in RunTests, not to + # regrtest run by tests + self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) + self._tmp_dir = get_temp_dir(self._tmp_dir) @property diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index e1d05777598f2e..fafa41e21c49f2 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -189,7 +189,8 @@ def print_github_annotation(self, runtests: RunTests) -> None: test file. Test case failures are annotated by the test runner, where they are reported (see RegressionTestResult.printErrorList()). """ - if (self.is_failed(runtests.fail_env_changed) + if (runtests.github_annotations + and self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) # Test file: "test_x" is test_x.py or test_x/__init__.py, diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a60..541a8146001cab 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 8909225d0265bf..d62194acd9c29e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -113,8 +113,8 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) if runtests.use_junit: - from .testresult import RegressionTestResult support.junit_xml_list = [] + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a6..4f755501301c13 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -16,7 +16,7 @@ from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests -from .testresult import get_test_runner +from .testresult import RegressionTestResult, get_test_runner from .parallel_case import ParallelTestCase from .utils import ( TestName, @@ -37,6 +37,8 @@ def run_unittest(test_mod, runtests: RunTests): if loader.errors: raise Exception("errors while loading tests") _filter_suite(tests, match_test) + RegressionTestResult.GITHUB_TEST_FILE = ( + test_mod.__file__ if runtests.github_annotations else None) if runtests.parallel_threads: _parallelize_tests(tests, runtests.parallel_threads) return _run_suite(tests) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index bdce67226fff3b..58ceae44a9dc67 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -11,18 +11,10 @@ from test import support from test.libregrtest.utils import print_github_annotation, sanitize_xml -def test_to_filename(test): - test = getattr(test, 'test_case', test) # subTest() - if dt_test := getattr(test, '_dt_test', None): # doctest.DocTestCase - return dt_test.filename - module = type(test).__module__ - # Fixture errors (setUpClass, setUpModule) are not supported - if module.startswith('unittest.'): - return None - return getattr(sys.modules.get(module), '__file__', None) - class RegressionTestResult(unittest.TextTestResult): USE_XML = False + # Test file of the GitHub Actions annotations: don't annotate if None + GITHUB_TEST_FILE: str | None = None def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -145,8 +137,8 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - if filename := test_to_filename(test): - print_github_annotation(str(test), err, filename, + if self.GITHUB_TEST_FILE: + print_github_annotation(str(test), err, self.GITHUB_TEST_FILE, file=self.stream) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 50de4ae9ab6d67..5a6a6f57a22747 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -142,56 +142,40 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/ -STDLIB_DIR = pathlib.Path(os.__file__).parent -# Stdlib directory relative to the repository -REPOSITORY_LIB_DIR = pathlib.Path("Lib") - - -def repository_path(filename: str) -> str: - """Path of a stdlib file relative to the repository. - - Map STDLIB_DIR to Lib/. Return other paths unchanged. - """ - path = pathlib.Path(filename) - if not path.is_relative_to(STDLIB_DIR): - return filename - return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix() - - # Escape the data and the properties of GitHub Actions workflow commands GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str | None, +def print_github_annotation(title: str, message: str, filename: str, file=None) -> None: - """Print a GitHub Actions error annotation, if run in GitHub Actions. + """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation in filename, the test file, at the last frame of the traceback - in this file, if any. + annotation at the last traceback frame in a test file, if any, or else + in filename, the test file. """ - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - if not filename: - raise ValueError(f"missing test file of annotation {title!r}") message = decolor(message) - props = {"file": repository_path(filename)} - # Line of the last traceback frame (' File "filename", line 123') or - # failed doctest example (same, but not indented) in filename - if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)', - message, re.MULTILINE): - props["line"] = lines[-1] - props["title"] = title + line = None + # Last traceback frame (' File "filename", line 123') or failed doctest + # example (same, but not indented) in a test file + if frames := re.findall(r'^ *File "(.*[/\\]test_[^/\\]*\.py)", line (\d+)', + message, re.MULTILINE): + filename, line = frames[-1] + # In the job, the stdlib can be outside of the repository checkout + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + props = {"file": filename, "line": line, "title": title} # Only keep the exception: strip the traceback frames, each one a # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" - for key, value in props.items()) + for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", file=file, flush=True) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index 57598715f8b288..ae4ee969f34d8b 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -11,8 +11,7 @@ import time import unittest from unittest import mock -from test.support import ( - os_helper, requires, requires_remote_subprocess_debugging) +from test.support import requires, requires_remote_subprocess_debugging from test.support.import_helper import import_module # Only run these tests if curses is available @@ -857,9 +856,6 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" - # Don't write to the GitHub Actions job summary of the test suite - env = self.enterContext(os_helper.EnvironmentVarGuard()) - env.unset('GITHUB_STEP_SUMMARY') args = [ "profiling.sampling.cli", "run", "--live", "-m", "test", diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2775a80e77ef8b..4fe80533ce9caf 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -624,9 +624,6 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) - # Don't annotate the GitHub Actions job running the test suite - env = self.enterContext(os_helper.EnvironmentVarGuard()) - env.unset('GITHUB_STEP_SUMMARY') self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) @@ -1642,31 +1639,9 @@ def test_env_changed(self): success=True), stats=2) - def run_tests_github(self, *args, exitcode=0): - # Run tests as in GitHub Actions: return the output, the annotations - # as (file, line, title) tuples and the job summary ('' if not written) - filename = os.path.join(self.tmptestdir, 'github_step_summary.md') - os_helper.unlink(filename) - env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) - env.pop('SOURCE_DATE_EPOCH', None) - output = self.run_tests(*args, env=env, exitcode=exitcode) - annotations = re.findall(r'^::error file=[^,]*?([^/,]+)' - r'(?:,line=(\d+))?,title=([^:]*)::', - output, re.MULTILINE) - # Every annotation has a file and a title - self.assertEqual(len(annotations), - len(re.findall('^::error', output, re.MULTILINE)), - output) - try: - with open(filename, encoding='utf-8') as fp: - return output, annotations, fp.read() - except FileNotFoundError: - return output, annotations, '' - - def test_github_annotations(self): - # Every failure is annotated in the GitHub Actions job log, located - # in its test file, at the failing line ("# fail") for test cases. - # The job summary lists the failures. + def test_github_actions(self): + # Failures are annotated in the GitHub Actions job log at the failing + # line ("# fail") of the test file, and listed in the job summary code = textwrap.dedent(""" import doctest, sys, unittest @@ -1681,36 +1656,11 @@ def doctest_fail(): ''' class Tests(unittest.TestCase): - def test_error(self): - {}['key'] # fail - - def test_fail(self): - self.assertEqual(1, 2) # fail - def test_subtest(self): with self.subTest(x=1): self.fail() # fail - - class SetUpClassTests(unittest.TestCase): - # Fixture errors are only listed in the job summary - @classmethod - def setUpClass(cls): - raise ValueError - - def test_never(self): - pass """) cases = self.create_test('github_cases', code) - fail_lines = {str(lineno) - for lineno, line in enumerate(code.splitlines(), 1) - if line.endswith('# fail')} - env_changed = self.create_test('github_env_changed', textwrap.dedent(""" - import os, unittest - - class Tests(unittest.TestCase): - def test_env_changed(self): - os.environ['REGRTEST_GITHUB_ENV_CHANGED'] = '1' - """)) crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest @@ -1718,56 +1668,33 @@ class Tests(unittest.TestCase): def test_crash(self): os._exit(1) """)) - case_titles = [ - f'test_error ({cases}.Tests.test_error)', - f'test_fail ({cases}.Tests.test_fail)', - f'test_subtest ({cases}.Tests.test_subtest) (x=1)', - f'doctest_fail ({cases}) [0]', - ] - - # Representative command lines of the CI jobs - for args in ( - # "make ci": only the failures of the re-run are annotated - ['--fast-ci', '-j2'], - # Sanitizers: env changed is not a failure - ['-j2', '-W'], - # iOS and WASI: a crash would kill the main process - ['--fast-ci', '--single-process'], - # PGO profile task - ['--pgo'], - ): - with self.subTest(args=args): - in_process = '--single-process' in args or '--pgo' in args - tests = [cases, env_changed] + ([] if in_process else [crash]) - output, annotations, summary = self.run_tests_github( - *args, *tests, exitcode=EXITCODE_BAD_TEST) - - expected = [(cases, title) for title in case_titles] - if '--fast-ci' in args: - expected.append((env_changed, env_changed)) - if crash in tests: - expected.append((crash, crash)) - self.assertCountEqual( - [(file, title) for file, _, title in annotations], - [(f'{name}.py', title) for name, title in expected], - output) - for file, line, title in annotations: - # Test files without a failed test case have no line - self.assertIn(line, fail_lines if file == f'{cases}.py' - else {''}, title) - - self.assertIn(f'## FAILURE: {len(tests)} test files and ' - f'5 test cases failed', summary) - for name in tests: - self.assertIn(f'### {name} ', summary) - self.assertIn('- os.environ was modified', summary) - self.assertEqual(summary.count(''), 5) - - def test_github_summary_success(self): + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) + env.pop('SOURCE_DATE_EPOCH', None) + output = self.run_tests('--fast-ci', '-j2', cases, crash, + env=env, exitcode=EXITCODE_BAD_TEST) + fail_lines = [str(lineno) + for lineno, line in enumerate(code.splitlines(), 1) + if line.endswith('# fail')] + # Only failures of the re-run are annotated + self.assertCountEqual( + re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,' + r'title=([^:]*)::', output, re.MULTILINE), + [(f'{cases}.py', fail_lines[1], + f'test_subtest ({cases}.Tests.test_subtest) (x=1)'), + (f'{cases}.py', fail_lines[0], f'doctest_fail ({cases}) [0]'), + (f'{crash}.py', '', crash)], + output) + with open(filename, encoding='utf-8') as fp: + summary = fp.read() + self.assertIn('## FAILURE: 2 test files and 2 test cases failed', + summary) + # No annotation and no job summary when all tests pass - output, annotations, summary = self.run_tests_github(self.create_test()) - self.assertEqual(annotations, []) - self.assertEqual(summary, '') + os_helper.unlink(filename) + output = self.run_tests(self.create_test(), env=env) + self.assertNotIn('::error', output) + self.assertFalse(os.path.exists(filename)) def test_rerun_fail(self): # FAILURE then FAILURE From 70cae9942c7c8a47653aed00b390ef8e8acd519f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:43:02 -0700 Subject: [PATCH 28/37] Set GITHUB_ANNOTATIONS in setup_tests(), like USE_XML Fall back to the first traceback frame (the test) instead of passing the test module file. Co-Authored-By: Claude Opus 5.5 --- Lib/test/libregrtest/setup.py | 3 ++- Lib/test/libregrtest/single.py | 4 +--- Lib/test/libregrtest/testresult.py | 8 +++----- Lib/test/libregrtest/utils.py | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..74e7132c3d52aa 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,9 +112,10 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + from .testresult import RegressionTestResult + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 4f755501301c13..d7a1c6bea202a6 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -16,7 +16,7 @@ from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests -from .testresult import RegressionTestResult, get_test_runner +from .testresult import get_test_runner from .parallel_case import ParallelTestCase from .utils import ( TestName, @@ -37,8 +37,6 @@ def run_unittest(test_mod, runtests: RunTests): if loader.errors: raise Exception("errors while loading tests") _filter_suite(tests, match_test) - RegressionTestResult.GITHUB_TEST_FILE = ( - test_mod.__file__ if runtests.github_annotations else None) if runtests.parallel_threads: _parallelize_tests(tests, runtests.parallel_threads) return _run_suite(tests) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 58ceae44a9dc67..9efa11cd0cf5cd 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -13,8 +13,7 @@ class RegressionTestResult(unittest.TextTestResult): USE_XML = False - # Test file of the GitHub Actions annotations: don't annotate if None - GITHUB_TEST_FILE: str | None = None + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -137,9 +136,8 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - if self.GITHUB_TEST_FILE: - print_github_annotation(str(test), err, self.GITHUB_TEST_FILE, - file=self.stream) + if self.GITHUB_ANNOTATIONS: + print_github_annotation(str(test), err, file=self.stream) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 5a6a6f57a22747..89e89c1e09a20d 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,22 +148,26 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str, - file=None) -> None: +def print_github_annotation(title: str, message: str, + filename: str | None = None, file=None) -> None: """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation at the last traceback frame in a test file, if any, or else - in filename, the test file. + annotation at the last traceback frame in a test file, or else at the + first frame (the test), or else in filename (the test file). """ message = decolor(message) line = None - # Last traceback frame (' File "filename", line 123') or failed doctest - # example (same, but not indented) in a test file - if frames := re.findall(r'^ *File "(.*[/\\]test_[^/\\]*\.py)", line (\d+)', - message, re.MULTILINE): - filename, line = frames[-1] + # Traceback frames (' File "filename", line 123') and failed doctest + # examples (same, but not indented) + if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, + re.MULTILINE): + test_frames = [frame for frame in frames + if os.path.basename(frame[0]).startswith('test_')] + filename, line = test_frames[-1] if test_frames else frames[0] + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") # In the job, the stdlib can be outside of the repository checkout path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent From 8f231a6c092619f4930cd16143eb05313c21d4f0 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:58:11 -0700 Subject: [PATCH 29/37] Simplify findall --- Lib/test/test_regrtest.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 4fe80533ce9caf..36e81fa5c656c5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1673,17 +1673,15 @@ def test_crash(self): env.pop('SOURCE_DATE_EPOCH', None) output = self.run_tests('--fast-ci', '-j2', cases, crash, env=env, exitcode=EXITCODE_BAD_TEST) - fail_lines = [str(lineno) + # One annotation per failure of the re-run, not of the first run: + # one per "# fail" line, and the crash (without a line) + fail_lines = [(f'{cases}.py', str(lineno)) for lineno, line in enumerate(code.splitlines(), 1) if line.endswith('# fail')] - # Only failures of the re-run are annotated self.assertCountEqual( - re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,' - r'title=([^:]*)::', output, re.MULTILINE), - [(f'{cases}.py', fail_lines[1], - f'test_subtest ({cases}.Tests.test_subtest) (x=1)'), - (f'{cases}.py', fail_lines[0], f'doctest_fail ({cases}) [0]'), - (f'{crash}.py', '', crash)], + re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,title=', + output, re.MULTILINE), + fail_lines + [(f'{crash}.py', '')], output) with open(filename, encoding='utf-8') as fp: summary = fp.read() From f2ba9db49524b950ae264681cc00cdd330c91033 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 18:00:19 -0700 Subject: [PATCH 30/37] Review fixes. Directly check error annotation list --- Lib/test/libregrtest/setup.py | 2 +- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 74e7132c3d52aa..8216541e454f82 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,6 +12,7 @@ from .filter import set_match_tests from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,7 +113,6 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - from .testresult import RegressionTestResult RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 36e81fa5c656c5..f445eb2b6a0760 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1641,8 +1641,8 @@ def test_env_changed(self): def test_github_actions(self): # Failures are annotated in the GitHub Actions job log at the failing - # line ("# fail") of the test file, and listed in the job summary - code = textwrap.dedent(""" + # line of the test file, and listed in the job summary + cases = self.create_test('github_cases', textwrap.dedent(""" import doctest, sys, unittest def load_tests(loader, tests, pattern): @@ -1651,16 +1651,15 @@ def load_tests(loader, tests, pattern): def doctest_fail(): ''' - >>> 1 + 1 # fail + >>> 1 + 1 3 ''' class Tests(unittest.TestCase): def test_subtest(self): with self.subTest(x=1): - self.fail() # fail - """) - cases = self.create_test('github_cases', code) + self.fail() + """)) crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest @@ -1671,18 +1670,40 @@ def test_crash(self): filename = os.path.join(self.tmptestdir, 'github_step_summary.md') env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) env.pop('SOURCE_DATE_EPOCH', None) - output = self.run_tests('--fast-ci', '-j2', cases, crash, + # A single worker without --randomize runs the tests in order + output = self.run_tests('-j1', '--rerun', cases, crash, env=env, exitcode=EXITCODE_BAD_TEST) - # One annotation per failure of the re-run, not of the first run: - # one per "# fail" line, and the crash (without a line) - fail_lines = [(f'{cases}.py', str(lineno)) - for lineno, line in enumerate(code.splitlines(), 1) - if line.endswith('# fail')] - self.assertCountEqual( - re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,title=', - output, re.MULTILINE), - fail_lines + [(f'{crash}.py', '')], - output) + + def unescape(text): + for char in '\r\n:,%': + text = text.replace(f'%{ord(char):02X}', char) + return text + + annotations = [] + for props, message in re.findall(r'^::error (.*?)::(.*)$', output, + re.MULTILINE): + props = dict(prop.split('=', 1) for prop in props.split(',')) + annotations.append(({key: unescape(value) + for key, value in props.items()}, + unescape(message))) + # One annotation per failure of the re-run, not of the first run + cases_file = os.path.join(self.tmptestdir, f'{cases}.py') + crash_file = os.path.join(self.tmptestdir, f'{crash}.py') + self.assertEqual(annotations, [ + ({'file': cases_file, 'line': '17', + 'title': f'test_subtest ({cases}.Tests.test_subtest) (x=1)'}, + 'AssertionError: None'), + ({'file': cases_file, 'line': '10', + 'title': f'doctest_fail ({cases}) [0]'}, + 'AssertionError: Failed example:\n' + ' 1 + 1\n' + 'Expected:\n' + ' 3\n' + 'Got:\n' + ' 2'), + ({'file': crash_file, 'title': crash}, + f'{crash} worker non-zero exit code'), + ], output) with open(filename, encoding='utf-8') as fp: summary = fp.read() self.assertIn('## FAILURE: 2 test files and 2 test cases failed', From 42d7918b3ef8a1b18d92e8cb9efb10bafd3915cd Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 20:29:27 -0700 Subject: [PATCH 31/37] Personal review adjustments, move more into TestRestul --- Lib/test/libregrtest/main.py | 31 ++++------------ Lib/test/libregrtest/result.py | 59 +++++++++++++++++++++--------- Lib/test/libregrtest/setup.py | 3 +- Lib/test/libregrtest/testresult.py | 11 +----- Lib/test/libregrtest/utils.py | 11 +++--- 5 files changed, 56 insertions(+), 59 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b6bf35e057f01d..e039a209db6a35 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,4 +1,3 @@ -import html import os import random import re @@ -493,30 +492,14 @@ def write_github_summary(self) -> None: self.fail_rerun) if not self.github_summary or not failed or not exitcode: return - cases = [(result.errors or []) + (result.failures or []) - for result in failed] - ncases = sum(map(len, cases)) + ncases = sum(len(result.test_cases) for result in failed) with open(self.github_summary, "a", encoding="utf-8") as fp: - def write(text: str) -> None: - # Separate Markdown blocks with an empty line - print(text, end="\n\n", file=fp) - - write(f"## {decolor(self.get_state())}: " - f"{count(len(failed), 'test file')} and " - f"{count(ncases, 'test case')} failed") - for result, result_cases in zip(failed, cases): - write(f"### {decolor(str(result))}") - if result.env_changed_reasons: - write("\n".join(f"- {reason}" - for reason in result.env_changed_reasons)) - for name, traceback in result_cases: - # Expand the tracebacks if they fit on a screen: up to 5 - # test cases with a traceback of less than 30 lines - is_open = ncases <= 5 and traceback.count("\n") < 30 - write(f"" - f"{html.escape(name)}") - write(f"```pytb\n{decolor(traceback).rstrip()}\n```") - write("") + fp.write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed\n\n") + for result in failed: + # Expand the tracebacks of up to 5 test cases + fp.write(result.format_github_summary(expand=ncases <= 5)) def display_summary(self) -> None: if self.first_runtests is None: diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index fafa41e21c49f2..142740b34a44c6 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,7 +1,8 @@ import dataclasses +import html import json -import os -from _colorize import get_colors # type: ignore[import-not-found] +import pathlib +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any from .findtests import findtestdir @@ -182,25 +183,49 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def test_cases(self) -> list[tuple[str, str]]: + """Failed test cases: list of (name, traceback).""" + return (self.errors or []) + (self.failures or []) + + def format_github_summary(self, expand: bool) -> str: + """Format the Markdown section of the GitHub Actions job summary.""" + blocks = [f"### {decolor(str(self))}"] + if self.env_changed_reasons: + blocks.append("\n".join(f"- {reason}" + for reason in self.env_changed_reasons)) + for name, traceback in self.test_cases: + # Expand the tracebacks if they fit on a screen: a traceback of + # less than 30 lines + is_open = expand and traceback.count("\n") < 30 + blocks += [f"" + f"{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + ""] + # Separate Markdown blocks with an empty line + return "".join(f"{block}\n\n" for block in blocks) + def print_github_annotation(self, runtests: RunTests) -> None: - """Annotate a failed test without test case failures. + """Annotate each failed test case, or else the failed test file. - For example: crash, timeout, env changed. Locate the annotation in the - test file. Test case failures are annotated by the test runner, where - they are reported (see RegressionTestResult.printErrorList()). + For example, the test file for a crash, timeout or env changed. """ - if (runtests.github_annotations - and self.is_failed(runtests.fail_env_changed) - and not self.errors and not self.failures): + if not (runtests.github_annotations + and self.is_failed(runtests.fail_env_changed)): + return + # Test file: "test_x" is test_x.py or test_x/__init__.py, + # "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) + filename = path.with_suffix(".py") + if not filename.exists(): + filename = path / "__init__.py" + test_cases = self.test_cases + if not test_cases: message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - # Test file: "test_x" is test_x.py or test_x/__init__.py, - # "test.test_x.test_y" is test_x/test_y.py - name = self.test_name.removeprefix("test.") - path = os.path.join(findtestdir(runtests.test_dir), *name.split(".")) - filename = path + ".py" - if not os.path.exists(filename): - filename = os.path.join(path, "__init__.py") - print_github_annotation(self.test_name, message, filename) + test_cases = [(self.test_name, message)] + for title, message in test_cases: + print_github_annotation(title, message, str(filename)) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 8216541e454f82..d62194acd9c29e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,7 +12,6 @@ from .filter import set_match_tests from .runtests import RunTests -lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -113,9 +112,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 9efa11cd0cf5cd..605f1f4e6a89fb 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,11 +9,10 @@ import traceback import unittest from test import support -from test.libregrtest.utils import print_github_annotation, sanitize_xml +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -131,14 +130,6 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) - def printErrorList(self, flavour, errors): - for test, err in errors: - super().printErrorList(flavour, [(test, err)]) - # Annotate just after the failure report, so that the annotation - # links to the report in the job log - if self.GITHUB_ANNOTATIONS: - print_github_annotation(str(test), err, file=self.stream) - def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 89e89c1e09a20d..7eeb011ce80b23 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,8 +148,7 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, - filename: str | None = None, file=None) -> None: +def print_github_annotation(title: str, message: str, filename: str) -> None: """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception @@ -157,6 +156,8 @@ def print_github_annotation(title: str, message: str, annotation at the last traceback frame in a test file, or else at the first frame (the test), or else in filename (the test file). """ + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") message = decolor(message) line = None # Traceback frames (' File "filename", line 123') and failed doctest @@ -164,10 +165,8 @@ def print_github_annotation(title: str, message: str, if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames - if os.path.basename(frame[0]).startswith('test_')] + if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] - if not filename: - raise ValueError(f"missing test file of annotation {title!r}") # In the job, the stdlib can be outside of the repository checkout path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent @@ -181,7 +180,7 @@ def print_github_annotation(title: str, message: str, props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", - file=file, flush=True) + flush=True) def regrtest_unraisable_hook(unraisable) -> None: From e4d41bfbe5fb30fd079752964b01a55f8c0ef9ff Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 20:53:52 -0700 Subject: [PATCH 32/37] Self review fixes --- Lib/test/libregrtest/main.py | 8 +++----- Lib/test/libregrtest/result.py | 26 +++++++++++--------------- Lib/test/libregrtest/run_workers.py | 1 - Lib/test/libregrtest/utils.py | 17 ++++++----------- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e039a209db6a35..cb7382604f80bd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -492,7 +492,7 @@ def write_github_summary(self) -> None: self.fail_rerun) if not self.github_summary or not failed or not exitcode: return - ncases = sum(len(result.test_cases) for result in failed) + ncases = sum(len(result.failed_test_cases) for result in failed) with open(self.github_summary, "a", encoding="utf-8") as fp: fp.write(f"## {decolor(self.get_state())}: " f"{count(len(failed), 'test file')} and " @@ -538,8 +538,8 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate failures of the last run: not the first run if - # failed tests will be re-run (with --python, they are not) + # Only annotate the failure that results in job failure. + # Ex: When reruning failed tests, only annotate the the final run. github_annotations=(bool(self.github_summary) and not (self.want_rerun and not self.python_cmd)), coverage=self.coverage, @@ -789,8 +789,6 @@ def _init(self): strip_py_suffix(self.cmdline_args) - # Pass GitHub Actions annotations to workers in RunTests, not to - # regrtest run by tests self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) self._tmp_dir = get_temp_dir(self._tmp_dir) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 142740b34a44c6..69071562b14e19 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -184,8 +184,7 @@ def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) @property - def test_cases(self) -> list[tuple[str, str]]: - """Failed test cases: list of (name, traceback).""" + def failed_test_cases(self) -> list[tuple[str, str]]: return (self.errors or []) + (self.failures or []) def format_github_summary(self, expand: bool) -> str: @@ -194,37 +193,34 @@ def format_github_summary(self, expand: bool) -> str: if self.env_changed_reasons: blocks.append("\n".join(f"- {reason}" for reason in self.env_changed_reasons)) - for name, traceback in self.test_cases: - # Expand the tracebacks if they fit on a screen: a traceback of - # less than 30 lines + for name, traceback in self.failed_test_cases: + # Only default open short tracebacks. is_open = expand and traceback.count("\n") < 30 blocks += [f"" f"{html.escape(name)}", f"```pytb\n{decolor(traceback).rstrip()}\n```", ""] + # Separate Markdown blocks with an empty line return "".join(f"{block}\n\n" for block in blocks) def print_github_annotation(self, runtests: RunTests) -> None: - """Annotate each failed test case, or else the failed test file. - - For example, the test file for a crash, timeout or env changed. - """ + """Print github error annotation line for each failure.""" if not (runtests.github_annotations and self.is_failed(runtests.fail_env_changed)): return - # Test file: "test_x" is test_x.py or test_x/__init__.py, - # "test.test_x.test_y" is test_x/test_y.py + # Convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py name = self.test_name.removeprefix("test.") path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) filename = path.with_suffix(".py") if not filename.exists(): filename = path / "__init__.py" - test_cases = self.test_cases - if not test_cases: + annotations = self.failed_test_cases + if not annotations: message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - test_cases = [(self.test_name, message)] - for title, message in test_cases: + annotations = [(self.test_name, message)] + for title, message in annotations: print_github_annotation(title, message, str(filename)) def set_env_changed(self, *reasons): diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 995543d676876c..3327269ae1c622 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,7 +624,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - # Annotate after the output: env changed warnings, crash traceback result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7eeb011ce80b23..2ac30fb4cba5fc 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -149,25 +149,20 @@ def print_warning(msg: str) -> None: def print_github_annotation(title: str, message: str, filename: str) -> None: - """Print a GitHub Actions error annotation. - - message is a traceback or a failure description. Only keep the exception - which ends the traceback: the job log has the full traceback. Locate the - annotation at the last traceback frame in a test file, or else at the - first frame (the test), or else in filename (the test file). - """ + """Print a GitHub Actions error annotation for the failure.""" if not filename: - raise ValueError(f"missing test file of annotation {title!r}") + raise ValueError(f"missing filename to annotate {title!r}") message = decolor(message) + + # Try to get a line number from backtrace. line = None - # Traceback frames (' File "filename", line 123') and failed doctest - # examples (same, but not indented) if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] - # In the job, the stdlib can be outside of the repository checkout + + # Normalize stdlib path to be relative inside the repository checkout. path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent if path.is_relative_to(stdlib_dir): From b0e9c5e42cbe777b0eb76a78b96aefa369c71f14 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 21:45:18 -0700 Subject: [PATCH 33/37] Personal review fixes, adjust output locaiton --- Lib/test/libregrtest/result.py | 36 +++++++++++++++++------------- Lib/test/libregrtest/setup.py | 3 ++- Lib/test/libregrtest/single.py | 1 + Lib/test/libregrtest/testresult.py | 16 ++++++++++++- Lib/test/libregrtest/utils.py | 12 +++++----- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 69071562b14e19..448581d5a6108e 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -108,6 +108,9 @@ class TestResult: # short descriptions of how the test altered the execution environment env_changed_reasons: list[str] | None = None + # test file, set once the test module is imported + filename: str | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -205,23 +208,24 @@ def format_github_summary(self, expand: bool) -> str: return "".join(f"{block}\n\n" for block in blocks) def print_github_annotation(self, runtests: RunTests) -> None: - """Print github error annotation line for each failure.""" - if not (runtests.github_annotations - and self.is_failed(runtests.fail_env_changed)): + """Annotate a failed test file without failed test cases.""" + if (not runtests.github_annotations + or not self.is_failed(runtests.fail_env_changed) + or self.failed_test_cases): return - # Convert python import test name to relative file path. - # Ex: "test.test_x.test_y" is test_x/test_y.py - name = self.test_name.removeprefix("test.") - path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) - filename = path.with_suffix(".py") - if not filename.exists(): - filename = path / "__init__.py" - annotations = self.failed_test_cases - if not annotations: - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - annotations = [(self.test_name, message)] - for title, message in annotations: - print_github_annotation(title, message, str(filename)) + filename = self.filename + if not filename: + # The worker crashed or timed out before reporting the test file: + # convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), + *name.split(".")) + filename = path.with_suffix(".py") + if not filename.exists(): + filename = path / "__init__.py" + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print_github_annotation(self.test_name, message, str(filename)) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..8216541e454f82 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,6 +12,7 @@ from .filter import set_match_tests from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +113,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a6..3aa4ffd00f8f8f 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -153,6 +153,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) test_mod = importlib.import_module(module_name) + result.filename = test_mod.__file__ if hasattr(test_mod, "test_main"): # https://github.com/python/cpython/issues/89392 diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb..79add3d39b373a 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + if self.GITHUB_ANNOTATIONS: + print_github_annotation(str(test), err, stream=self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +155,11 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not reported in quiet mode: annotate them at the end + if self.result.GITHUB_ANNOTATIONS: + for test, err in self.result.errors + self.result.failures: + print_github_annotation(str(test), err, + stream=self.result.stream) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 2ac30fb4cba5fc..6034fa50b00817 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,19 +148,21 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str) -> None: +def print_github_annotation(title: str, message: str, + filename: str | None = None, + stream=None) -> None: """Print a GitHub Actions error annotation for the failure.""" - if not filename: - raise ValueError(f"missing filename to annotate {title!r}") message = decolor(message) - # Try to get a line number from backtrace. + # Try to get a file + line number from backtrace. line = None if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] + if not filename: + raise ValueError(f"missing filename to annotate {title!r}") # Normalize stdlib path to be relative inside the repository checkout. path = pathlib.Path(filename) @@ -175,7 +177,7 @@ def print_github_annotation(title: str, message: str, filename: str) -> None: props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", - flush=True) + file=stream, flush=True) def regrtest_unraisable_hook(unraisable) -> None: From 01ad16c0af64ca57071bd8333314b4532149697f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 21:52:49 -0700 Subject: [PATCH 34/37] Fix mypy --- Lib/test/libregrtest/result.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 448581d5a6108e..8f59fcd124900c 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -221,11 +221,12 @@ def print_github_annotation(self, runtests: RunTests) -> None: name = self.test_name.removeprefix("test.") path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) - filename = path.with_suffix(".py") - if not filename.exists(): - filename = path / "__init__.py" + if path.with_suffix(".py").exists(): + filename = str(path.with_suffix(".py")) + else: + filename = str(path / "__init__.py") message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print_github_annotation(self.test_name, message, str(filename)) + print_github_annotation(self.test_name, message, filename) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: From 475549ace55636c69cb0f0ebfdd227b2d1fcbe2f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 13 Oct 2026 11:49:03 -0700 Subject: [PATCH 35/37] Refactor to use a reporter of which github is one --- Lib/test/libregrtest/main.py | 51 +++++------ Lib/test/libregrtest/reporter.py | 130 ++++++++++++++++++++++++++++ Lib/test/libregrtest/result.py | 47 +--------- Lib/test/libregrtest/results.py | 2 + Lib/test/libregrtest/run_workers.py | 1 - Lib/test/libregrtest/runtests.py | 3 +- Lib/test/libregrtest/setup.py | 3 +- Lib/test/libregrtest/testresult.py | 23 ++--- Lib/test/libregrtest/utils.py | 40 --------- 9 files changed, 171 insertions(+), 129 deletions(-) create mode 100644 Lib/test/libregrtest/reporter.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cb7382604f80bd..3f84af27a4f126 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,7 +6,7 @@ import sysconfig import time import trace -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -15,6 +15,7 @@ from .findtests import findtests, split_test_packages, list_cases, collect_cases from .logger import Logger from .pgo import setup_pgo_tests +from .reporter import get_reporter from .result import TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak @@ -285,7 +286,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False, - github_annotations=bool(self.github_summary)) + github_summary=self.github_summary) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -298,7 +299,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: self.run_tests_sequentially(runtests) return runtests - def rerun_failed_tests(self, runtests: RunTests) -> None: + def rerun_failed_tests(self, runtests: RunTests) -> RunTests: ansi = get_colors() red, reset = ansi.BOLD_RED, ansi.RESET @@ -308,7 +309,7 @@ def rerun_failed_tests(self, runtests: RunTests) -> None: "Re-running failed tests is not supported with --python " "host runner option." ) - return + return runtests self.first_state = self.get_state() @@ -323,6 +324,7 @@ def rerun_failed_tests(self, runtests: RunTests) -> None: printlist(self.results.bad) self.display_result(rerun_runtests) + return rerun_runtests def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: print() @@ -396,7 +398,6 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - result.print_github_annotation(runtests) return result @@ -463,7 +464,8 @@ def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() - def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: + def finalize_tests(self, coverage: trace.CoverageResults | None, + runtests: RunTests) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -482,24 +484,11 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) - self.write_github_summary() - - def write_github_summary(self) -> None: - # Tests which failed in the last run (the re-run, if any) - failed = self.results.rerun_results - # Only write a summary on failure: a CI run has many test jobs - exitcode = self.results.get_exitcode(self.fail_env_changed, - self.fail_rerun) - if not self.github_summary or not failed or not exitcode: - return - ncases = sum(len(result.failed_test_cases) for result in failed) - with open(self.github_summary, "a", encoding="utf-8") as fp: - fp.write(f"## {decolor(self.get_state())}: " - f"{count(len(failed), 'test file')} and " - f"{count(ncases, 'test case')} failed\n\n") - for result in failed: - # Expand the tracebacks of up to 5 test cases - fp.write(result.format_github_summary(expand=ncases <= 5)) + # Test files which failed in the last run (the re-run, if any) + failed = [result for result in self.results.rerun_results + if result.is_failed(self.fail_env_changed)] + if failed: + get_reporter(runtests).run_failed(failed, self.get_state()) def display_summary(self) -> None: if self.first_runtests is None: @@ -538,10 +527,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate the failure that results in job failure. - # Ex: When reruning failed tests, only annotate the the final run. - github_annotations=(bool(self.github_summary) and - not (self.want_rerun and not self.python_cmd)), + # Only report the failures which fail the job: when failed tests + # are re-run, only report the re-run. + github_summary=(None if self.want_rerun and not self.python_cmd + else self.github_summary), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -612,8 +601,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: coverage = self.results.get_coverage_results() self.display_result(runtests) + # The re-run, if any, is the last run + last_runtests = runtests if self.want_rerun and self.results.need_rerun(): - self.rerun_failed_tests(runtests) + last_runtests = self.rerun_failed_tests(runtests) if self.want_bisect and self.results.need_rerun(): self.run_bisect(runtests) @@ -622,7 +613,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.logger.stop_load_tracker() self.display_summary() - self.finalize_tests(coverage) + self.finalize_tests(coverage, last_runtests) return self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py new file mode 100644 index 00000000000000..5a82ad72a1c7e3 --- /dev/null +++ b/Lib/test/libregrtest/reporter.py @@ -0,0 +1,130 @@ +import html +import os +import pathlib +import re +from _colorize import decolor # type: ignore[import-not-found] +from typing import TextIO + +from .findtests import findtestdir +from .result import TestResult +from .runtests import RunTests +from .utils import count + + +class Reporter: + """Report test failures, for example to a CI service. + + The default reporter does nothing. + """ + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + """Report a failed test case. + + Called just after unittest printed the failure, or at the end of the + test file in quiet mode (failures are not printed). + """ + + def test_result(self, result: TestResult, runtests: RunTests) -> None: + """Report the result of a test file, in the main process.""" + + def run_failed(self, failed: list[TestResult], state: str) -> None: + """Report a failed run: failed is the list of failed test files.""" + + +def get_reporter(runtests: RunTests) -> Reporter: + if runtests.github_summary: + return GitHubActionsReporter(runtests.github_summary) + return Reporter() + + +class GitHubActionsReporter(Reporter): + """Report test failures to GitHub Actions. + + Annotate failures in the job log and list them in the job summary. + """ + + # Escape the data and the properties of workflow commands + ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) + ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) + + def __init__(self, summary_filename: str) -> None: + self.summary_filename = summary_filename + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + traceback = decolor(traceback) + # Get a file + line number from the backtrace: the last frame in a + # test file, or else the first frame (the test). Without frames, the + # failure is still annotated, just not located. + filename = line = None + if frames := re.findall(r'^ *File "(.*)", line (\d+)', traceback, + re.MULTILINE): + test_frames = [frame for frame in frames + if pathlib.Path(frame[0]).name.startswith('test_')] + filename, line = test_frames[-1] if test_frames else frames[0] + # Only keep the exception: strip the traceback frames, each one a + # ' File' line followed by its indented source lines + message = re.split(r'^ File .*\n(?: .*\n)*', traceback, + flags=re.MULTILINE)[-1].strip() + self.annotate(test_case, message, filename, line, stream) + + def test_result(self, result: TestResult, runtests: RunTests) -> None: + # Annotate a test file which failed without failed test cases, for + # example: crash, timeout, env changed. Failed test cases are + # annotated by test_case_failed(). + if (not result.is_failed(runtests.fail_env_changed) + or result.failed_test_cases): + return + filename = result.filename + if not filename: + # The worker crashed or timed out before reporting the test file: + # convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py + name = result.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), + *name.split(".")) + if path.with_suffix(".py").exists(): + filename = str(path.with_suffix(".py")) + elif (path / "__init__.py").exists(): + filename = str(path / "__init__.py") + # else not a test file (ex: "" bug): no location + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + self.annotate(result.test_name, decolor(message), filename) + + def annotate(self, title: str, message: str, filename: str | None, + line: str | None = None, stream: TextIO | None = None) -> None: + if filename: + # Normalize stdlib path to be relative inside the repository + # checkout. + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + props = {"file": filename, "line": line, "title": title} + props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" + for key, value in props.items() if value) + print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", + file=stream, flush=True) + + def run_failed(self, failed: list[TestResult], state: str) -> None: + ncases = sum(len(result.failed_test_cases) for result in failed) + blocks = [f"## {decolor(state)}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed"] + for result in failed: + blocks.append(f"### {decolor(str(result))}") + if result.env_changed_reasons: + blocks.append("\n".join(f"- {reason}" + for reason in result.env_changed_reasons)) + for name, traceback in result.failed_test_cases: + # Only default open short tracebacks of up to 5 test cases. + is_open = ncases <= 5 and traceback.count("\n") < 30 + blocks += [f"" + f"{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + ""] + # Separate Markdown blocks with an empty line + with open(self.summary_filename, "a", encoding="utf-8") as fp: + fp.write("".join(f"{block}\n\n" for block in blocks)) + diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 8f59fcd124900c..00e6e5a9767c03 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,16 +1,11 @@ import dataclasses -import html import json -import pathlib -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any -from .findtests import findtestdir -from .runtests import RunTests from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning, - print_github_annotation) + format_duration, normalize_test_name, print_warning) @dataclasses.dataclass(slots=True) @@ -190,44 +185,6 @@ def has_meaningful_duration(self): def failed_test_cases(self) -> list[tuple[str, str]]: return (self.errors or []) + (self.failures or []) - def format_github_summary(self, expand: bool) -> str: - """Format the Markdown section of the GitHub Actions job summary.""" - blocks = [f"### {decolor(str(self))}"] - if self.env_changed_reasons: - blocks.append("\n".join(f"- {reason}" - for reason in self.env_changed_reasons)) - for name, traceback in self.failed_test_cases: - # Only default open short tracebacks. - is_open = expand and traceback.count("\n") < 30 - blocks += [f"" - f"{html.escape(name)}", - f"```pytb\n{decolor(traceback).rstrip()}\n```", - ""] - - # Separate Markdown blocks with an empty line - return "".join(f"{block}\n\n" for block in blocks) - - def print_github_annotation(self, runtests: RunTests) -> None: - """Annotate a failed test file without failed test cases.""" - if (not runtests.github_annotations - or not self.is_failed(runtests.fail_env_changed) - or self.failed_test_cases): - return - filename = self.filename - if not filename: - # The worker crashed or timed out before reporting the test file: - # convert python import test name to relative file path. - # Ex: "test.test_x.test_y" is test_x/test_y.py - name = self.test_name.removeprefix("test.") - path = pathlib.Path(findtestdir(runtests.test_dir), - *name.split(".")) - if path.with_suffix(".py").exists(): - filename = str(path.with_suffix(".py")) - else: - filename = str(path / "__init__.py") - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print_github_annotation(self.test_name, message, filename) - def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3475f645729182..aa7dea414d9aed 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -3,6 +3,7 @@ from _colorize import get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element +from .reporter import get_reporter from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( @@ -142,6 +143,7 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: xml_data = result.xml_data if xml_data: self.add_junit(xml_data) + get_reporter(runtests).test_result(result, runtests) def get_coverage_results(self) -> trace.CoverageResults: counts = {loc: 1 for loc in self.covered_lines} diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 3327269ae1c622..c6db78e0882ba3 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,7 +624,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 541a8146001cab..5f638f10a5c5f1 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,7 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - github_annotations: bool + # GitHub Actions job summary file, to report the failures of this run + github_summary: str | None coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 8216541e454f82..ca97943cc2d905 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,6 +11,7 @@ from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests +from .reporter import get_reporter from .runtests import RunTests lazy from .testresult import RegressionTestResult from .utils import ( @@ -113,7 +114,7 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations + RegressionTestResult.reporter = get_reporter(runtests) if runtests.use_junit: support.junit_xml_list = [] RegressionTestResult.USE_XML = True diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 79add3d39b373a..d28ea88996a379 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,11 +9,12 @@ import traceback import unittest from test import support -from test.libregrtest.utils import print_github_annotation, sanitize_xml +from test.libregrtest.reporter import Reporter +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False + reporter = Reporter() def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -134,10 +135,12 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: super().printErrorList(flavour, [(test, err)]) - # Annotate just after the failure report, so that the annotation - # links to the report in the job log - if self.GITHUB_ANNOTATIONS: - print_github_annotation(str(test), err, stream=self.stream) + # Report just after unittest printed the failure, so that a CI + # annotation links to it in the job log + self.report_failure(test, err) + + def report_failure(self, test, err): + self.reporter.test_case_failed(str(test), err, self.stream) def get_xml_element(self): if not self.USE_XML: @@ -155,11 +158,9 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) - # Failures are not reported in quiet mode: annotate them at the end - if self.result.GITHUB_ANNOTATIONS: - for test, err in self.result.errors + self.result.failures: - print_github_annotation(str(test), err, - stream=self.result.stream) + # Failures are not printed in quiet mode: report them at the end + for test, err in self.result.errors + self.result.failures: + self.result.report_failure(test, err) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 6034fa50b00817..83e0575619c99a 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -3,7 +3,6 @@ import locale import math import os.path -import pathlib import platform import random import re @@ -14,7 +13,6 @@ import tempfile import textwrap import types -from _colorize import decolor # type: ignore[import-not-found] from collections.abc import Callable _winapi: types.ModuleType | None try: @@ -142,44 +140,6 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -# Escape the data and the properties of GitHub Actions workflow commands -GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) -GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", - ",": "%2C"}) - - -def print_github_annotation(title: str, message: str, - filename: str | None = None, - stream=None) -> None: - """Print a GitHub Actions error annotation for the failure.""" - message = decolor(message) - - # Try to get a file + line number from backtrace. - line = None - if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, - re.MULTILINE): - test_frames = [frame for frame in frames - if pathlib.Path(frame[0]).name.startswith('test_')] - filename, line = test_frames[-1] if test_frames else frames[0] - if not filename: - raise ValueError(f"missing filename to annotate {title!r}") - - # Normalize stdlib path to be relative inside the repository checkout. - path = pathlib.Path(filename) - stdlib_dir = pathlib.Path(os.__file__).parent - if path.is_relative_to(stdlib_dir): - filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() - props = {"file": filename, "line": line, "title": title} - # Only keep the exception: strip the traceback frames, each one a - # ' File' line followed by its indented source lines - message = re.split(r'^ File .*\n(?: .*\n)*', message, - flags=re.MULTILINE)[-1].strip() - props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" - for key, value in props.items() if value) - print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", - file=stream, flush=True) - - def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From dc83a3c18eaa8ef258f1f8093664f19a26ac4a30 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 13 Oct 2026 12:13:20 -0700 Subject: [PATCH 36/37] Personal review fixes --- Lib/test/libregrtest/main.py | 21 ++- Lib/test/libregrtest/reporter.py | 99 +++++++------- Lib/test/libregrtest/results.py | 2 - Lib/test/libregrtest/run_workers.py | 4 + Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/testresult.py | 4 +- Lib/test/test_regrtest.py | 204 ++++++++++++++++++++++------ 7 files changed, 228 insertions(+), 108 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3f84af27a4f126..00d9eb90fb6907 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -299,7 +299,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: self.run_tests_sequentially(runtests) return runtests - def rerun_failed_tests(self, runtests: RunTests) -> RunTests: + def rerun_failed_tests(self, runtests: RunTests) -> None: ansi = get_colors() red, reset = ansi.BOLD_RED, ansi.RESET @@ -309,7 +309,7 @@ def rerun_failed_tests(self, runtests: RunTests) -> RunTests: "Re-running failed tests is not supported with --python " "host runner option." ) - return runtests + return self.first_state = self.get_state() @@ -324,7 +324,6 @@ def rerun_failed_tests(self, runtests: RunTests) -> RunTests: printlist(self.results.bad) self.display_result(rerun_runtests) - return rerun_runtests def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: print() @@ -398,6 +397,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + get_reporter(runtests.github_summary).test_result(result, runtests) return result @@ -464,8 +464,7 @@ def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers RunWorkers(num_workers, runtests, self.logger, self.results).run() - def finalize_tests(self, coverage: trace.CoverageResults | None, - runtests: RunTests) -> None: + def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -488,7 +487,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None, failed = [result for result in self.results.rerun_results if result.is_failed(self.fail_env_changed)] if failed: - get_reporter(runtests).run_failed(failed, self.get_state()) + get_reporter(self.github_summary).run_failed(failed, + self.get_state()) def display_summary(self) -> None: if self.first_runtests is None: @@ -527,8 +527,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only report the failures which fail the job: when failed tests - # are re-run, only report the re-run. + # When failed tests are re-run, only report the re-run github_summary=(None if self.want_rerun and not self.python_cmd else self.github_summary), coverage=self.coverage, @@ -601,10 +600,8 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: coverage = self.results.get_coverage_results() self.display_result(runtests) - # The re-run, if any, is the last run - last_runtests = runtests if self.want_rerun and self.results.need_rerun(): - last_runtests = self.rerun_failed_tests(runtests) + self.rerun_failed_tests(runtests) if self.want_bisect and self.results.need_rerun(): self.run_bisect(runtests) @@ -613,7 +610,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.logger.stop_load_tracker() self.display_summary() - self.finalize_tests(coverage, last_runtests) + self.finalize_tests(coverage) return self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py index 5a82ad72a1c7e3..a5175be1f433cb 100644 --- a/Lib/test/libregrtest/reporter.py +++ b/Lib/test/libregrtest/reporter.py @@ -8,7 +8,7 @@ from .findtests import findtestdir from .result import TestResult from .runtests import RunTests -from .utils import count +from .utils import StrPath, count class Reporter: @@ -22,27 +22,58 @@ def test_case_failed(self, test_case: str, traceback: str, """Report a failed test case. Called just after unittest printed the failure, or at the end of the - test file in quiet mode (failures are not printed). + test file in quiet mode. """ def test_result(self, result: TestResult, runtests: RunTests) -> None: """Report the result of a test file, in the main process.""" def run_failed(self, failed: list[TestResult], state: str) -> None: - """Report a failed run: failed is the list of failed test files.""" + """Report the test files which failed the run.""" + @staticmethod + def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: + """Parse the traceback of a failure into (filename, line, exception). -def get_reporter(runtests: RunTests) -> Reporter: - if runtests.github_summary: - return GitHubActionsReporter(runtests.github_summary) + The location is the last frame in a test file, or else the first frame + (the test). + """ + frames = re.findall(r'^ *File "(.*)", line (\d+)', traceback, + re.MULTILINE) + test_frames = [frame for frame in frames + if pathlib.Path(frame[0]).name.startswith('test_')] + filename, line = (test_frames[-1] if test_frames + else frames[0] if frames + else (None, None)) + exception = re.split(r'^ File .*\n(?: .*\n)*', traceback, + flags=re.MULTILINE)[-1].strip() + return filename, line, exception + + @staticmethod + def test_filename(result: TestResult, + test_dir: StrPath | None) -> str | None: + """Get the filename of a test, or None if it is not found.""" + if result.filename: + return result.filename + # The worker crashed or timed out before importing the test: + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py + name = result.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(test_dir), *name.split(".")) + for filename in (path.with_suffix(".py"), path / "__init__.py"): + if filename.exists(): + return str(filename) + return None + + +def get_reporter(github_summary: str | None) -> Reporter: + if github_summary: + return GitHubActionsReporter(github_summary) return Reporter() class GitHubActionsReporter(Reporter): - """Report test failures to GitHub Actions. - - Annotate failures in the job log and list them in the job summary. - """ + """Annotate failures in the GitHub Actions job log and list them in the + job summary.""" # Escape the data and the properties of workflow commands ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) @@ -53,54 +84,29 @@ def __init__(self, summary_filename: str) -> None: def test_case_failed(self, test_case: str, traceback: str, stream: TextIO) -> None: - traceback = decolor(traceback) - # Get a file + line number from the backtrace: the last frame in a - # test file, or else the first frame (the test). Without frames, the - # failure is still annotated, just not located. - filename = line = None - if frames := re.findall(r'^ *File "(.*)", line (\d+)', traceback, - re.MULTILINE): - test_frames = [frame for frame in frames - if pathlib.Path(frame[0]).name.startswith('test_')] - filename, line = test_frames[-1] if test_frames else frames[0] - # Only keep the exception: strip the traceback frames, each one a - # ' File' line followed by its indented source lines - message = re.split(r'^ File .*\n(?: .*\n)*', traceback, - flags=re.MULTILINE)[-1].strip() - self.annotate(test_case, message, filename, line, stream) + # The job log has the full traceback: only keep the exception + filename, line, exception = self.parse_traceback(decolor(traceback)) + self.annotate(test_case, exception, filename, line, stream) def test_result(self, result: TestResult, runtests: RunTests) -> None: - # Annotate a test file which failed without failed test cases, for - # example: crash, timeout, env changed. Failed test cases are - # annotated by test_case_failed(). + # Failed test cases are annotated by test_case_failed() if (not result.is_failed(runtests.fail_env_changed) or result.failed_test_cases): return - filename = result.filename - if not filename: - # The worker crashed or timed out before reporting the test file: - # convert python import test name to relative file path. - # Ex: "test.test_x.test_y" is test_x/test_y.py - name = result.test_name.removeprefix("test.") - path = pathlib.Path(findtestdir(runtests.test_dir), - *name.split(".")) - if path.with_suffix(".py").exists(): - filename = str(path.with_suffix(".py")) - elif (path / "__init__.py").exists(): - filename = str(path / "__init__.py") - # else not a test file (ex: "" bug): no location message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - self.annotate(result.test_name, decolor(message), filename) + self.annotate(result.test_name, decolor(message), + self.test_filename(result, runtests.test_dir)) def annotate(self, title: str, message: str, filename: str | None, line: str | None = None, stream: TextIO | None = None) -> None: if filename: - # Normalize stdlib path to be relative inside the repository - # checkout. + # The stdlib can be outside of the repository checkout path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent if path.is_relative_to(stdlib_dir): filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + # Without a line, the annotation is not shown on the file + line = line or "1" props = {"file": filename, "line": line, "title": title} props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" for key, value in props.items() if value) @@ -118,13 +124,12 @@ def run_failed(self, failed: list[TestResult], state: str) -> None: blocks.append("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result.failed_test_cases: - # Only default open short tracebacks of up to 5 test cases. + # Expand short tracebacks if there are only a few is_open = ncases <= 5 and traceback.count("\n") < 30 blocks += [f"" f"{html.escape(name)}", f"```pytb\n{decolor(traceback).rstrip()}\n```", ""] - # Separate Markdown blocks with an empty line + # Markdown blocks are separated by an empty line with open(self.summary_filename, "a", encoding="utf-8") as fp: fp.write("".join(f"{block}\n\n" for block in blocks)) - diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index aa7dea414d9aed..3475f645729182 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -3,7 +3,6 @@ from _colorize import get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element -from .reporter import get_reporter from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( @@ -143,7 +142,6 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: xml_data = result.xml_data if xml_data: self.add_junit(xml_data) - get_reporter(runtests).test_result(result, runtests) def get_coverage_results(self) -> trace.CoverageResults: counts = {loc: 1 for loc in self.covered_lines} diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..205b8fbc3b940c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -16,6 +16,7 @@ from test.support import os_helper, MS_WINDOWS from .logger import Logger +from .reporter import get_reporter from .result import TestResult, State from .results import TestResults from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType @@ -624,6 +625,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Report after the output: env changed warnings, crash traceback + get_reporter(self.runtests.github_summary).test_result(result, + self.runtests) return result diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index ca97943cc2d905..5e9cded3b25a4e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -114,7 +114,7 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - RegressionTestResult.reporter = get_reporter(runtests) + RegressionTestResult.reporter = get_reporter(runtests.github_summary) if runtests.use_junit: support.junit_xml_list = [] RegressionTestResult.USE_XML = True diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index d28ea88996a379..3e4243282236cf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -135,8 +135,6 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: super().printErrorList(flavour, [(test, err)]) - # Report just after unittest printed the failure, so that a CI - # annotation links to it in the job log self.report_failure(test, err) def report_failure(self, test, err): @@ -158,7 +156,7 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) - # Failures are not printed in quiet mode: report them at the end + # Failures are not printed in quiet mode for test, err in self.result.errors + self.result.failures: self.result.report_failure(test, err) return self.result diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f445eb2b6a0760..7f08fb38195626 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,6 +22,7 @@ import tempfile import threading import textwrap +import types import unittest import unittest.mock from xml.etree import ElementTree @@ -37,7 +38,8 @@ from test.libregrtest import setup from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test -from test.libregrtest.result import TestStats +from test.libregrtest.reporter import GitHubActionsReporter, Reporter +from test.libregrtest.result import State, TestResult, TestStats from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -1639,10 +1641,9 @@ def test_env_changed(self): success=True), stats=2) - def test_github_actions(self): - # Failures are annotated in the GitHub Actions job log at the failing - # line of the test file, and listed in the job summary - cases = self.create_test('github_cases', textwrap.dedent(""" + def test_reporter(self): + # Failures which fail the run are reported, with their location + cases = self.create_test('reporter_cases', textwrap.dedent(""" import doctest, sys, unittest def load_tests(loader, tests, pattern): @@ -1660,59 +1661,51 @@ def test_subtest(self): with self.subTest(x=1): self.fail() """)) - crash = self.create_test('github_crash', textwrap.dedent(""" + crash = self.create_test('reporter_crash', textwrap.dedent(""" import os, unittest class Tests(unittest.TestCase): def test_crash(self): os._exit(1) """)) + cases_file = os.path.join(self.tmptestdir, f'{cases}.py') + crash_file = os.path.join(self.tmptestdir, f'{crash}.py') + reported_cases = [ + (f'test_subtest ({cases}.Tests.test_subtest) (x=1)', + cases_file, '17'), + (f'doctest_fail ({cases}) [0]', cases_file, '10'), + ] + reported_crash = (crash, crash_file, '1') filename = os.path.join(self.tmptestdir, 'github_step_summary.md') env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) env.pop('SOURCE_DATE_EPOCH', None) - # A single worker without --randomize runs the tests in order + + def reported(output): + annotations = re.findall(r'^::error (.*?)::', output, re.MULTILINE) + props = [dict(prop.split('=', 1) for prop in text.split(',')) + for text in annotations] + return [(prop['title'], prop.get('file'), prop.get('line')) + for prop in props] + + # Only the failures of the re-run are reported. A single worker + # without --randomize runs the tests in order. output = self.run_tests('-j1', '--rerun', cases, crash, env=env, exitcode=EXITCODE_BAD_TEST) - - def unescape(text): - for char in '\r\n:,%': - text = text.replace(f'%{ord(char):02X}', char) - return text - - annotations = [] - for props, message in re.findall(r'^::error (.*?)::(.*)$', output, - re.MULTILINE): - props = dict(prop.split('=', 1) for prop in props.split(',')) - annotations.append(({key: unescape(value) - for key, value in props.items()}, - unescape(message))) - # One annotation per failure of the re-run, not of the first run - cases_file = os.path.join(self.tmptestdir, f'{cases}.py') - crash_file = os.path.join(self.tmptestdir, f'{crash}.py') - self.assertEqual(annotations, [ - ({'file': cases_file, 'line': '17', - 'title': f'test_subtest ({cases}.Tests.test_subtest) (x=1)'}, - 'AssertionError: None'), - ({'file': cases_file, 'line': '10', - 'title': f'doctest_fail ({cases}) [0]'}, - 'AssertionError: Failed example:\n' - ' 1 + 1\n' - 'Expected:\n' - ' 3\n' - 'Got:\n' - ' 2'), - ({'file': crash_file, 'title': crash}, - f'{crash} worker non-zero exit code'), - ], output) + self.assertEqual(reported(output), [*reported_cases, reported_crash], + output) with open(filename, encoding='utf-8') as fp: - summary = fp.read() - self.assertIn('## FAILURE: 2 test files and 2 test cases failed', - summary) + self.assertIn('2 test files and 2 test cases failed', fp.read()) + os_helper.unlink(filename) - # No annotation and no job summary when all tests pass + # Quiet mode doesn't print failed test cases, but reports them + output = self.run_tests('-j1', cases, env=env, + exitcode=EXITCODE_BAD_TEST) + self.assertEqual(reported(output), reported_cases, output) os_helper.unlink(filename) + + # Nothing is reported when all tests pass output = self.run_tests(self.create_test(), env=env) - self.assertNotIn('::error', output) + self.assertEqual(reported(output), []) self.assertFalse(os.path.exists(filename)) def test_rerun_fail(self): @@ -3022,6 +3015,131 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') +class ReporterTestCase(unittest.TestCase): + def test_parse_traceback(self): + parse = Reporter.parse_traceback + self.assertEqual(parse('AssertionError\n'), + (None, None, 'AssertionError')) + traceback = textwrap.dedent(""" + Traceback (most recent call last): + File "/lib/test_x.py", line 5, in test_x + helper() + File "/lib/test_x.py", line 9, in helper + check() + File "/lib/check.py", line 2, in check + raise ValueError + ValueError + """) + # The last frame in a test file + self.assertEqual(parse(traceback), + ('/lib/test_x.py', '9', 'ValueError')) + # Without a test file, the first frame + traceback = traceback.replace('test_x.py', 'x.py') + self.assertEqual(parse(traceback), ('/lib/x.py', '5', 'ValueError')) + + def test_test_filename(self): + test_dir = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(os_helper.rmtree, test_dir) + os.mkdir(os.path.join(test_dir, 'test_pkg')) + for name in ('test_x.py', os.path.join('test_pkg', '__init__.py')): + open(os.path.join(test_dir, name), 'w').close() + + def filename(test_name, filename=None): + result = TestResult(test_name, filename=filename) + return Reporter.test_filename(result, test_dir) + + self.assertEqual(filename('test_x', '/imported/test_x.py'), + '/imported/test_x.py') + self.assertEqual(filename('test_x'), + os.path.join(test_dir, 'test_x.py')) + self.assertEqual(filename('test_pkg'), + os.path.join(test_dir, 'test_pkg', '__init__.py')) + self.assertIsNone(filename('')) + + +class GitHubActionsReporterTestCase(unittest.TestCase): + def setUp(self): + self.summary = os.path.join(tempfile.mkdtemp(), 'summary.md') + self.addCleanup(os_helper.rmtree, os.path.dirname(self.summary)) + self.reporter = GitHubActionsReporter(self.summary) + + def annotate_test_case(self, traceback, test_case='test_x (test_x.T)'): + stream = io.StringIO() + self.reporter.test_case_failed(test_case, traceback, stream) + return stream.getvalue() + + def test_test_case_failed(self): + stdlib_file = os.path.join(os.path.dirname(os.__file__), 'test', + 'test_x.py') + traceback = textwrap.dedent(f"""\ + Traceback (most recent call last): + File "{stdlib_file}", line 3, in test_x + self.fail('50%') + AssertionError: 50% + more + """) + # Only the exception is kept, the stdlib path is relative to the + # checkout, and the data and properties are escaped + self.assertEqual( + self.annotate_test_case(traceback, 'test_x (a: b, c)'), + '::error file=Lib/test/test_x.py,line=3,' + 'title=test_x (a%3A b%2C c)::AssertionError: 50%25%0Amore\n') + # Without traceback, the failure is not located + self.assertEqual(self.annotate_test_case('AssertionError\n'), + '::error title=test_x (test_x.T)::AssertionError\n') + + def test_test_result(self): + result = TestResult('test_x', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread'], + filename='/tests/test_x.py') + runtests = types.SimpleNamespace(fail_env_changed=True, test_dir=None) + with support.captured_stdout() as stdout: + self.reporter.test_result(result, runtests) + self.assertEqual(stdout.getvalue(), + '::error file=/tests/test_x.py,line=1,title=test_x::' + 'test_x failed (env changed)%0Adangling thread\n') + + # Not reported: the test didn't fail the run, or its failed test + # cases are reported by test_case_failed() + runtests.fail_env_changed = False + failed_cases = TestResult('test_x', state=State.FAILED, + failures=[('test_a', 'AssertionError\n')]) + for result in (result, failed_cases, TestResult('test_x', State.PASSED)): + with support.captured_stdout() as stdout: + self.reporter.test_result(result, runtests) + self.assertEqual(stdout.getvalue(), '') + + def test_run_failed(self): + failed = [ + TestResult('test_x', state=State.FAILED, + failures=[('test_a ()', 'Traceback\nAssertionError\n')]), + TestResult('test_y', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread']), + ] + self.reporter.run_failed(failed, 'FAILURE') + with open(self.summary, encoding='utf-8') as fp: + summary = fp.read() + self.assertEqual(summary, textwrap.dedent("""\ + ## FAILURE: 2 test files and 1 test case failed + + ### test_x failed (1 failure) + +
test_a (<T>) + + ```pytb + Traceback + AssertionError + ``` + +
+ + ### test_y failed (env changed) + + - dangling thread + + """)) + + from test.libregrtest.results import TestResults From 589e6a7f691e63b55fbbb6edd603dedee270d3d2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 13 Oct 2026 19:36:52 -0700 Subject: [PATCH 37/37] Simplify, generlize reporter --- Lib/test/libregrtest/main.py | 23 +++--- Lib/test/libregrtest/reporter.py | 69 ++++++++++-------- Lib/test/libregrtest/result.py | 17 ++++- Lib/test/libregrtest/run_workers.py | 9 +-- Lib/test/libregrtest/runtests.py | 4 +- Lib/test/libregrtest/setup.py | 4 +- Lib/test/test_regrtest.py | 108 ++++++++++++++++++---------- 7 files changed, 142 insertions(+), 92 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 00d9eb90fb6907..5d403ed8a16eb9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -69,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Logger self.logger = Logger(self.results, self.quiet, self.pgo) + self.reporter = get_reporter() # Actions self.want_header: bool = ns.header @@ -286,8 +287,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False, - github_summary=self.github_summary) + final_run=True) self.logger.set_tests(runtests) + self.reporter.run_started(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" if not self.single_process: @@ -397,7 +399,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - get_reporter(runtests.github_summary).test_result(result, runtests) + self.reporter.test_result(result) return result @@ -462,7 +464,8 @@ def get_state(self) -> str: def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers - RunWorkers(num_workers, runtests, self.logger, self.results).run() + RunWorkers(num_workers, runtests, self.logger, self.results, + self.reporter).run() def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: @@ -483,12 +486,7 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) - # Test files which failed in the last run (the re-run, if any) - failed = [result for result in self.results.rerun_results - if result.is_failed(self.fail_env_changed)] - if failed: - get_reporter(self.github_summary).run_failed(failed, - self.get_state()) + self.reporter.summary(self.results, self.get_state()) def display_summary(self) -> None: if self.first_runtests is None: @@ -527,9 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # When failed tests are re-run, only report the re-run - github_summary=(None if self.want_rerun and not self.python_cmd - else self.github_summary), + final_run=not (self.want_rerun and not self.python_cmd), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -581,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: runtests = runtests.copy(tests=case_ids, case_groups=case_groups) self.first_runtests = runtests self.logger.set_tests(runtests) + self.reporter.run_started(runtests) if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check @@ -777,8 +774,6 @@ def _init(self): strip_py_suffix(self.cmdline_args) - self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) - self._tmp_dir = get_temp_dir(self._tmp_dir) @property diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py index a5175be1f433cb..f7eb71e049a489 100644 --- a/Lib/test/libregrtest/reporter.py +++ b/Lib/test/libregrtest/reporter.py @@ -5,10 +5,10 @@ from _colorize import decolor # type: ignore[import-not-found] from typing import TextIO -from .findtests import findtestdir from .result import TestResult +lazy from .results import TestResults from .runtests import RunTests -from .utils import StrPath, count +from .utils import count class Reporter: @@ -17,6 +17,14 @@ class Reporter: The default reporter does nothing. """ + def run_started(self, runtests: RunTests) -> None: + """Start a run of tests. + + Called in the main process at the start of each run (the first run + and the re-run), and in the process running the tests before each + test file. + """ + def test_case_failed(self, test_case: str, traceback: str, stream: TextIO) -> None: """Report a failed test case. @@ -25,11 +33,11 @@ def test_case_failed(self, test_case: str, traceback: str, test file in quiet mode. """ - def test_result(self, result: TestResult, runtests: RunTests) -> None: + def test_result(self, result: TestResult) -> None: """Report the result of a test file, in the main process.""" - def run_failed(self, failed: list[TestResult], state: str) -> None: - """Report the test files which failed the run.""" + def summary(self, results: TestResults, state: str) -> None: + """Report the results of all runs, in the main process.""" @staticmethod def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: @@ -49,25 +57,10 @@ def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: flags=re.MULTILINE)[-1].strip() return filename, line, exception - @staticmethod - def test_filename(result: TestResult, - test_dir: StrPath | None) -> str | None: - """Get the filename of a test, or None if it is not found.""" - if result.filename: - return result.filename - # The worker crashed or timed out before importing the test: - # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py - name = result.test_name.removeprefix("test.") - path = pathlib.Path(findtestdir(test_dir), *name.split(".")) - for filename in (path.with_suffix(".py"), path / "__init__.py"): - if filename.exists(): - return str(filename) - return None - - -def get_reporter(github_summary: str | None) -> Reporter: - if github_summary: - return GitHubActionsReporter(github_summary) + +def get_reporter() -> Reporter: + if GitHubActionsReporter.enabled(): + return GitHubActionsReporter() return Reporter() @@ -79,23 +72,31 @@ class GitHubActionsReporter(Reporter): ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) - def __init__(self, summary_filename: str) -> None: - self.summary_filename = summary_filename + @staticmethod + def enabled() -> bool: + return "GITHUB_STEP_SUMMARY" in os.environ + + def run_started(self, runtests: RunTests) -> None: + self.runtests = runtests def test_case_failed(self, test_case: str, traceback: str, stream: TextIO) -> None: + # Only report the failures which decide the result + if not self.runtests.final_run: + return # The job log has the full traceback: only keep the exception filename, line, exception = self.parse_traceback(decolor(traceback)) self.annotate(test_case, exception, filename, line, stream) - def test_result(self, result: TestResult, runtests: RunTests) -> None: + def test_result(self, result: TestResult) -> None: # Failed test cases are annotated by test_case_failed() - if (not result.is_failed(runtests.fail_env_changed) + if (not self.runtests.final_run + or not result.is_failed(self.runtests.fail_env_changed) or result.failed_test_cases): return message = "\n".join([str(result), *(result.env_changed_reasons or ())]) self.annotate(result.test_name, decolor(message), - self.test_filename(result, runtests.test_dir)) + result.find_filename(self.runtests.test_dir)) def annotate(self, title: str, message: str, filename: str | None, line: str | None = None, stream: TextIO | None = None) -> None: @@ -113,7 +114,12 @@ def annotate(self, title: str, message: str, filename: str | None, print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", file=stream, flush=True) - def run_failed(self, failed: list[TestResult], state: str) -> None: + def summary(self, results: TestResults, state: str) -> None: + # Test files which failed in the last run (the re-run, if any) + failed = [result for result in results.rerun_results + if result.is_failed(self.runtests.fail_env_changed)] + if not failed: + return ncases = sum(len(result.failed_test_cases) for result in failed) blocks = [f"## {decolor(state)}: " f"{count(len(failed), 'test file')} and " @@ -131,5 +137,6 @@ def run_failed(self, failed: list[TestResult], state: str) -> None: f"```pytb\n{decolor(traceback).rstrip()}\n```", ""] # Markdown blocks are separated by an empty line - with open(self.summary_filename, "a", encoding="utf-8") as fp: + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", + encoding="utf-8") as fp: fp.write("".join(f"{block}\n\n" for block in blocks)) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 00e6e5a9767c03..328cbc7f6128ff 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,10 +1,12 @@ import dataclasses import json +import pathlib from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir from .utils import ( - StrJSON, TestName, FilterTuple, + StrJSON, StrPath, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) @@ -185,6 +187,19 @@ def has_meaningful_duration(self): def failed_test_cases(self) -> list[tuple[str, str]]: return (self.errors or []) + (self.failures or []) + def find_filename(self, test_dir: StrPath | None) -> str | None: + """Get the filename of the test, or None if it is not found.""" + if self.filename: + return self.filename + # The worker crashed or timed out before importing the test: + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(test_dir), *name.split(".")) + for filename in (path.with_suffix(".py"), path / "__init__.py"): + if filename.exists(): + return str(filename) + return None + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 205b8fbc3b940c..ad02d8dba7c23e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -16,7 +16,7 @@ from test.support import os_helper, MS_WINDOWS from .logger import Logger -from .reporter import get_reporter +from .reporter import Reporter from .result import TestResult, State from .results import TestResults from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType @@ -500,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None: class RunWorkers: def __init__(self, num_workers: int, runtests: RunTests, - logger: Logger, results: TestResults) -> None: + logger: Logger, results: TestResults, + reporter: Reporter) -> None: self.num_workers = num_workers self.runtests = runtests self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.reporter = reporter self.live_worker_count = 0 self.output: queue.Queue[QueueContent] = queue.Queue() @@ -626,8 +628,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: if stdout: print(stdout, flush=True) # Report after the output: env changed warnings, crash traceback - get_reporter(self.runtests.github_summary).test_result(result, - self.runtests) + self.reporter.test_result(result) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 5f638f10a5c5f1..525f9573ad992d 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,8 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # GitHub Actions job summary file, to report the failures of this run - github_summary: str | None + # Failed tests are not re-run after this run + final_run: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 5e9cded3b25a4e..36b3a10ebf2f7e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -114,7 +114,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - RegressionTestResult.reporter = get_reporter(runtests.github_summary) + reporter = get_reporter() + reporter.run_started(runtests) + RegressionTestResult.reporter = reporter if runtests.use_junit: support.junit_xml_list = [] RegressionTestResult.USE_XML = True diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7f08fb38195626..e5852b9aa399e0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -40,6 +40,7 @@ from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.reporter import GitHubActionsReporter, Reporter from test.libregrtest.result import State, TestResult, TestStats +from test.libregrtest.results import TestResults from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -630,6 +631,10 @@ def setUp(self): self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) + # Don't report the failures of the tests to the CI running this test + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') + def create_test(self, name=None, code=None): if not name: name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID @@ -3037,16 +3042,15 @@ def test_parse_traceback(self): traceback = traceback.replace('test_x.py', 'x.py') self.assertEqual(parse(traceback), ('/lib/x.py', '5', 'ValueError')) - def test_test_filename(self): - test_dir = os.path.realpath(tempfile.mkdtemp()) - self.addCleanup(os_helper.rmtree, test_dir) + def test_find_filename(self): + test_dir = self.enterContext(os_helper.temp_dir()) os.mkdir(os.path.join(test_dir, 'test_pkg')) for name in ('test_x.py', os.path.join('test_pkg', '__init__.py')): open(os.path.join(test_dir, name), 'w').close() def filename(test_name, filename=None): result = TestResult(test_name, filename=filename) - return Reporter.test_filename(result, test_dir) + return result.find_filename(test_dir) self.assertEqual(filename('test_x', '/imported/test_x.py'), '/imported/test_x.py') @@ -3058,16 +3062,34 @@ def filename(test_name, filename=None): class GitHubActionsReporterTestCase(unittest.TestCase): - def setUp(self): - self.summary = os.path.join(tempfile.mkdtemp(), 'summary.md') - self.addCleanup(os_helper.rmtree, os.path.dirname(self.summary)) - self.reporter = GitHubActionsReporter(self.summary) + ENV_CHANGED = TestResult('test_x', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread'], + filename='/tests/test_x.py') - def annotate_test_case(self, traceback, test_case='test_x (test_x.T)'): + def setUp(self): + self.summary = os_helper.TESTFN + self.addCleanup(os_helper.unlink, self.summary) + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['GITHUB_STEP_SUMMARY'] = self.summary + + def create_reporter(self, *, final_run=True, fail_env_changed=True): + reporter = GitHubActionsReporter() + reporter.run_started(types.SimpleNamespace( + final_run=final_run, fail_env_changed=fail_env_changed, + test_dir=None)) + return reporter + + def annotate_test_case(self, test_case, traceback, **kwargs): stream = io.StringIO() - self.reporter.test_case_failed(test_case, traceback, stream) + self.create_reporter(**kwargs).test_case_failed(test_case, traceback, + stream) return stream.getvalue() + def annotate_result(self, result, **kwargs): + with support.captured_stdout() as stdout: + self.create_reporter(**kwargs).test_result(result) + return stdout.getvalue() + def test_test_case_failed(self): stdlib_file = os.path.join(os.path.dirname(os.__file__), 'test', 'test_x.py') @@ -3081,42 +3103,49 @@ def test_test_case_failed(self): # Only the exception is kept, the stdlib path is relative to the # checkout, and the data and properties are escaped self.assertEqual( - self.annotate_test_case(traceback, 'test_x (a: b, c)'), + self.annotate_test_case('test_x (a: b, c)', traceback), '::error file=Lib/test/test_x.py,line=3,' 'title=test_x (a%3A b%2C c)::AssertionError: 50%25%0Amore\n') - # Without traceback, the failure is not located - self.assertEqual(self.annotate_test_case('AssertionError\n'), - '::error title=test_x (test_x.T)::AssertionError\n') + + def test_test_case_failed_without_traceback(self): + self.assertEqual(self.annotate_test_case('test_x', 'AssertionError\n'), + '::error title=test_x::AssertionError\n') + + def test_test_case_failed_not_final_run(self): + # Failed tests will be re-run: only the re-run is reported + self.assertEqual(self.annotate_test_case('test_x', 'AssertionError\n', + final_run=False), + '') def test_test_result(self): - result = TestResult('test_x', state=State.ENV_CHANGED, - env_changed_reasons=['dangling thread'], - filename='/tests/test_x.py') - runtests = types.SimpleNamespace(fail_env_changed=True, test_dir=None) - with support.captured_stdout() as stdout: - self.reporter.test_result(result, runtests) - self.assertEqual(stdout.getvalue(), + self.assertEqual(self.annotate_result(self.ENV_CHANGED), '::error file=/tests/test_x.py,line=1,title=test_x::' 'test_x failed (env changed)%0Adangling thread\n') - # Not reported: the test didn't fail the run, or its failed test - # cases are reported by test_case_failed() - runtests.fail_env_changed = False - failed_cases = TestResult('test_x', state=State.FAILED, - failures=[('test_a', 'AssertionError\n')]) - for result in (result, failed_cases, TestResult('test_x', State.PASSED)): - with support.captured_stdout() as stdout: - self.reporter.test_result(result, runtests) - self.assertEqual(stdout.getvalue(), '') - - def test_run_failed(self): - failed = [ + def test_test_result_not_reported(self): + failed_test_cases = TestResult( + 'test_x', state=State.FAILED, + failures=[('test_a', 'AssertionError\n')]) + cases = { + 'not final run': (self.ENV_CHANGED, {'final_run': False}), + 'not failing the run': (self.ENV_CHANGED, + {'fail_env_changed': False}), + 'passed': (TestResult('test_x', state=State.PASSED), {}), + # Annotated by test_case_failed() + 'failed test cases': (failed_test_cases, {}), + } + for name, (result, kwargs) in cases.items(): + with self.subTest(name): + self.assertEqual(self.annotate_result(result, **kwargs), '') + + def test_summary(self): + results = TestResults() + results.rerun_results = [ TestResult('test_x', state=State.FAILED, failures=[('test_a ()', 'Traceback\nAssertionError\n')]), - TestResult('test_y', state=State.ENV_CHANGED, - env_changed_reasons=['dangling thread']), + self.ENV_CHANGED, ] - self.reporter.run_failed(failed, 'FAILURE') + self.create_reporter().summary(results, 'FAILURE') with open(self.summary, encoding='utf-8') as fp: summary = fp.read() self.assertEqual(summary, textwrap.dedent("""\ @@ -3133,14 +3162,15 @@ def test_run_failed(self): - ### test_y failed (env changed) + ### test_x failed (env changed) - dangling thread """)) - -from test.libregrtest.results import TestResults + def test_summary_nothing_failed(self): + self.create_reporter().summary(TestResults(), 'SUCCESS') + self.assertFalse(os.path.exists(self.summary)) class TestColorized(unittest.TestCase):