diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d32..5d403ed8a16eb9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -15,6 +15,7 @@ from .findtests import findtests, split_test_packages, list_cases, collect_cases from .logger import Logger from .pgo import setup_pgo_tests +from .reporter import get_reporter from .result import TestResult from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak @@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Logger self.logger = Logger(self.results, self.quiet, self.pgo) + self.reporter = get_reporter() # Actions self.want_header: bool = ns.header @@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + final_run=True) self.logger.set_tests(runtests) + self.reporter.run_started(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" if not self.single_process: @@ -395,6 +399,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + self.reporter.test_result(result) return result @@ -459,7 +464,8 @@ def get_state(self) -> str: def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: from .run_workers import RunWorkers - RunWorkers(num_workers, runtests, self.logger, self.results).run() + RunWorkers(num_workers, runtests, self.logger, self.results, + self.reporter).run() def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: @@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.reporter.summary(self.results, self.get_state()) + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( @@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + final_run=not (self.want_rerun and not self.python_cmd), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: runtests = runtests.copy(tests=case_ids, case_groups=case_groups) self.first_runtests = runtests self.logger.set_tests(runtests) + self.reporter.run_started(runtests) if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check diff --git a/Lib/test/libregrtest/reporter.py b/Lib/test/libregrtest/reporter.py new file mode 100644 index 00000000000000..f7eb71e049a489 --- /dev/null +++ b/Lib/test/libregrtest/reporter.py @@ -0,0 +1,142 @@ +import html +import os +import pathlib +import re +from _colorize import decolor # type: ignore[import-not-found] +from typing import TextIO + +from .result import TestResult +lazy from .results import TestResults +from .runtests import RunTests +from .utils import count + + +class Reporter: + """Report test failures, for example to a CI service. + + The default reporter does nothing. + """ + + def run_started(self, runtests: RunTests) -> None: + """Start a run of tests. + + Called in the main process at the start of each run (the first run + and the re-run), and in the process running the tests before each + test file. + """ + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + """Report a failed test case. + + Called just after unittest printed the failure, or at the end of the + test file in quiet mode. + """ + + def test_result(self, result: TestResult) -> None: + """Report the result of a test file, in the main process.""" + + def summary(self, results: TestResults, state: str) -> None: + """Report the results of all runs, in the main process.""" + + @staticmethod + def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: + """Parse the traceback of a failure into (filename, line, exception). + + The location is the last frame in a test file, or else the first frame + (the test). + """ + frames = re.findall(r'^ *File "(.*)", line (\d+)', traceback, + re.MULTILINE) + test_frames = [frame for frame in frames + if pathlib.Path(frame[0]).name.startswith('test_')] + filename, line = (test_frames[-1] if test_frames + else frames[0] if frames + else (None, None)) + exception = re.split(r'^ File .*\n(?: .*\n)*', traceback, + flags=re.MULTILINE)[-1].strip() + return filename, line, exception + + +def get_reporter() -> Reporter: + if GitHubActionsReporter.enabled(): + return GitHubActionsReporter() + return Reporter() + + +class GitHubActionsReporter(Reporter): + """Annotate failures in the GitHub Actions job log and list them in the + job summary.""" + + # Escape the data and the properties of workflow commands + ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) + ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) + + @staticmethod + def enabled() -> bool: + return "GITHUB_STEP_SUMMARY" in os.environ + + def run_started(self, runtests: RunTests) -> None: + self.runtests = runtests + + def test_case_failed(self, test_case: str, traceback: str, + stream: TextIO) -> None: + # Only report the failures which decide the result + if not self.runtests.final_run: + return + # The job log has the full traceback: only keep the exception + filename, line, exception = self.parse_traceback(decolor(traceback)) + self.annotate(test_case, exception, filename, line, stream) + + def test_result(self, result: TestResult) -> None: + # Failed test cases are annotated by test_case_failed() + if (not self.runtests.final_run + or not result.is_failed(self.runtests.fail_env_changed) + or result.failed_test_cases): + return + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + self.annotate(result.test_name, decolor(message), + result.find_filename(self.runtests.test_dir)) + + def annotate(self, title: str, message: str, filename: str | None, + line: str | None = None, stream: TextIO | None = None) -> None: + if filename: + # The stdlib can be outside of the repository checkout + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + # Without a line, the annotation is not shown on the file + line = line or "1" + props = {"file": filename, "line": line, "title": title} + props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" + for key, value in props.items() if value) + print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", + file=stream, flush=True) + + def summary(self, results: TestResults, state: str) -> None: + # Test files which failed in the last run (the re-run, if any) + failed = [result for result in results.rerun_results + if result.is_failed(self.runtests.fail_env_changed)] + if not failed: + return + ncases = sum(len(result.failed_test_cases) for result in failed) + blocks = [f"## {decolor(state)}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed"] + for result in failed: + blocks.append(f"### {decolor(str(result))}") + if result.env_changed_reasons: + blocks.append("\n".join(f"- {reason}" + for reason in result.env_changed_reasons)) + for name, traceback in result.failed_test_cases: + # Expand short tracebacks if there are only a few + is_open = ncases <= 5 and traceback.count("\n") < 30 + blocks += [f"" + f"{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + ""] + # Markdown blocks are separated by an empty line + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", + encoding="utf-8") as fp: + fp.write("".join(f"{block}\n\n" for block in blocks)) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c..328cbc7f6128ff 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,10 +1,12 @@ import dataclasses import json +import pathlib from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir from .utils import ( - StrJSON, TestName, FilterTuple, + StrJSON, StrPath, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) @@ -103,6 +105,9 @@ class TestResult: # short descriptions of how the test altered the execution environment env_changed_reasons: list[str] | None = None + # test file, set once the test module is imported + filename: str | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -178,6 +183,23 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def failed_test_cases(self) -> list[tuple[str, str]]: + return (self.errors or []) + (self.failures or []) + + def find_filename(self, test_dir: StrPath | None) -> str | None: + """Get the filename of the test, or None if it is not found.""" + if self.filename: + return self.filename + # The worker crashed or timed out before importing the test: + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(test_dir), *name.split(".")) + for filename in (path.with_suffix(".py"), path / "__init__.py"): + if filename.exists(): + return str(filename) + return None + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba3..ad02d8dba7c23e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -16,6 +16,7 @@ from test.support import os_helper, MS_WINDOWS from .logger import Logger +from .reporter import Reporter from .result import TestResult, State from .results import TestResults from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType @@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None: class RunWorkers: def __init__(self, num_workers: int, runtests: RunTests, - logger: Logger, results: TestResults) -> None: + logger: Logger, results: TestResults, + reporter: Reporter) -> None: self.num_workers = num_workers self.runtests = runtests self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.reporter = reporter self.live_worker_count = 0 self.output: queue.Queue[QueueContent] = queue.Queue() @@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Report after the output: env changed warnings, crash traceback + self.reporter.test_result(result) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a60..525f9573ad992d 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Failed tests are not re-run after this run + final_run: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..36b3a10ebf2f7e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,9 @@ from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests +from .reporter import get_reporter from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + reporter = get_reporter() + reporter.run_started(runtests) + RegressionTestResult.reporter = reporter if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a6..3aa4ffd00f8f8f 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -153,6 +153,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) test_mod = importlib.import_module(module_name) + result.filename = test_mod.__file__ if hasattr(test_mod, "test_main"): # https://github.com/python/cpython/issues/89392 diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb..3e4243282236cf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,12 @@ import traceback import unittest from test import support +from test.libregrtest.reporter import Reporter from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + reporter = Reporter() def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + self.report_failure(test, err) + + def report_failure(self, test, err): + self.reporter.test_case_failed(str(test), err, self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not printed in quiet mode + for test, err in self.result.errors + self.result.failures: + self.result.report_failure(test, err) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index dcdf7bdce03b80..2214e7408aa560 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -4,6 +4,7 @@ from test.support import os_helper import os +import sys class BoolTest(unittest.TestCase): @@ -25,6 +26,11 @@ def test_repr(self): self.assertIs(eval(repr(True)), True) def test_str(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.assertEqual( + str(True), + 'Yes') self.assertEqual(str(False), 'False') self.assertEqual(str(True), 'True') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 7bff16ddf2155e..d01368254f6701 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import sys import unittest.mock from io import StringIO from collections import OrderedDict @@ -9,6 +10,9 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.loads('{"bad": }') self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a1250c668223e7..40dc0d9acc75ff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -513,6 +513,11 @@ def testFabs(self): def testFloor(self): self.assertRaises(TypeError, math.floor) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + for x in (0.5, 1.5): + with self.subTest(x=x): + self.assertEqual(math.floor(x), x) self.assertEqual(int, type(math.floor(0.5))) self.assertEqual(math.floor(0.5), 0) self.assertEqual(math.floor(1.0), 1) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 81b3043eb7e75b..04321264e6f121 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -4080,7 +4080,8 @@ def test_memfd_create(self): self.assertFalse(os.get_inheritable(fd)) with open(fd, "wb", closefd=False) as f: f.write(b'memfd_create') - self.assertEqual(f.tell(), 12) + # XXX: Intentional failure to test GitHub Actions job summaries + self.assertEqual(f.tell(), 13) fd2 = os.memfd_create("Hi") self.addCleanup(os.close, fd2) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e4..e5852b9aa399e0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,6 +22,7 @@ import tempfile import threading import textwrap +import types import unittest import unittest.mock from xml.etree import ElementTree @@ -37,7 +38,9 @@ from test.libregrtest import setup from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test -from test.libregrtest.result import TestStats +from test.libregrtest.reporter import GitHubActionsReporter, Reporter +from test.libregrtest.result import State, TestResult, TestStats +from test.libregrtest.results import TestResults from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -628,6 +631,10 @@ def setUp(self): self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) + # Don't report the failures of the tests to the CI running this test + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') + def create_test(self, name=None, code=None): if not name: name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID @@ -1639,6 +1646,73 @@ def test_env_changed(self): success=True), stats=2) + def test_reporter(self): + # Failures which fail the run are reported, with their location + cases = self.create_test('reporter_cases', textwrap.dedent(""" + import doctest, sys, unittest + + def load_tests(loader, tests, pattern): + tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) + return tests + + def doctest_fail(): + ''' + >>> 1 + 1 + 3 + ''' + + class Tests(unittest.TestCase): + def test_subtest(self): + with self.subTest(x=1): + self.fail() + """)) + crash = self.create_test('reporter_crash', textwrap.dedent(""" + import os, unittest + + class Tests(unittest.TestCase): + def test_crash(self): + os._exit(1) + """)) + cases_file = os.path.join(self.tmptestdir, f'{cases}.py') + crash_file = os.path.join(self.tmptestdir, f'{crash}.py') + reported_cases = [ + (f'test_subtest ({cases}.Tests.test_subtest) (x=1)', + cases_file, '17'), + (f'doctest_fail ({cases}) [0]', cases_file, '10'), + ] + reported_crash = (crash, crash_file, '1') + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) + env.pop('SOURCE_DATE_EPOCH', None) + + def reported(output): + annotations = re.findall(r'^::error (.*?)::', output, re.MULTILINE) + props = [dict(prop.split('=', 1) for prop in text.split(',')) + for text in annotations] + return [(prop['title'], prop.get('file'), prop.get('line')) + for prop in props] + + # Only the failures of the re-run are reported. A single worker + # without --randomize runs the tests in order. + output = self.run_tests('-j1', '--rerun', cases, crash, + env=env, exitcode=EXITCODE_BAD_TEST) + self.assertEqual(reported(output), [*reported_cases, reported_crash], + output) + with open(filename, encoding='utf-8') as fp: + self.assertIn('2 test files and 2 test cases failed', fp.read()) + os_helper.unlink(filename) + + # Quiet mode doesn't print failed test cases, but reports them + output = self.run_tests('-j1', cases, env=env, + exitcode=EXITCODE_BAD_TEST) + self.assertEqual(reported(output), reported_cases, output) + os_helper.unlink(filename) + + # Nothing is reported when all tests pass + output = self.run_tests(self.create_test(), env=env) + self.assertEqual(reported(output), []) + self.assertFalse(os.path.exists(filename)) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" @@ -2946,7 +3020,157 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') -from test.libregrtest.results import TestResults +class ReporterTestCase(unittest.TestCase): + def test_parse_traceback(self): + parse = Reporter.parse_traceback + self.assertEqual(parse('AssertionError\n'), + (None, None, 'AssertionError')) + traceback = textwrap.dedent(""" + Traceback (most recent call last): + File "/lib/test_x.py", line 5, in test_x + helper() + File "/lib/test_x.py", line 9, in helper + check() + File "/lib/check.py", line 2, in check + raise ValueError + ValueError + """) + # The last frame in a test file + self.assertEqual(parse(traceback), + ('/lib/test_x.py', '9', 'ValueError')) + # Without a test file, the first frame + traceback = traceback.replace('test_x.py', 'x.py') + self.assertEqual(parse(traceback), ('/lib/x.py', '5', 'ValueError')) + + def test_find_filename(self): + test_dir = self.enterContext(os_helper.temp_dir()) + os.mkdir(os.path.join(test_dir, 'test_pkg')) + for name in ('test_x.py', os.path.join('test_pkg', '__init__.py')): + open(os.path.join(test_dir, name), 'w').close() + + def filename(test_name, filename=None): + result = TestResult(test_name, filename=filename) + return result.find_filename(test_dir) + + self.assertEqual(filename('test_x', '/imported/test_x.py'), + '/imported/test_x.py') + self.assertEqual(filename('test_x'), + os.path.join(test_dir, 'test_x.py')) + self.assertEqual(filename('test_pkg'), + os.path.join(test_dir, 'test_pkg', '__init__.py')) + self.assertIsNone(filename('')) + + +class GitHubActionsReporterTestCase(unittest.TestCase): + ENV_CHANGED = TestResult('test_x', state=State.ENV_CHANGED, + env_changed_reasons=['dangling thread'], + filename='/tests/test_x.py') + + def setUp(self): + self.summary = os_helper.TESTFN + self.addCleanup(os_helper.unlink, self.summary) + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['GITHUB_STEP_SUMMARY'] = self.summary + + def create_reporter(self, *, final_run=True, fail_env_changed=True): + reporter = GitHubActionsReporter() + reporter.run_started(types.SimpleNamespace( + final_run=final_run, fail_env_changed=fail_env_changed, + test_dir=None)) + return reporter + + def annotate_test_case(self, test_case, traceback, **kwargs): + stream = io.StringIO() + self.create_reporter(**kwargs).test_case_failed(test_case, traceback, + stream) + return stream.getvalue() + + def annotate_result(self, result, **kwargs): + with support.captured_stdout() as stdout: + self.create_reporter(**kwargs).test_result(result) + return stdout.getvalue() + + def test_test_case_failed(self): + stdlib_file = os.path.join(os.path.dirname(os.__file__), 'test', + 'test_x.py') + traceback = textwrap.dedent(f"""\ + Traceback (most recent call last): + File "{stdlib_file}", line 3, in test_x + self.fail('50%') + AssertionError: 50% + more + """) + # Only the exception is kept, the stdlib path is relative to the + # checkout, and the data and properties are escaped + self.assertEqual( + self.annotate_test_case('test_x (a: b, c)', traceback), + '::error file=Lib/test/test_x.py,line=3,' + 'title=test_x (a%3A b%2C c)::AssertionError: 50%25%0Amore\n') + + def test_test_case_failed_without_traceback(self): + self.assertEqual(self.annotate_test_case('test_x', 'AssertionError\n'), + '::error title=test_x::AssertionError\n') + + def test_test_case_failed_not_final_run(self): + # Failed tests will be re-run: only the re-run is reported + self.assertEqual(self.annotate_test_case('test_x', 'AssertionError\n', + final_run=False), + '') + + def test_test_result(self): + self.assertEqual(self.annotate_result(self.ENV_CHANGED), + '::error file=/tests/test_x.py,line=1,title=test_x::' + 'test_x failed (env changed)%0Adangling thread\n') + + def test_test_result_not_reported(self): + failed_test_cases = TestResult( + 'test_x', state=State.FAILED, + failures=[('test_a', 'AssertionError\n')]) + cases = { + 'not final run': (self.ENV_CHANGED, {'final_run': False}), + 'not failing the run': (self.ENV_CHANGED, + {'fail_env_changed': False}), + 'passed': (TestResult('test_x', state=State.PASSED), {}), + # Annotated by test_case_failed() + 'failed test cases': (failed_test_cases, {}), + } + for name, (result, kwargs) in cases.items(): + with self.subTest(name): + self.assertEqual(self.annotate_result(result, **kwargs), '') + + def test_summary(self): + results = TestResults() + results.rerun_results = [ + TestResult('test_x', state=State.FAILED, + failures=[('test_a ()', 'Traceback\nAssertionError\n')]), + self.ENV_CHANGED, + ] + self.create_reporter().summary(results, 'FAILURE') + with open(self.summary, encoding='utf-8') as fp: + summary = fp.read() + self.assertEqual(summary, textwrap.dedent("""\ + ## FAILURE: 2 test files and 1 test case failed + + ### test_x failed (1 failure) + +
test_a (<T>) + + ```pytb + Traceback + AssertionError + ``` + +
+ + ### test_x failed (env changed) + + - dangling thread + + """)) + + def test_summary_nothing_failed(self): + self.create_reporter().summary(TestResults(), 'SUCCESS') + self.assertFalse(os.path.exists(self.summary)) class TestColorized(unittest.TestCase): diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb5..c1d670c978c970 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,8 @@ # $Id$ # +import os +import sys import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -66,6 +68,9 @@ def test_simple(self): self.check_wrap(text, 80, [text]) def test_empty_string(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + os.environ["GHA_ANNOTATION_TEST"] = "1" # Check that wrapping the empty string returns an empty list. self.check_wrap("", 6, []) self.check_wrap("", 6, [], drop_whitespace=False) diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 00000000000000..a1c9e648e5c744 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,3 @@ +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary.