bpo-44708: Only re-run test methods that match names of previously failing test methods (GH-27287)

* Move to a static argparse.Namespace subclass
* Roughly annotate runtest.py
* Refactor libregrtest to use lossless test result objects
* Only re-run test methods that match names of previously failing test methods
* Adopt tests to cover test method name matching

Co-authored-by: Pablo Galindo Salgado <[email protected]>

Author: ambv

Lib/test/libregrtest/cmdline.py

@@ -140,6 +140,39 @@
 #   default (see bpo-30822).
 RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
 
+
+class Namespace(argparse.Namespace):
+    def __init__(self, **kwargs) -> None:
+        self.testdir = None
+        self.verbose = 0
+        self.quiet = False
+        self.exclude = False
+        self.single = False
+        self.randomize = False
+        self.fromfile = None
+        self.findleaks = 1
+        self.fail_env_changed = False
+        self.use_resources = None
+        self.trace = False
+        self.coverdir = 'coverage'
+        self.runleaks = False
+        self.huntrleaks = False
+        self.verbose2 = False
+        self.verbose3 = False
+        self.print_slow = False
+        self.random_seed = None
+        self.use_mp = None
+        self.forever = False
+        self.header = False
+        self.failfast = False
+        self.match_tests = None
+        self.ignore_tests = None
+        self.pgo = False
+        self.pgo_extended = False
+
+        super().__init__(**kwargs)
+
+
 class _ArgParser(argparse.ArgumentParser):
 
     def error(self, message):
@@ -320,13 +353,7 @@ def resources_list(string):
 
 def _parse_args(args, **kwargs):
     # Defaults
-    ns = argparse.Namespace(testdir=None, verbose=0, quiet=False,
-         exclude=False, single=False, randomize=False, fromfile=None,
-         findleaks=1, use_resources=None, trace=False, coverdir='coverage',
-         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
-         random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None, ignore_tests=None,
-         pgo=False)
+    ns = Namespace()
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '

Lib/test/libregrtest/main.py

@@ -11,10 +11,10 @@
 import unittest
 from test.libregrtest.cmdline import _parse_args
 from test.libregrtest.runtest import (
-    findtests, runtest, get_abs_module,
-    STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED,
-    INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, TIMEOUT,
-    PROGRESS_MIN_TIME, format_test_result, is_failed)
+    findtests, runtest, get_abs_module, is_failed,
+    STDTESTS, NOTTESTS, PROGRESS_MIN_TIME,
+    Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
+    ChildError, DidNotRun)
 from test.libregrtest.setup import setup_tests
 from test.libregrtest.pgo import setup_pgo_tests
 from test.libregrtest.utils import removepy, count, format_duration, printlist
@@ -99,34 +99,32 @@ def get_executed(self):
                 | set(self.run_no_tests))
 
     def accumulate_result(self, result, rerun=False):
-        test_name = result.test_name
-        ok = result.result
+        test_name = result.name
 
-        if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun:
-            self.test_times.append((result.test_time, test_name))
+        if not isinstance(result, (ChildError, Interrupted)) and not rerun:
+            self.test_times.append((result.duration_sec, test_name))
 
-        if ok == PASSED:
+        if isinstance(result, Passed):
             self.good.append(test_name)
-        elif ok in (FAILED, CHILD_ERROR):
-            if not rerun:
-                self.bad.append(test_name)
-        elif ok == ENV_CHANGED:
-            self.environment_changed.append(test_name)
-        elif ok == SKIPPED:
-            self.skipped.append(test_name)
-        elif ok == RESOURCE_DENIED:
+        elif isinstance(result, ResourceDenied):
             self.skipped.append(test_name)
             self.resource_denieds.append(test_name)
-        elif ok == TEST_DID_NOT_RUN:
+        elif isinstance(result, Skipped):
+            self.skipped.append(test_name)
+        elif isinstance(result, EnvChanged):
+            self.environment_changed.append(test_name)
+        elif isinstance(result, Failed):
+            if not rerun:
+                self.bad.append(test_name)
+                self.rerun.append(result)
+        elif isinstance(result, DidNotRun):
             self.run_no_tests.append(test_name)
-        elif ok == INTERRUPTED:
+        elif isinstance(result, Interrupted):
             self.interrupted = True
-        elif ok == TIMEOUT:
-            self.bad.append(test_name)
         else:
-            raise ValueError("invalid test result: %r" % ok)
+            raise ValueError("invalid test result: %r" % result)
 
-        if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}:
+        if rerun and not isinstance(result, (Failed, Interrupted)):
             self.bad.remove(test_name)
 
         xml_data = result.xml_data
@@ -314,15 +312,31 @@ def rerun_failed_tests(self):
 
         self.log()
         self.log("Re-running failed tests in verbose mode")
-        self.rerun = self.bad[:]
-        for test_name in self.rerun:
-            self.log(f"Re-running {test_name} in verbose mode")
+        rerun_list = self.rerun[:]
+        self.rerun = []
+        for result in rerun_list:
+            test_name = result.name
+            errors = result.errors or []
+            failures = result.failures or []
+            error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
+            failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
             self.ns.verbose = True
+            orig_match_tests = self.ns.match_tests
+            if errors or failures:
+                if self.ns.match_tests is None:
+                    self.ns.match_tests = []
+                self.ns.match_tests.extend(error_names)
+                self.ns.match_tests.extend(failure_names)
+                matching = "matching: " + ", ".join(self.ns.match_tests)
+                self.log(f"Re-running {test_name} in verbose mode ({matching})")
+            else:
+                self.log(f"Re-running {test_name} in verbose mode")
             result = runtest(self.ns, test_name)
+            self.ns.match_tests = orig_match_tests
 
             self.accumulate_result(result, rerun=True)
 
-            if result.result == INTERRUPTED:
+            if isinstance(result, Interrupted):
                 break
 
         if self.bad:
@@ -383,7 +397,7 @@ def display_result(self):
         if self.rerun:
             print()
             print("%s:" % count(len(self.rerun), "re-run test"))
-            printlist(self.rerun)
+            printlist(r.name for r in self.rerun)
 
         if self.run_no_tests:
             print()
@@ -423,14 +437,14 @@ def run_tests_sequential(self):
                 result = runtest(self.ns, test_name)
                 self.accumulate_result(result)
 
-            if result.result == INTERRUPTED:
+            if isinstance(result, Interrupted):
                 break
 
-            previous_test = format_test_result(result)
+            previous_test = str(result)
             test_time = time.monotonic() - start_time
             if test_time >= PROGRESS_MIN_TIME:
                 previous_test = "%s in %s" % (previous_test, format_duration(test_time))
-            elif result.result == PASSED:
+            elif isinstance(result, Passed):
                 # be quiet: say nothing if the test passed shortly
                 previous_test = None