pythongh-109566: regrtest reexecutes the process

regrtest now replaces the process with a new process to add options
to the Python command line. Add options: "-u -W default -bb". When
--fast-ci and --slow-ci options are used, the -E option is also
added.

The following methods to run the Python test suite don't replace the
process:

* "import test.autotest"
* "from test.regrtest import main; main()"

Changes:

* PCbuild/rt.bat and Tools/scripts/run_tests.py no longer need to add
  "-u -W default -bb -E" options to Python: it's now done by
  regrtest.
* Fix Tools/scripts/run_tests.py: flush stdout before replacing the
  process. Previously, buffered messages were lost.

Author: vstinner

Lib/test/autotest.py

@@ -2,4 +2,4 @@
 # It can be especially handy if you're in an interactive shell, e.g.,
 # from test import autotest.
 from test.libregrtest.main import main
-main()
+main(reexec=False)

Lib/test/libregrtest/cmdline.py

@@ -184,6 +184,7 @@ def __init__(self, **kwargs) -> None:
         self.threshold = None
         self.fail_rerun = False
         self.tempdir = None
+        self.no_reexec = False
 
         super().__init__(**kwargs)
 
@@ -343,6 +344,8 @@ def _create_parser():
                        help='override the working directory for the test run')
     group.add_argument('--cleanup', action='store_true',
                        help='remove old test_python_* directories')
+    group.add_argument('--no-reexec', action='store_true',
+                       help="internal option, don't use it")
     return parser
 
 

Lib/test/libregrtest/main.py

@@ -1,6 +1,7 @@
 import os
 import random
 import re
+import shlex
 import sys
 import time
 
@@ -20,7 +21,7 @@
     StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple,
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
-    display_header, cleanup_temp_dir,
+    display_header, cleanup_temp_dir, print_warning,
     MS_WINDOWS)
 
 
@@ -47,7 +48,7 @@ class Regrtest:
     directly to set the values that would normally be set by flags
     on the command line.
     """
-    def __init__(self, ns: Namespace):
+    def __init__(self, ns: Namespace, reexec: bool = True):
         # Log verbosity
         self.verbose: int = int(ns.verbose)
         self.quiet: bool = ns.quiet
@@ -69,6 +70,7 @@ def __init__(self, ns: Namespace):
         self.want_cleanup: bool = ns.cleanup
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
+        self.want_reexec: bool = (reexec and not ns.no_reexec)
 
         # Select tests
         if ns.match_tests:
@@ -95,6 +97,7 @@ def __init__(self, ns: Namespace):
         self.worker_json: StrJSON | None = ns.worker_json
 
         # Options to run tests
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.fail_fast: bool = ns.failfast
         self.fail_env_changed: bool = ns.fail_env_changed
         self.fail_rerun: bool = ns.fail_rerun
@@ -411,6 +414,11 @@ def create_run_tests(self, tests: TestTuple):
         )
 
     def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
+        # Set sys.stdout encoder error handler to backslashreplace,
+        # similar to sys.stderr error handler, to avoid UnicodeEncodeError
+        # when printing a traceback or any other non-encodable character.
+        sys.stdout.reconfigure(errors="backslashreplace")
+
         if self.hunt_refleak and self.hunt_refleak.warmups < 3:
             msg = ("WARNING: Running tests with --huntrleaks/-R and "
                    "less than 3 warmup repetitions can give false positives!")
@@ -483,7 +491,40 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 # processes.
                 return self._run_tests(selected, tests)
 
+    def _reexecute_python(self):
+        if self.python_cmd:
+            # Do nothing if --python=cmd option is used
+            return
+
+        python_opts = [
+            '-u',                 # Unbuffered stdout and stderr
+            '-W', 'default',      # Add warnings filter 'default'
+            '-bb',                # Error on bytes/str comparison
+        ]
+        if self.ci_mode:
+            python_opts.append('-E')  # Ignore PYTHON* environment variables
+
+        cmd = [*sys.orig_argv, "--no-reexec"]
+        cmd[1:1] = python_opts
+
+        # Make sure that messages before execv() are logged
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        try:
+            os.execv(cmd[0], cmd)
+            # execv() do no return and so we don't get to this line on success
+        except OSError as exc:
+            cmd_text = shlex.join(cmd)
+            print_warning(f"Failed to reexecute Python: {exc!r}\n"
+                          f"Command: {cmd_text}")
+
     def main(self, tests: TestList | None = None):
+        if self.want_reexec:
+            self._reexecute_python()
+
+        print(f"{sys.flags.ignore_environment=}")
+
         if self.junit_filename and not os.path.isabs(self.junit_filename):
             self.junit_filename = os.path.abspath(self.junit_filename)
 
@@ -515,7 +556,7 @@ def main(self, tests: TestList | None = None):
         sys.exit(exitcode)
 
 
-def main(tests=None, **kwargs):
+def main(tests=None, reexec=True, **kwargs):
     """Run the Python suite."""
     ns = _parse_args(sys.argv[1:], **kwargs)
-    Regrtest(ns).main(tests=tests)
+    Regrtest(ns, reexec=reexec).main(tests=tests)

Lib/test/libregrtest/setup.py

@@ -11,7 +11,7 @@
 from .runtests import RunTests
 from .utils import (
     setup_unraisable_hook, setup_threading_excepthook, fix_umask,
-    replace_stdout, adjust_rlimit_nofile)
+    adjust_rlimit_nofile)
 
 
 UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD"
@@ -49,7 +49,7 @@ def setup_process():
             faulthandler.register(signum, chain=True, file=stderr_fd)
 
     adjust_rlimit_nofile()
-    replace_stdout()
+
     support.record_original_stdout(sys.stdout)
 
     # Some times __path__ and __file__ are not absolute (e.g. while running from

Lib/test/libregrtest/utils.py

@@ -495,32 +495,6 @@ def normalize_test_name(test_full_name, *, is_error=False):
     return short_name
 
 
-def replace_stdout():
-    """Set stdout encoder error handler to backslashreplace (as stderr error
-    handler) to avoid UnicodeEncodeError when printing a traceback"""
-    stdout = sys.stdout
-    try:
-        fd = stdout.fileno()
-    except ValueError:
-        # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper
-        # object. Leaving sys.stdout unchanged.
-        #
-        # Catch ValueError to catch io.UnsupportedOperation on TextIOBase
-        # and ValueError on a closed stream.
-        return
-
-    sys.stdout = open(fd, 'w',
-        encoding=stdout.encoding,
-        errors="backslashreplace",
-        closefd=False,
-        newline='\n')
-
-    def restore_stdout():
-        sys.stdout.close()
-        sys.stdout = stdout
-    atexit.register(restore_stdout)
-
-
 def adjust_rlimit_nofile():
     """
     On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256)
@@ -569,7 +543,6 @@ def display_header(use_resources: tuple[str, ...]):
     print("== encodings: locale=%s, FS=%s"
           % (locale.getencoding(), sys.getfilesystemencoding()))
 
-
     if use_resources:
         print(f"== resources ({len(use_resources)}): "
               f"{', '.join(sorted(use_resources))}")
@@ -588,18 +561,18 @@ def display_header(use_resources: tuple[str, ...]):
         sanitizers.append("memory")
     if ubsan:
         sanitizers.append("undefined behavior")
-    if not sanitizers:
-        return
-
-    print(f"== sanitizers: {', '.join(sanitizers)}")
-    for sanitizer, env_var in (
-        (asan, "ASAN_OPTIONS"),
-        (msan, "MSAN_OPTIONS"),
-        (ubsan, "UBSAN_OPTIONS"),
-    ):
-        options= os.environ.get(env_var)
-        if sanitizer and options is not None:
-            print(f"== {env_var}={options!r}")
+    if sanitizers:
+        print(f"== sanitizers: {', '.join(sanitizers)}")
+        for sanitizer, env_var in (
+            (asan, "ASAN_OPTIONS"),
+            (msan, "MSAN_OPTIONS"),
+            (ubsan, "UBSAN_OPTIONS"),
+        ):
+            options= os.environ.get(env_var)
+            if sanitizer and options is not None:
+                print(f"== {env_var}={options!r}")
+
+    print(flush=True)
 
 
 def cleanup_temp_dir(tmp_dir: StrPath):

Lib/test/regrtest.py

@@ -40,7 +40,7 @@ def _main():
     # sanity check
     assert __file__ == os.path.abspath(sys.argv[0])
 
-    main()
+    main(reexec=False)
 
 
 if __name__ == '__main__':

Lib/test/support/__init__.py

@@ -1346,6 +1346,7 @@ def print_warning(msg):
     stream = print_warning.orig_stderr
     for line in msg.splitlines():
         print(f"Warning -- {line}", file=stream)
+    print(file=stream)
     stream.flush()
 
 # bpo-39983: Store the original sys.stderr at Python startup to be able to

Lib/test/test_regrtest.py

@@ -382,7 +382,8 @@ def check_ci_mode(self, args, use_resources):
         # Check Regrtest attributes which are more reliable than Namespace
         # which has an unclear API
         regrtest = main.Regrtest(ns)
-        self.assertNotEqual(regrtest.num_workers, 0)
+        self.assertTrue(regrtest.ci_mode)
+        self.assertGreaterEqual(regrtest.num_workers, 1)
         self.assertTrue(regrtest.want_rerun)
         self.assertTrue(regrtest.randomize)
         self.assertIsNone(regrtest.random_seed)

PCbuild/rt.bat

@@ -48,7 +48,7 @@ if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts
 
 if not defined prefix set prefix=%pcbuild%amd64
 set exe=%prefix%\python%suffix%.exe
-set cmd="%exe%" %dashO% -u -Wd -E -bb -m test %regrtestargs%
+set cmd="%exe%" %dashO% -m test %regrtestargs%
 if defined qmode goto Qmode
 
 echo Deleting .pyc files ...

Tools/scripts/run_tests.py

@@ -23,11 +23,7 @@ def is_python_flag(arg):
 
 
 def main(regrtest_args):
-    args = [sys.executable,
-            '-u',                 # Unbuffered stdout and stderr
-            '-W', 'default',      # Warnings set to 'default'
-            '-bb',                # Warnings about bytes/bytearray
-            ]
+    args = [sys.executable]
 
     cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
     if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
@@ -47,7 +43,6 @@ def main(regrtest_args):
         }
     else:
         environ = os.environ.copy()
-        args.append("-E")
 
     # Allow user-specified interpreter options to override our defaults.
     args.extend(test.support.args_from_interpreter_flags())
@@ -70,7 +65,8 @@ def main(regrtest_args):
 
     args.extend(regrtest_args)
 
-    print(shlex.join(args))
+    print(shlex.join(args), flush=True)
+
     if sys.platform == 'win32':
         from subprocess import call
         sys.exit(call(args))