[3.11] gh-90876: Restore the ability to import multiprocessing when sys.executable is None (GH-106464) (#106495)

gh-90876: Restore the ability to import multiprocessing when `sys.executable` is `None` (GH-106464)

Prevent `multiprocessing.spawn` from failing to *import* in environments
where `sys.executable` is `None`.  This regressed in 3.11 with the addition
of support for path-like objects in multiprocessing.

Adds a test decorator to have tests only run when part of test_multiprocessing_spawn to `_test_multiprocessing.py` so we can start to avoid re-running the same not-global-state specific test in all 3 modes when there is no need.
(cherry picked from commit c60df36)

Co-authored-by: Gregory P. Smith <[email protected]>

Author: miss-islington

Lib/multiprocessing/spawn.py

@@ -31,11 +31,13 @@
     WINSERVICE = False
 else:
     WINEXE = getattr(sys, 'frozen', False)
-    WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+    WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe")
 
 def set_executable(exe):
     global _python_exe
-    if sys.platform == 'win32':
+    if exe is None:
+        _python_exe = exe
+    elif sys.platform == 'win32':
         _python_exe = os.fsdecode(exe)
     else:
         _python_exe = os.fsencode(exe)

Lib/test/_test_multiprocessing.py

@@ -13,6 +13,7 @@
 import os
 import gc
 import errno
+import functools
 import signal
 import array
 import socket
@@ -31,6 +32,7 @@
 from test.support import hashlib_helper
 from test.support import import_helper
 from test.support import os_helper
+from test.support import script_helper
 from test.support import socket_helper
 from test.support import threading_helper
 from test.support import warnings_helper
@@ -170,6 +172,59 @@ def check_enough_semaphores():
                             "to run the test (required: %d)." % nsems_min)
 
 
+def only_run_in_spawn_testsuite(reason):
+    """Returns a decorator: raises SkipTest when SM != spawn at test time.
+
+    This can be useful to save overall Python test suite execution time.
+    "spawn" is the universal mode available on all platforms so this limits the
+    decorated test to only execute within test_multiprocessing_spawn.
+
+    This would not be necessary if we refactored our test suite to split things
+    into other test files when they are not start method specific to be rerun
+    under all start methods.
+    """
+
+    def decorator(test_item):
+
+        @functools.wraps(test_item)
+        def spawn_check_wrapper(*args, **kwargs):
+            if (start_method := multiprocessing.get_start_method()) != "spawn":
+                raise unittest.SkipTest(f"{start_method=}, not 'spawn'; {reason}")
+            return test_item(*args, **kwargs)
+
+        return spawn_check_wrapper
+
+    return decorator
+
+
+class TestInternalDecorators(unittest.TestCase):
+    """Logic within a test suite that could errantly skip tests? Test it!"""
+
+    @unittest.skipIf(sys.platform == "win32", "test requires that fork exists.")
+    def test_only_run_in_spawn_testsuite(self):
+        if multiprocessing.get_start_method() != "spawn":
+            raise unittest.SkipTest("only run in test_multiprocessing_spawn.")
+
+        try:
+            @only_run_in_spawn_testsuite("testing this decorator")
+            def return_four_if_spawn():
+                return 4
+        except Exception as err:
+            self.fail(f"expected decorated `def` not to raise; caught {err}")
+
+        orig_start_method = multiprocessing.get_start_method(allow_none=True)
+        try:
+            multiprocessing.set_start_method("spawn", force=True)
+            self.assertEqual(return_four_if_spawn(), 4)
+            multiprocessing.set_start_method("fork", force=True)
+            with self.assertRaises(unittest.SkipTest) as ctx:
+                return_four_if_spawn()
+            self.assertIn("testing this decorator", str(ctx.exception))
+            self.assertIn("start_method=", str(ctx.exception))
+        finally:
+            multiprocessing.set_start_method(orig_start_method, force=True)
+
+
 #
 # Creates a wrapper for a function which records the time it takes to finish
 #
@@ -5775,6 +5830,7 @@ def test_namespace(self):
 
 
 class TestNamedResource(unittest.TestCase):
+    @only_run_in_spawn_testsuite("spawn specific test.")
     def test_global_named_resource_spawn(self):
         #
         # gh-90549: Check that global named resources in main module
@@ -5785,22 +5841,18 @@ def test_global_named_resource_spawn(self):
         with open(testfn, 'w', encoding='utf-8') as f:
             f.write(textwrap.dedent('''\
                 import multiprocessing as mp
-
                 ctx = mp.get_context('spawn')
-
                 global_resource = ctx.Semaphore()
-
                 def submain(): pass
-
                 if __name__ == '__main__':
                     p = ctx.Process(target=submain)
                     p.start()
                     p.join()
             '''))
-        rc, out, err = test.support.script_helper.assert_python_ok(testfn)
+        rc, out, err = script_helper.assert_python_ok(testfn)
         # on error, err = 'UserWarning: resource_tracker: There appear to
         # be 1 leaked semaphore objects to clean up at shutdown'
-        self.assertEqual(err, b'')
+        self.assertFalse(err, msg=err.decode('utf-8'))
 
 
 class MiscTestCase(unittest.TestCase):
@@ -5809,6 +5861,24 @@ def test__all__(self):
         support.check__all__(self, multiprocessing, extra=multiprocessing.__all__,
                              not_exported=['SUBDEBUG', 'SUBWARNING'])
 
+    @only_run_in_spawn_testsuite("avoids redundant testing.")
+    def test_spawn_sys_executable_none_allows_import(self):
+        # Regression test for a bug introduced in
+        # https://github.com/python/cpython/issues/90876 that caused an
+        # ImportError in multiprocessing when sys.executable was None.
+        # This can be true in embedded environments.
+        rc, out, err = script_helper.assert_python_ok(
+            "-c",
+            """if 1:
+            import sys
+            sys.executable = None
+            assert "multiprocessing" not in sys.modules, "already imported!"
+            import multiprocessing
+            import multiprocessing.spawn  # This should not fail\n""",
+        )
+        self.assertEqual(rc, 0)
+        self.assertFalse(err, msg=err.decode('utf-8'))
+
 
 #
 # Mixins

Misc/NEWS.d/next/Library/2023-07-05-13-08-23.gh-issue-90876.Qvlkfl.rst

@@ -0,0 +1,3 @@
+Prevent :mod:`multiprocessing.spawn` from failing to *import* in environments
+where ``sys.executable`` is ``None``.  This regressed in 3.11 with the addition
+of support for path-like objects in multiprocessing.