gh-84559: multiprocessing: detect if forkserver cannot work due to missing hmac-sha256

Doc/library/hashlib.rst

@@ -55,11 +55,14 @@ hash supplied more than 2047 bytes of data at once in its constructor or
 .. index:: single: OpenSSL; (use in module hashlib)
 
 Constructors for hash algorithms that are always present in this module are
-:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`,
+:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`,
 :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`,
 :func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`.
-:func:`md5` is normally available as well, though it may be missing or blocked
-if you are using a rare "FIPS compliant" build of Python.
+Some of these may be missing or blocked if you are running in an environment
+with OpenSSL's "FIPS mode" configured to exclude some hash algorithms from its
+default provider and are using a Python runtime built with that in mind.  Such
+environments are unusual.
+
 These correspond to :data:`algorithms_guaranteed`.
 
 Additional algorithms may also be available if your Python distribution's
@@ -119,7 +122,7 @@ More condensed:
 Constructors
 ------------
 
-.. function:: new(name[, data], *, usedforsecurity=True)
+.. function:: new(name[, data], \*, usedforsecurity=True)
 
    Is a generic constructor that takes the string *name* of the desired
    algorithm as its first parameter.  It also exists to allow access to the
@@ -134,16 +137,16 @@ Using :func:`new` with an algorithm name:
    '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406'
 
 
-.. function:: md5([, data], *, usedforsecurity=True)
-.. function:: sha1([, data], *, usedforsecurity=True)
-.. function:: sha224([, data], *, usedforsecurity=True)
-.. function:: sha256([, data], *, usedforsecurity=True)
-.. function:: sha384([, data], *, usedforsecurity=True)
-.. function:: sha512([, data], *, usedforsecurity=True)
-.. function:: sha3_224([, data], *, usedforsecurity=True)
-.. function:: sha3_256([, data], *, usedforsecurity=True)
-.. function:: sha3_384([, data], *, usedforsecurity=True)
-.. function:: sha3_512([, data], *, usedforsecurity=True)
+.. function:: md5([, data], \*, usedforsecurity=True)
+.. function:: sha1([, data], \*, usedforsecurity=True)
+.. function:: sha224([, data], \*, usedforsecurity=True)
+.. function:: sha256([, data], \*, usedforsecurity=True)
+.. function:: sha384([, data], \*, usedforsecurity=True)
+.. function:: sha512([, data], \*, usedforsecurity=True)
+.. function:: sha3_224([, data], \*, usedforsecurity=True)
+.. function:: sha3_256([, data], \*, usedforsecurity=True)
+.. function:: sha3_384([, data], \*, usedforsecurity=True)
+.. function:: sha3_512([, data], \*, usedforsecurity=True)
 
 Named constructors such as these are faster than passing an algorithm name to
 :func:`new`.
@@ -156,9 +159,10 @@ Hashlib provides the following constant module attributes:
 .. data:: algorithms_guaranteed
 
    A set containing the names of the hash algorithms guaranteed to be supported
-   by this module on all platforms.  Note that 'md5' is in this list despite
-   some upstream vendors offering an odd "FIPS compliant" Python build that
-   excludes it.
+   by this module on all platforms.  Note that the guarnatees do not hold true
+   in the face of vendors offering "FIPS compliant" Python builds that exclude
+   some algorithms entirely.  Similarly when OpenSSL is used and its FIPS mode
+   configuration disables some in the default provider.
 
    .. versionadded:: 3.2
 

Lib/hashlib.py

@@ -55,17 +55,18 @@
 
 # This tuple and __get_builtin_constructor() must be modified if a new
 # always available algorithm is added.
-__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
-                      'blake2b', 'blake2s',
-                      'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-                      'shake_128', 'shake_256')
+__always_supported = [
+    'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
+    'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+    'shake_128', 'shake_256', 'blake2b', 'blake2s'
+]
 
 
 algorithms_guaranteed = set(__always_supported)
 algorithms_available = set(__always_supported)
 
-__all__ = __always_supported + ('new', 'algorithms_guaranteed',
-                                'algorithms_available', 'file_digest')
+__all__ = __always_supported + [
+    'new', 'algorithms_guaranteed', 'algorithms_available', 'file_digest']
 
 
 __builtin_constructor_cache = {}
@@ -243,9 +244,11 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18):
     # version not supporting that algorithm.
     try:
         globals()[__func_name] = __get_hash(__func_name)
-    except ValueError:
-        import logging
-        logging.exception('code for hash %s was not found.', __func_name)
+    except ValueError as exc:
+        # Errors logged here would be seen as noise by most people.
+        # Code using a missing hash will get an obvious exception.
+        __all__.remove(__func_name)
+        algorithms_available.remove(__func_name)
 
 
 # Cleanup locals()

Lib/multiprocessing/connection.py

@@ -23,6 +23,7 @@
 from . import util
 
 from . import AuthenticationError, BufferTooShort
+from . import context
 from .context import reduction
 _ForkingPickler = reduction.ForkingPickler
 
@@ -899,9 +900,9 @@ def _create_response(authkey, message):
             return hmac.new(authkey, message, 'md5').digest()
         except ValueError:
             # HMAC-MD5 is not available (FIPS mode?), fall back to
-            # HMAC-SHA2-256 modern protocol. The legacy server probably
+            # our modern HMAC-SHA* protocol. The legacy server probably
             # doesn't support it and will reject us anyways. :shrug:
-            digest_name = 'sha256'
+            digest_name = context._DIGEST_FOR_CONNECTION_HMAC
     # Modern protocol, indicate the digest used in the reply.
     response = hmac.new(authkey, message, digest_name).digest()
     return b'{%s}%s' % (digest_name.encode('ascii'), response)
@@ -932,10 +933,12 @@ def _verify_challenge(authkey, message, response):
         raise AuthenticationError('digest received was wrong')
 
 
-def deliver_challenge(connection, authkey: bytes, digest_name='sha256'):
+def deliver_challenge(connection, authkey: bytes, digest_name: str = ''):
     if not isinstance(authkey, bytes):
         raise ValueError(
             "Authkey must be bytes, not {0!s}".format(type(authkey)))
+    if not digest_name:
+        digest_name = context._DIGEST_FOR_CONNECTION_HMAC
     assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint"
     message = os.urandom(MESSAGE_LENGTH)
     message = b'{%s}%s' % (digest_name.encode('ascii'), message)

Lib/multiprocessing/context.py

@@ -23,6 +23,12 @@ class TimeoutError(ProcessError):
 class AuthenticationError(ProcessError):
     pass
 
+# The default digest for multiprocessing.connection to use for auth.
+# We configure it here so that it can be tested when choosing a
+# default context without a circular import.
+# Must be the str of a value seen in connection._ALLOWED_DIGESTS.
+_DIGEST_FOR_CONNECTION_HMAC = 'sha256'
+
 #
 # Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py
 #
@@ -313,6 +319,21 @@ class ForkServerContext(BaseContext):
         def _check_available(self):
             if not reduction.HAVE_SEND_HANDLE:
                 raise ValueError('forkserver start method not available')
+            if not _test_if_connection_can_work():
+                raise ValueError(f'forkserver start method not available due to missing hmac-{_DIGEST_FOR_CONNECTION_HMAC}')
+
+    def _test_if_connection_can_work() -> bool:
+        # Authenticated connections required for forkserver using hmac.
+        # If the algorithm is unavailable (poor FIPS mode config?) at
+        # import time, we cannot default to forkserver.  If a user
+        # changes the _DIGEST_FOR_CONNECTION_HMAC to one that works in
+        # their strange config, the forkserver context will still work.
+        import hmac
+        try:
+            hmac.new(b'test-key'*8, b'', _DIGEST_FOR_CONNECTION_HMAC)
+        except ValueError:
+            return False
+        return True
 
     _concrete_contexts = {
         'fork': ForkContext(),
@@ -322,7 +343,8 @@ def _check_available(self):
     # bpo-33725: running arbitrary code after fork() is no longer reliable
     # on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
     # gh-84559: We changed everyones default to a thread safeish one in 3.14.
-    if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
+    if (reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin' and
+        _test_if_connection_can_work()):
         _default_context = DefaultContext(_concrete_contexts['forkserver'])
     else:
         _default_context = DefaultContext(_concrete_contexts['spawn'])

Lib/test/_test_multiprocessing.py

@@ -1431,6 +1431,7 @@ def _acquire_event(lock, event):
         event.set()
         time.sleep(1.0)
 
+    @hashlib_helper.requires_hashdigest('sha256')  # Manager RPC connection auth
     def test_repr_lock(self):
         if self.TYPE != 'processes':
             self.skipTest('test not appropriate for {}'.format(self.TYPE))
@@ -1496,6 +1497,7 @@ def _acquire_release(lock, timeout, l=None, n=1):
         for _ in range(n):
             lock.release()
 
+    @hashlib_helper.requires_hashdigest('sha256')  # Manager RPC connection auth
     def test_repr_rlock(self):
         if self.TYPE != 'processes':
             self.skipTest('test not appropriate for {}'.format(self.TYPE))
@@ -3417,6 +3419,7 @@ def tearDown(self):
         self.mgr.shutdown()
         self.mgr.join()
 
+    @hashlib_helper.requires_hashdigest('sha256')  # multiprocessing.connection
     def test_queue_get(self):
         queue = self.mgr.Queue()
         if gc.isenabled():
@@ -3730,6 +3733,7 @@ def test_context(self):
         if self.TYPE == 'processes':
             self.assertRaises(OSError, l.accept)
 
+    @hashlib_helper.requires_hashdigest('sha256')  # connection auth
     def test_empty_authkey(self):
         # bpo-43952: allow empty bytes as authkey
         def handler(*args):
@@ -5782,9 +5786,11 @@ def test_get_all_start_methods(self):
             self.assertIn(methods[0], {'forkserver', 'spawn'},
                           msg='3.14+ default must not be fork')
             if methods[0] == 'spawn':
-                # Confirm that the current default selection logic prefers
-                # forkserver vs spawn when available.
-                self.assertNotIn('forkserver', methods)
+                if not hashlib_helper.in_openssl_fips_mode():
+                    # Confirm that the current default selection logic prefers
+                    # forkserver vs spawn when available.
+                    # OpenSSL FIPS mode can disable this by blocking sha256.
+                    self.assertNotIn('forkserver', methods)
 
     def test_preload_resources(self):
         if multiprocessing.get_start_method() != 'forkserver':
@@ -5805,7 +5811,11 @@ def test_mixed_startmethod(self):
         # Fork-based locks cannot be used with spawned process
         for process_method in ["spawn", "forkserver"]:
             queue = multiprocessing.get_context("fork").Queue()
-            process_ctx = multiprocessing.get_context(process_method)
+            try:
+                process_ctx = multiprocessing.get_context(process_method)
+            except ValueError as err:
+                self.skipTest(err)
+                continue
             p = process_ctx.Process(target=close_queue, args=(queue,))
             err_msg = "A SemLock created in a fork"
             with self.assertRaisesRegex(RuntimeError, err_msg):
@@ -5814,8 +5824,13 @@ def test_mixed_startmethod(self):
         # non-fork-based locks can be used with all other start methods
         for queue_method in ["spawn", "forkserver"]:
             for process_method in multiprocessing.get_all_start_methods():
-                queue = multiprocessing.get_context(queue_method).Queue()
-                process_ctx = multiprocessing.get_context(process_method)
+                try:
+                    queue_ctx = multiprocessing.get_context(queue_method)
+                    process_ctx = multiprocessing.get_context(process_method)
+                except ValueError as err:
+                    self.skipTest(err)
+                    continue
+                queue = queue_ctx.Queue()
                 p = process_ctx.Process(target=close_queue, args=(queue,))
                 p.start()
                 p.join()

Lib/test/support/hashlib_helper.py

@@ -8,6 +8,13 @@
     _hashlib = None
 
 
+def in_openssl_fips_mode() -> bool:
+    """Is OpenSSL based _hashlib is present & operating in FIPS mode?"""
+    if _hashlib and _hashlib.get_fips_mode() != 0:
+        return True
+    return False
+
+
 def requires_hashdigest(digestname, openssl=None, usedforsecurity=True):
     """Decorator raising SkipTest if a hashing algorithm is not available
 

Lib/test/test_concurrent_futures/test_init.py

@@ -141,6 +141,11 @@ def test_spawn(self):
 
     @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
     def test_forkserver(self):
+        import multiprocessing
+        try:
+            multiprocessing.get_context("forkserver")
+        except ValueError as err:
+            self.skipTest(str(err))
         self._test(ProcessPoolForkserverFailingInitializerTest)
 
 

Lib/test/test_concurrent_futures/util.py

@@ -119,7 +119,10 @@ def get_context(self):
             self.skipTest("require unix system")
         if support.check_sanitizer(thread=True):
             self.skipTest("TSAN doesn't support threads after fork")
-        return super().get_context()
+        try:
+            return super().get_context()
+        except ValueError as err:
+            self.skipTest(str(err))
 
 
 def create_executor_tests(remote_globals, mixin, bases=(BaseTestCase,),