pythongh-100227: Add a Granular Lock for _PyRuntime.imports.extensions.dict (pythongh-103460)

The lock is unnecessary as long as there's a GIL, but completely
necessary with a per-interpreter GIL.

Author: ericsnowcurrently

Include/internal/pycore_import.h

@@ -19,6 +19,8 @@ struct _import_runtime_state {
            used exclusively for when the extensions dict is access/modified
            from an arbitrary thread. */
         PyThreadState main_tstate;
+        /* A lock to guard the dict. */
+        PyThread_type_lock mutex;
         /* A dict mapping (filename, name) to PyModuleDef for modules.
            Only legacy (single-phase init) extension modules are added
            and only if they support multiple initialization (m_size >- 0)

Python/import.c

@@ -413,8 +413,11 @@ remove_module(PyThreadState *tstate, PyObject *name)
 Py_ssize_t
 _PyImport_GetNextModuleIndex(void)
 {
+    PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);
     LAST_MODULE_INDEX++;
-    return LAST_MODULE_INDEX;
+    Py_ssize_t index = LAST_MODULE_INDEX;
+    PyThread_release_lock(EXTENSIONS.mutex);
+    return index;
 }
 
 static const char *
@@ -703,21 +706,25 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp)
 const char *
 _PyImport_ResolveNameWithPackageContext(const char *name)
 {
+    PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);
     if (PKGCONTEXT != NULL) {
         const char *p = strrchr(PKGCONTEXT, '.');
         if (p != NULL && strcmp(name, p+1) == 0) {
             name = PKGCONTEXT;
             PKGCONTEXT = NULL;
         }
     }
+    PyThread_release_lock(EXTENSIONS.mutex);
     return name;
 }
 
 const char *
 _PyImport_SwapPackageContext(const char *newcontext)
 {
+    PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);
     const char *oldcontext = PKGCONTEXT;
     PKGCONTEXT = newcontext;
+    PyThread_release_lock(EXTENSIONS.mutex);
     return oldcontext;
 }
 
@@ -865,13 +872,13 @@ gets even messier.
 static inline void
 extensions_lock_acquire(void)
 {
-    // XXX For now the GIL is sufficient.
+    PyThread_acquire_lock(_PyRuntime.imports.extensions.mutex, WAIT_LOCK);
 }
 
 static inline void
 extensions_lock_release(void)
 {
-    // XXX For now the GIL is sufficient.
+    PyThread_release_lock(_PyRuntime.imports.extensions.mutex);
 }
 
 /* Magic for extension modules (built-in as well as dynamically

Python/pystate.c

@@ -380,7 +380,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS
 static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime);
 _Py_COMP_DIAG_POP
 
-#define NUMLOCKS 4
+#define NUMLOCKS 5
 
 static int
 alloc_for_runtime(PyThread_type_lock locks[NUMLOCKS])
@@ -434,6 +434,7 @@ init_runtime(_PyRuntimeState *runtime,
         &runtime->xidregistry.mutex,
         &runtime->getargs.mutex,
         &runtime->unicode_state.ids.lock,
+        &runtime->imports.extensions.mutex,
     };
     for (int i = 0; i < NUMLOCKS; i++) {
         assert(locks[i] != NULL);
@@ -518,6 +519,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime)
         &runtime->xidregistry.mutex,
         &runtime->getargs.mutex,
         &runtime->unicode_state.ids.lock,
+        &runtime->imports.extensions.mutex,
     };
     for (int i = 0; i < NUMLOCKS; i++) {
         FREE_LOCK(*lockptrs[i]);
@@ -546,6 +548,7 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
         &runtime->xidregistry.mutex,
         &runtime->getargs.mutex,
         &runtime->unicode_state.ids.lock,
+        &runtime->imports.extensions.mutex,
     };
     int reinit_err = 0;
     for (int i = 0; i < NUMLOCKS; i++) {