gh-71936: Fix race condition in multiprocessing.Pool

Proxes of shared objects register a Finalizer in BaseProxy._incref(), and it
will call BaseProxy._decref() when it is GCed. This may cause a race condition
with Pool(maxtasksperchild=None) on Windows.

A connection will be closed and raised TypeError when a GC occurs between
_ConnectionBase._check_writable() and _ConnectionBase._send_bytes() in
_ConnectionBase.send() in the second or later task.

Finalizers should be processed after each task is finished, because all proxies
of shared objects will lost their references in the next task.

Also, BaseProxy does not count references well.

Author: hattya

Lib/multiprocessing/managers.py

@@ -850,20 +850,21 @@ def _incref(self):
         dispatch(conn, None, 'incref', (self._id,))
         util.debug('INCREF %r', self._token.id)
 
-        self._idset.add(self._id)
+        refid = f'{self._id}.{id(self)}'
+        self._idset.add(refid)
 
         state = self._manager and self._manager._state
 
         self._close = util.Finalize(
             self, BaseProxy._decref,
-            args=(self._token, self._authkey, state,
+            args=(self._token, refid, self._authkey, state,
                   self._tls, self._idset, self._Client),
             exitpriority=10
             )
 
     @staticmethod
-    def _decref(token, authkey, state, tls, idset, _Client):
-        idset.discard(token.id)
+    def _decref(token, refid, authkey, state, tls, idset, _Client):
+        idset.discard(refid)
 
         # check whether manager is still alive
         if state is None or state.value == State.STARTED:

Lib/multiprocessing/pool.py

@@ -136,6 +136,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
             put((job, i, (False, wrapped)))
 
         task = job = result = func = args = kwds = None
+        util._run_finalizers()
         completed += 1
     util.debug('worker exiting after %d tasks' % completed)
 

Misc/ACKS

@@ -706,6 +706,7 @@ Larry Hastings
 Tim Hatch
 Zac Hatfield-Dodds
 Shane Hathaway
+Akinori Hattori
 Michael Haubenwallner
 Janko Hauser
 Flavian Hautbois