Skip to content

Commit 379d59e

Browse files
authored
Fix multiprocessing warnings when runnign tests on Python 3.12 (#16564)
I saw a bunch of warnings when running tests in parallel using pytest. When running tests sequentially using `-n0` I didn't see warnings. This only seems to happen on Linux. The warnings were like these, which can be fixed by avoiding the use of fork, and using forkserver instead: ``` mypy/test/teststubgen.py::StubgenPythonSuite::stubgen.test::testNestedClass_inspect /usr/local/lib/python3.12/multiprocessing/popen_fork.py:66: DeprecationWarning: This process (pid=84587) is multi-threaded, use of fork() may lead to deadlocks in the child. self.pid = os.fork() ``` Relevant discussion: https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555
1 parent e69c5cd commit 379d59e

File tree

2 files changed

+22
-11
lines changed

2 files changed

+22
-11
lines changed

mypy/moduleinspect.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import pkgutil
99
import queue
1010
import sys
11-
from multiprocessing import Process, Queue
11+
from multiprocessing import Queue, get_context
1212
from types import ModuleType
1313

1414

@@ -123,9 +123,13 @@ def __init__(self) -> None:
123123
self._start()
124124

125125
def _start(self) -> None:
126-
self.tasks: Queue[str] = Queue()
127-
self.results: Queue[ModuleProperties | str] = Queue()
128-
self.proc = Process(target=worker, args=(self.tasks, self.results, sys.path))
126+
if sys.platform == "linux":
127+
ctx = get_context("forkserver")
128+
else:
129+
ctx = get_context("spawn")
130+
self.tasks: Queue[str] = ctx.Queue()
131+
self.results: Queue[ModuleProperties | str] = ctx.Queue()
132+
self.proc = ctx.Process(target=worker, args=(self.tasks, self.results, sys.path))
129133
self.proc.start()
130134
self.counter = 0 # Number of successful roundtrips
131135

mypy/test/testipc.py

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import sys
44
import time
5-
from multiprocessing import Process, Queue
5+
from multiprocessing import Queue, get_context
66
from unittest import TestCase, main
77

88
import pytest
@@ -35,10 +35,17 @@ def server_multi_message_echo(q: Queue[str]) -> None:
3535

3636

3737
class IPCTests(TestCase):
38+
def setUp(self) -> None:
39+
if sys.platform == "linux":
40+
# The default "fork" start method is potentially unsafe
41+
self.ctx = get_context("forkserver")
42+
else:
43+
self.ctx = get_context("spawn")
44+
3845
def test_transaction_large(self) -> None:
39-
queue: Queue[str] = Queue()
46+
queue: Queue[str] = self.ctx.Queue()
4047
msg = "t" * 200000 # longer than the max read size of 100_000
41-
p = Process(target=server, args=(msg, queue), daemon=True)
48+
p = self.ctx.Process(target=server, args=(msg, queue), daemon=True)
4249
p.start()
4350
connection_name = queue.get()
4451
with IPCClient(connection_name, timeout=1) as client:
@@ -49,9 +56,9 @@ def test_transaction_large(self) -> None:
4956
p.join()
5057

5158
def test_connect_twice(self) -> None:
52-
queue: Queue[str] = Queue()
59+
queue: Queue[str] = self.ctx.Queue()
5360
msg = "this is a test message"
54-
p = Process(target=server, args=(msg, queue), daemon=True)
61+
p = self.ctx.Process(target=server, args=(msg, queue), daemon=True)
5562
p.start()
5663
connection_name = queue.get()
5764
with IPCClient(connection_name, timeout=1) as client:
@@ -67,8 +74,8 @@ def test_connect_twice(self) -> None:
6774
assert p.exitcode == 0
6875

6976
def test_multiple_messages(self) -> None:
70-
queue: Queue[str] = Queue()
71-
p = Process(target=server_multi_message_echo, args=(queue,), daemon=True)
77+
queue: Queue[str] = self.ctx.Queue()
78+
p = self.ctx.Process(target=server_multi_message_echo, args=(queue,), daemon=True)
7279
p.start()
7380
connection_name = queue.get()
7481
with IPCClient(connection_name, timeout=1) as client:

0 commit comments

Comments
 (0)