From 60e7b12d9093e1ff5cd6987e9c782c0eccff1196 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 11 Mar 2024 23:12:19 +0900 Subject: [PATCH 01/12] gh-112536: Add --tsan test for reasonable TSAN execution times. --- Lib/test/libregrtest/cmdline.py | 3 +++ Lib/test/libregrtest/main.py | 11 ++++++++- Lib/test/libregrtest/tsan.py | 43 +++++++++++++++++++++++++++++++++ Tools/tsan/supressions.txt | 3 +++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Lib/test/libregrtest/tsan.py create mode 100644 Tools/tsan/supressions.txt diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 608b12bb6f2a38..2cf63330366aed 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -164,6 +164,7 @@ def __init__(self, **kwargs) -> None: self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False + self.tsan = False self.worker_json = None self.start = None self.timeout = None @@ -333,6 +334,8 @@ def _create_parser(): help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='enable TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 126daca388fd7f..28f3c5abd0bc2e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -18,6 +18,7 @@ from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, @@ -56,6 +57,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan # Test results self.results: TestResults = TestResults() @@ -87,7 +89,11 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Workers if ns.use_mp is None: - num_workers = 0 # run sequentially + if ns.tsan: + # For TSAN tests, use number of CPU of the current process. + num_workers = os.process_cpu_count() + else: + num_workers = 0 # run sequentially elif ns.use_mp <= 0: num_workers = -1 # use the number of CPUs else: @@ -183,6 +189,9 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) + if self.tsan: + setup_tsan_tests(self.cmdline_args) + exclude_tests = set() if self.exclude: for arg in self.cmdline_args: diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000000..53c7957b5efed2 --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,43 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen to run a thread sanitizer in a reasonable executable time. + +TSAN_TESTS = [ + 'test_asyncio', + 'test_capi', + 'test_code', + 'test_compilerall', + 'test_concurrent_futures', + 'test_enum', + 'test_fork1', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_import', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_multiprocessing_forkserver', + 'test_multiprocessing_spawn', + 'test_pickle', + 'test_queue', + 'test_list', + 'test_dict', + 'test_set', + 'test_tuple', + 'test_smtpnet', + 'test_socketserver', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_urllib2net', + 'test_weakref' +] + + +def setup_tsan_tests(cmdline_args): + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt new file mode 100644 index 00000000000000..b70dcc30d3aaf5 --- /dev/null +++ b/Tools/tsan/supressions.txt @@ -0,0 +1,3 @@ +## reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:get_allocator_unlocked +race:set_allocator_unlocked From a0c5961f819084cc9bd8014c5c70b913dc148a95 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Mon, 11 Mar 2024 23:17:15 +0900 Subject: [PATCH 02/12] Update documentation --- Doc/library/test.rst | 7 +++++++ .../Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7d28f625345726..8865b47452c8a4 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -352,6 +352,13 @@ The :mod:`test.support` module defines the following constants: Set to the ``data`` directory within the test package. +.. data:: TSAN + + Set when tests can be skipped when they are not useful for ThreadSanitizer. + + .. versionadded:: 3.13 + + .. data:: MAX_Py_ssize_t Set to :data:`sys.maxsize` for big memory tests. diff --git a/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst b/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst new file mode 100644 index 00000000000000..de9e1c557b093c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst @@ -0,0 +1,2 @@ +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. From 4230c92db456a49185236bcc59d148f3627a8348 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Mon, 11 Mar 2024 23:25:31 +0900 Subject: [PATCH 03/12] fix --- Lib/test/libregrtest/tsan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 53c7957b5efed2..d833542bd03aca 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -5,7 +5,7 @@ 'test_asyncio', 'test_capi', 'test_code', - 'test_compilerall', + 'test_compileall', 'test_concurrent_futures', 'test_enum', 'test_fork1', From a13ecb71f8ed8fa99d5e775bca0bdfd726a59fdf Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Tue, 12 Mar 2024 00:36:18 +0900 Subject: [PATCH 04/12] Update --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 28f3c5abd0bc2e..968bf480a64fe1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -91,7 +91,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): if ns.use_mp is None: if ns.tsan: # For TSAN tests, use number of CPU of the current process. - num_workers = os.process_cpu_count() + num_workers = os.process_cpu_count() # type: ignore[attr-defined] else: num_workers = 0 # run sequentially elif ns.use_mp <= 0: From 9a4b5a3f324d547b08186d9b935130d225ff5bad Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 12 Mar 2024 00:45:05 +0900 Subject: [PATCH 05/12] Update Lib/test/libregrtest/main.py Co-authored-by: Alex Waygood --- Lib/test/libregrtest/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 968bf480a64fe1..f242e58a8bc812 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -91,6 +91,9 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): if ns.use_mp is None: if ns.tsan: # For TSAN tests, use number of CPU of the current process. + # + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet num_workers = os.process_cpu_count() # type: ignore[attr-defined] else: num_workers = 0 # run sequentially From 2df350233ef16663c1fd9a65bdd6fc428c4ca8bc Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Tue, 12 Mar 2024 00:54:15 +0900 Subject: [PATCH 06/12] lint --- Lib/test/libregrtest/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f242e58a8bc812..df6602e97d2ade 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -92,8 +92,8 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): if ns.tsan: # For TSAN tests, use number of CPU of the current process. # - # os.process.cpu_count() is new in Python 3.13; - # mypy doesn't know about it yet + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet num_workers = os.process_cpu_count() # type: ignore[attr-defined] else: num_workers = 0 # run sequentially From b3fac0b56f81c058bf4572b701681dfe70856fff Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 12 Mar 2024 01:03:22 +0900 Subject: [PATCH 07/12] Exclude some tests --- Lib/test/libregrtest/tsan.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index d833542bd03aca..74175f79128dbb 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -2,13 +2,10 @@ # chosen to run a thread sanitizer in a reasonable executable time. TSAN_TESTS = [ - 'test_asyncio', 'test_capi', 'test_code', 'test_compileall', - 'test_concurrent_futures', 'test_enum', - 'test_fork1', 'test_functools', 'test_httpservers', 'test_imaplib', @@ -16,14 +13,7 @@ 'test_importlib', 'test_io', 'test_logging', - 'test_multiprocessing_forkserver', - 'test_multiprocessing_spawn', - 'test_pickle', 'test_queue', - 'test_list', - 'test_dict', - 'test_set', - 'test_tuple', 'test_smtpnet', 'test_socketserver', 'test_ssl', From d8d2e8ba301c92166a6f5c1468160f9d99e35ff0 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 12 Mar 2024 01:29:48 +0900 Subject: [PATCH 08/12] Adjust more tests --- Lib/test/libregrtest/main.py | 6 +----- Lib/test/libregrtest/tsan.py | 4 ---- Tools/tsan/supressions.txt | 2 ++ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index df6602e97d2ade..cfee9fcfa8b56a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -90,11 +90,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Workers if ns.use_mp is None: if ns.tsan: - # For TSAN tests, use number of CPU of the current process. - # - # os.process.cpu_count() is new in Python 3.13; - # mypy doesn't know about it yet - num_workers = os.process_cpu_count() # type: ignore[attr-defined] + num_workers = -1 # use the number of CPUs else: num_workers = 0 # run sequentially elif ns.use_mp <= 0: diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 74175f79128dbb..f462c7cfcda716 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -4,7 +4,6 @@ TSAN_TESTS = [ 'test_capi', 'test_code', - 'test_compileall', 'test_enum', 'test_functools', 'test_httpservers', @@ -20,11 +19,8 @@ 'test_syslog', 'test_thread', 'test_threadedtempfile', - 'test_threading', 'test_threading_local', 'test_threadsignals', - 'test_urllib2net', - 'test_weakref' ] diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt index b70dcc30d3aaf5..448dfac8005c79 100644 --- a/Tools/tsan/supressions.txt +++ b/Tools/tsan/supressions.txt @@ -1,3 +1,5 @@ ## reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions race:get_allocator_unlocked race:set_allocator_unlocked +race:mi_heap_visit_pages +race:_mi_heap_delayed_free_partial From 06607e5c11d861a619d070e9f02e4d374b5cbc1d Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 12 Mar 2024 01:53:35 +0900 Subject: [PATCH 09/12] Adjust tests --- Lib/test/libregrtest/tsan.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index f462c7cfcda716..1398355ac013ec 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -2,19 +2,17 @@ # chosen to run a thread sanitizer in a reasonable executable time. TSAN_TESTS = [ - 'test_capi', 'test_code', 'test_enum', 'test_functools', 'test_httpservers', 'test_imaplib', - 'test_import', 'test_importlib', 'test_io', 'test_logging', - 'test_queue', - 'test_smtpnet', - 'test_socketserver', + 'test_tuple', + 'test_list', + 'test_dict', 'test_ssl', 'test_syslog', 'test_thread', From 0f1899ec37105e9abb153b1a57390eedc317db76 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Wed, 13 Mar 2024 07:36:36 +0900 Subject: [PATCH 10/12] Address code review --- Doc/library/test.rst | 7 ------- Lib/test/libregrtest/main.py | 5 +---- Lib/test/libregrtest/tsan.py | 3 --- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 8865b47452c8a4..7d28f625345726 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -352,13 +352,6 @@ The :mod:`test.support` module defines the following constants: Set to the ``data`` directory within the test package. -.. data:: TSAN - - Set when tests can be skipped when they are not useful for ThreadSanitizer. - - .. versionadded:: 3.13 - - .. data:: MAX_Py_ssize_t Set to :data:`sys.maxsize` for big memory tests. diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cfee9fcfa8b56a..70f723a92eb44a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -89,10 +89,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Workers if ns.use_mp is None: - if ns.tsan: - num_workers = -1 # use the number of CPUs - else: - num_workers = 0 # run sequentially + num_workers = 0 # run sequentially elif ns.use_mp <= 0: num_workers = -1 # use the number of CPUs else: diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 1398355ac013ec..5a5675e19c7ed3 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -10,9 +10,6 @@ 'test_importlib', 'test_io', 'test_logging', - 'test_tuple', - 'test_list', - 'test_dict', 'test_ssl', 'test_syslog', 'test_thread', From ff8c92e79d07f942a1dcceb4d675b8641adb7b9a Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 13 Mar 2024 07:37:02 +0900 Subject: [PATCH 11/12] Update Lib/test/libregrtest/tsan.py Co-authored-by: Sam Gross --- Lib/test/libregrtest/tsan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 5a5675e19c7ed3..150fcec8816179 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -1,5 +1,5 @@ # Set of tests run by default if --tsan is specified. The tests below were -# chosen to run a thread sanitizer in a reasonable executable time. +# chosen because they use threads and run in a reasonable amount of time. TSAN_TESTS = [ 'test_code', From b571b9d521fe68fab93094c29ac21fd6619274a4 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Wed, 13 Mar 2024 07:42:44 +0900 Subject: [PATCH 12/12] Address code review --- Lib/test/libregrtest/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2cf63330366aed..876b1bcd2ca406 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -335,7 +335,7 @@ def _create_parser(): group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') group.add_argument('--tsan', dest='tsan', action='store_true', - help='enable TSAN test') + help='run a subset of test cases that are proper for the TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed')