pythongh-109615: Fix test_tools.test_freeze SRCDIR (python#109935)

Fix copy_source_tree() function of test_tools.test_freeze:

* Don't copy SRC_DIR/build/ anymore. This directory is modified by
  other tests running in parallel.
* Add test.support.copy_python_src_ignore().
* Use sysconfig to get the source directory.
* Use sysconfig.get_config_var() to get CONFIG_ARGS variable.

(cherry picked from commit 1512d6c)

Author: vstinner

Lib/test/support/__init__.py

@@ -2324,3 +2324,29 @@ def adjust_int_max_str_digits(max_digits):
         yield
     finally:
         sys.set_int_max_str_digits(current)
+
+_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({
+    # SRC_DIR/.git
+    '.git',
+    # ignore all __pycache__/ sub-directories
+    '__pycache__',
+})
+
+# Ignore function for shutil.copytree() to copy the Python source code.
+def copy_python_src_ignore(path, names):
+    ignored = _BASE_COPY_SRC_DIR_IGNORED_NAMES
+    if os.path.basename(path) == 'Doc':
+        ignored |= {
+            # SRC_DIR/Doc/build/
+            'build',
+            # SRC_DIR/Doc/venv/
+            'venv',
+        }
+
+    # check if we are at the root of the source code
+    elif 'Modules' in names:
+        ignored |= {
+            # SRC_DIR/build/
+            'build',
+        }
+    return ignored

Lib/test/test_support.py

@@ -7,6 +7,7 @@
 import stat
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 import time
@@ -777,6 +778,27 @@ def recursive_function(depth):
 
         #self.assertEqual(available, 2)
 
+    def test_copy_python_src_ignore(self):
+        src_dir = sysconfig.get_config_var('srcdir')
+        src_dir = os.path.abspath(src_dir)
+
+        ignored = {'.git', '__pycache__'}
+
+        # Source code directory
+        names = os.listdir(src_dir)
+        self.assertEqual(support.copy_python_src_ignore(src_dir, names),
+                         ignored | {'build'})
+
+        # Doc/ directory
+        path = os.path.join(src_dir, 'Doc')
+        self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)),
+                         ignored | {'build', 'venv'})
+
+        # An other directory
+        path = os.path.join(src_dir, 'Objects')
+        self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)),
+                         ignored)
+
     # XXX -follows a list of untested API
     # make_legacy_pyc
     # is_resource_enabled

Lib/test/test_venv.py

@@ -21,7 +21,7 @@
                           skip_if_broken_multiprocessing_synchronize, verbose,
                           requires_subprocess, is_emscripten, is_wasi,
                           requires_venv_with_pip, TEST_HOME_DIR,
-                          requires_resource)
+                          requires_resource, copy_python_src_ignore)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
@@ -563,12 +563,6 @@ def test_zippath_from_non_installed_posix(self):
                                     stdlib_zip)
         additional_pythonpath_for_non_installed = []
 
-        # gh-109748: Don't copy __pycache__/ sub-directories, because they can
-        # be modified by other Python tests running in parallel.
-        ignored_names = {'__pycache__'}
-        def ignore_pycache(src, names):
-            return ignored_names
-
         # Copy stdlib files to the non-installed python so venv can
         # correctly calculate the prefix.
         for eachpath in sys.path:
@@ -586,7 +580,7 @@ def ignore_pycache(src, names):
                         shutil.copy(fn, libdir)
                     elif os.path.isdir(fn):
                         shutil.copytree(fn, os.path.join(libdir, name),
-                                        ignore=ignore_pycache)
+                                        ignore=copy_python_src_ignore)
             else:
                 additional_pythonpath_for_non_installed.append(
                     eachpath)

Tools/freeze/test/freeze.py

@@ -1,14 +1,15 @@
 import os
 import os.path
-import re
 import shlex
 import shutil
 import subprocess
+import sysconfig
+from test import support
 
 
 TESTS_DIR = os.path.dirname(__file__)
 TOOL_ROOT = os.path.dirname(TESTS_DIR)
-SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT))
+SRCDIR = os.path.abspath(sysconfig.get_config_var('srcdir'))
 
 MAKE = shutil.which('make')
 FREEZE = os.path.join(TOOL_ROOT, 'freeze.py')
@@ -75,56 +76,17 @@ def ensure_opt(args, name, value):
 
 
 def copy_source_tree(newroot, oldroot):
-    print(f'copying the source tree into {newroot}...')
+    print(f'copying the source tree from {oldroot} to {newroot}...')
     if os.path.exists(newroot):
         if newroot == SRCDIR:
             raise Exception('this probably isn\'t what you wanted')
         shutil.rmtree(newroot)
 
-    def ignore_non_src(src, names):
-        """Turns what could be a 1000M copy into a 100M copy."""
-        # Don't copy the ~600M+ of needless git repo metadata.
-        # source only, ignore cached .pyc files.
-        subdirs_to_skip = {'.git', '__pycache__'}
-        if os.path.basename(src) == 'Doc':
-            # Another potential ~250M+ of non test related data.
-            subdirs_to_skip.add('build')
-            subdirs_to_skip.add('venv')
-        return subdirs_to_skip
-
-    shutil.copytree(oldroot, newroot, ignore=ignore_non_src)
+    shutil.copytree(oldroot, newroot, ignore=support.copy_python_src_ignore)
     if os.path.exists(os.path.join(newroot, 'Makefile')):
         _run_quiet([MAKE, 'clean'], newroot)
 
 
-def get_makefile_var(builddir, name):
-    regex = re.compile(rf'^{name} *=\s*(.*?)\s*$')
-    filename = os.path.join(builddir, 'Makefile')
-    try:
-        infile = open(filename, encoding='utf-8')
-    except FileNotFoundError:
-        return None
-    with infile:
-        for line in infile:
-            m = regex.match(line)
-            if m:
-                value, = m.groups()
-                return value or ''
-    return None
-
-
-def get_config_var(builddir, name):
-    python = os.path.join(builddir, 'python')
-    if os.path.isfile(python):
-        cmd = [python, '-c',
-               f'import sysconfig; print(sysconfig.get_config_var("{name}"))']
-        try:
-            return _run_stdout(cmd)
-        except subprocess.CalledProcessError:
-            pass
-    return get_makefile_var(builddir, name)
-
-
 ##################################
 # freezing
 
@@ -151,10 +113,8 @@ def prepare(script=None, outdir=None):
 
     # Run configure.
     print(f'configuring python in {builddir}...')
-    cmd = [
-        os.path.join(srcdir, 'configure'),
-        *shlex.split(get_config_var(srcdir, 'CONFIG_ARGS') or ''),
-    ]
+    config_args = shlex.split(sysconfig.get_config_var('CONFIG_ARGS') or '')
+    cmd = [os.path.join(srcdir, 'configure'), *config_args]
     ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache'))
     prefix = os.path.join(outdir, 'python-installation')
     ensure_opt(cmd, 'prefix', prefix)