diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44f2c756c886..e5285416405c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -100,7 +100,7 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - run: pip install -r requirements-tests.txt - - run: python ./tests/regr_test.py --all --quiet + - run: python ./tests/regr_test.py --all --verbosity QUIET pyright: name: Test typeshed with pyright diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 6eee3213a037..55f846b3d8f1 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -274,7 +274,7 @@ def add_third_party_files( seen_dists.add(distribution) stubs_dir = Path("stubs") - dependencies = get_recursive_requirements(distribution) + dependencies = get_recursive_requirements(distribution).typeshed_pkgs for dependency in dependencies: if dependency in seen_dists: diff --git a/tests/regr_test.py b/tests/regr_test.py index 87be4f8c8d7a..785a971c076a 100644 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -10,15 +10,19 @@ import subprocess import sys import tempfile +from enum import IntEnum from itertools import product from pathlib import Path from typing_extensions import TypeAlias from utils import ( PackageInfo, + VenvInfo, colored, get_all_testcase_directories, + get_mypy_req, get_recursive_requirements, + make_venv, print_error, print_success_msg, testcase_dir_from_package_name, @@ -26,6 +30,10 @@ ReturnCode: TypeAlias = int +TEST_CASES = "test_cases" +VENV_DIR = ".venv" +TYPESHED = "typeshed" + SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"] SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"] @@ -34,7 +42,7 @@ def package_with_test_cases(package_name: str) -> PackageInfo: """Helper function for argument-parsing""" if package_name == "stdlib": - return PackageInfo("stdlib", Path("test_cases")) + return PackageInfo("stdlib", Path(TEST_CASES)) test_case_dir = testcase_dir_from_package_name(package_name) if test_case_dir.is_dir(): if not os.listdir(test_case_dir): @@ -43,6 +51,12 @@ def package_with_test_cases(package_name: str) -> PackageInfo: raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!") +class Verbosity(IntEnum): + QUIET = 0 + NORMAL = 1 + VERBOSE = 2 + + parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs") parser.add_argument( "packages_to_test", @@ -59,7 +73,12 @@ def package_with_test_cases(package_name: str) -> PackageInfo: "Note that this cannot be specified if --platform and/or --python-version are specified." ), ) -parser.add_argument("--quiet", action="store_true", help="Print less output to the terminal") +parser.add_argument( + "--verbosity", + choices=[member.name for member in Verbosity], + default=Verbosity.NORMAL.name, + help="Control how much output to print to the terminal", +) parser.add_argument( "--platform", dest="platforms_to_test", @@ -85,16 +104,64 @@ def package_with_test_cases(package_name: str) -> PackageInfo: ) -def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode: - package_name, test_case_directory = package - is_stdlib = package_name == "stdlib" +def verbose_log(msg: str) -> None: + print(colored("\n" + msg, "blue")) - msg = f"Running mypy --platform {platform} --python-version {version} on the " - msg += "standard library test cases..." if is_stdlib else f"test cases for {package_name!r}..." - if not quiet: - print(msg, end=" ") - # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083 +def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path, verbosity: Verbosity) -> None: + if verbosity is verbosity.VERBOSE: + verbose_log(f"Setting up testcase dir in {tempdir}") + # --warn-unused-ignores doesn't work for files inside typeshed. + # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory, + # and run the test cases inside of that. + shutil.copytree(package.test_case_directory, new_test_case_dir) + if package.is_stdlib: + return + + # HACK: we want to run these test cases in an isolated environment -- + # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml + # (and all stub packages required by those stub packages, etc. etc.), + # but none of the other stubs in typeshed. + # + # The best way of doing that without stopping --warn-unused-ignore from working + # seems to be to create a "new typeshed" directory in a tempdir + # that has only the required stubs copied over. + new_typeshed = tempdir / TYPESHED + new_typeshed.mkdir() + shutil.copytree(Path("stdlib"), new_typeshed / "stdlib") + requirements = get_recursive_requirements(package.name) + # mypy refuses to consider a directory a "valid typeshed directory" + # unless there's a stubs/mypy-extensions path inside it, + # so add that to the list of stubs to copy over to the new directory + for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}: + shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement) + + if requirements.external_pkgs: + if verbosity is Verbosity.VERBOSE: + verbose_log(f"Setting up venv in {tempdir / VENV_DIR}") + pip_exe = make_venv(tempdir / VENV_DIR).pip_exe + pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs] + if verbosity is Verbosity.VERBOSE: + verbose_log(f"{pip_command=}") + try: + subprocess.run(pip_command, check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise + + +def run_testcases( + package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity +) -> subprocess.CompletedProcess[str]: + env_vars = dict(os.environ) + new_test_case_dir = tempdir / TEST_CASES + testcasedir_already_setup = new_test_case_dir.exists() and new_test_case_dir.is_dir() + + if not testcasedir_already_setup: + setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir, verbosity=verbosity) + + # "--enable-error-code ignore-without-code" is purposefully ommited. + # See https://github.com/python/typeshed/pull/8083 flags = [ "--python-version", version, @@ -103,67 +170,70 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q "--no-error-summary", "--platform", platform, - "--no-site-packages", "--strict", "--pretty", ] - # --warn-unused-ignores doesn't work for files inside typeshed. - # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory. - with tempfile.TemporaryDirectory() as td: - td_path = Path(td) - new_test_case_dir = td_path / "test_cases" - shutil.copytree(test_case_directory, new_test_case_dir) - env_vars = dict(os.environ) - if is_stdlib: - flags.extend(["--custom-typeshed-dir", str(Path(__file__).parent.parent)]) + if package.is_stdlib: + python_exe = sys.executable + custom_typeshed = Path(__file__).parent.parent + flags.append("--no-site-packages") + else: + custom_typeshed = tempdir / TYPESHED + env_vars["MYPYPATH"] = os.pathsep.join(map(str, custom_typeshed.glob("stubs/*"))) + has_non_types_dependencies = (tempdir / VENV_DIR).exists() + if has_non_types_dependencies: + python_exe = VenvInfo.of_existing_venv(tempdir / VENV_DIR).python_exe else: - # HACK: we want to run these test cases in an isolated environment -- - # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml - # (and all stub packages required by those stub packages, etc. etc.), - # but none of the other stubs in typeshed. - # - # The best way of doing that without stopping --warn-unused-ignore from working - # seems to be to create a "new typeshed" directory in a tempdir - # that has only the required stubs copied over. - new_typeshed = td_path / "typeshed" - os.mkdir(new_typeshed) - shutil.copytree(Path("stdlib"), new_typeshed / "stdlib") - requirements = get_recursive_requirements(package_name) - # mypy refuses to consider a directory a "valid typeshed directory" - # unless there's a stubs/mypy-extensions path inside it, - # so add that to the list of stubs to copy over to the new directory - for requirement in requirements + ["mypy-extensions"]: - shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement) - env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*"))) - flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")]) - - # If the test-case filename ends with -py39, - # only run the test if --python-version was set to 3.9 or higher (for example) - for path in new_test_case_dir.rglob("*.py"): - if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem): - minor_version_required = int(match[1]) - assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS - if minor_version_required <= int(version.split(".")[1]): - flags.append(str(path)) - else: - flags.append(str(path)) - - result = subprocess.run([sys.executable, "-m", "mypy", *flags], capture_output=True, env=env_vars) + python_exe = sys.executable + flags.append("--no-site-packages") + + flags.extend(["--custom-typeshed-dir", str(custom_typeshed)]) + + # If the test-case filename ends with -py39, + # only run the test if --python-version was set to 3.9 or higher (for example) + for path in new_test_case_dir.rglob("*.py"): + if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem): + minor_version_required = int(match[1]) + assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS + python_minor_version = int(version.split(".")[1]) + if minor_version_required > python_minor_version: + continue + flags.append(str(path)) + + mypy_command = [python_exe, "-m", "mypy"] + flags + if verbosity is Verbosity.VERBOSE: + verbose_log(f"{mypy_command=}") + if "MYPYPATH" in env_vars: + verbose_log(f"{env_vars['MYPYPATH']=}") + else: + verbose_log("MYPYPATH not set") + return subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) + + +def test_testcase_directory( + package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path +) -> ReturnCode: + msg = f"Running mypy --platform {platform} --python-version {version} on the " + msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..." + if verbosity > Verbosity.QUIET: + print(msg, end=" ", flush=True) + + result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity) if result.returncode: - if quiet: + if verbosity > Verbosity.QUIET: # We'll already have printed this if --quiet wasn't passed. - # If--quiet was passed, only print this if there were errors. + # If --quiet was passed, only print this if there were errors. # If there are errors, the output is inscrutable if this isn't printed. print(msg, end=" ") print_error("failure\n") - replacements = (str(new_test_case_dir), str(test_case_directory)) + replacements = (str(tempdir / TEST_CASES), str(package.test_case_directory)) if result.stderr: - print_error(result.stderr.decode(), fix_path=replacements) + print_error(result.stderr, fix_path=replacements) if result.stdout: - print_error(result.stdout.decode(), fix_path=replacements) - elif not quiet: + print_error(result.stdout, fix_path=replacements) + elif verbosity > Verbosity.QUIET: print_success_msg() return result.returncode @@ -172,6 +242,7 @@ def main() -> ReturnCode: args = parser.parse_args() testcase_directories = args.packages_to_test or get_all_testcase_directories() + verbosity = Verbosity[args.verbosity] if args.all: if args.platforms_to_test: parser.error("Cannot specify both --platform and --all") @@ -183,8 +254,12 @@ def main() -> ReturnCode: versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"] code = 0 - for platform, version, directory in product(platforms_to_test, versions_to_test, testcase_directories): - code = max(code, test_testcase_directory(directory, version, platform, args.quiet)) + for testcase_dir in testcase_directories: + with tempfile.TemporaryDirectory() as td: + tempdir = Path(td) + for platform, version in product(platforms_to_test, versions_to_test): + this_code = test_testcase_directory(testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) + code = max(code, this_code) if code: print_error("\nTest completed with errors") else: diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index d9b612396f82..98e8015a30ee 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -4,23 +4,15 @@ from __future__ import annotations import argparse -import functools import os import subprocess import sys import tempfile -import venv from pathlib import Path from typing import NoReturn import tomli -from utils import colored, print_error, print_success_msg - - -@functools.lru_cache() -def get_mypy_req() -> str: - with open("requirements-tests.txt", encoding="UTF-8") as f: - return next(line.strip() for line in f if "mypy" in line) +from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: bool = False) -> bool: @@ -44,25 +36,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: boo with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) try: - venv.create(venv_dir, with_pip=True, clear=True) - except subprocess.CalledProcessError as e: - if "ensurepip" in e.cmd: - print_error("fail") - print_error( - "stubtest requires a Python installation with ensurepip. " - "If on Linux, you may need to install the python3-venv package." - ) + pip_exe, python_exe = make_venv(venv_dir) + except Exception: + print_error("fail") raise - - if sys.platform == "win32": - pip = venv_dir / "Scripts" / "pip.exe" - python = venv_dir / "Scripts" / "python.exe" - else: - pip = venv_dir / "bin" / "pip" - python = venv_dir / "bin" / "python" - - pip_exe, python_exe = str(pip), str(python) - dist_version = metadata["version"] extras = stubtest_meta.get("extras", []) assert isinstance(dist_version, str) diff --git a/tests/utils.py b/tests/utils.py index 5c8c9aec5a74..cdfba98000b9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,13 +4,18 @@ import os import re +import subprocess +import sys +import venv +from collections.abc import Mapping from functools import cache -from itertools import filterfalse from pathlib import Path from typing import NamedTuple +from typing_extensions import Annotated import pathspec # type: ignore[import] import tomli +from packaging.requirements import Requirement # Used to install system-wide packages for different OS types: METADATA_MAPPING = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"} @@ -45,25 +50,112 @@ def print_success_msg() -> None: # ==================================================================== +class PackageDependencies(NamedTuple): + typeshed_pkgs: tuple[str, ...] + external_pkgs: tuple[str, ...] + + +@cache +def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]: + stub_name_map = {} + for typeshed_name in os.listdir("stubs"): + with Path("stubs", typeshed_name, "METADATA.toml").open("rb") as f: + pypi_name = tomli.load(f).get("stub_distribution", f"types-{typeshed_name}") + assert isinstance(pypi_name, str) + stub_name_map[pypi_name] = typeshed_name + return stub_name_map + + @cache -def read_dependencies(distribution: str) -> tuple[str, ...]: +def read_dependencies(distribution: str) -> PackageDependencies: + """Read the dependencies listed in a METADATA.toml file for a stubs package. + + Once the dependencies have been read, + determine which dependencies are typeshed-internal dependencies, + and which dependencies are external (non-types) dependencies. + For typeshed dependencies, translate the "dependency name" into the "package name"; + for external dependencies, leave them as they are in the METADATA.toml file. + + Note that this function may consider things to be typeshed stubs + even if they haven't yet been uploaded to PyPI. + If a typeshed stub is removed, this function will consider it to be an external dependency. + """ + pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping() with Path("stubs", distribution, "METADATA.toml").open("rb") as f: - data = tomli.load(f) - requires = data.get("requires", []) - assert isinstance(requires, list) - dependencies = [] - for dependency in requires: + dependencies = tomli.load(f).get("requires", []) + assert isinstance(dependencies, list) + typeshed, external = [], [] + for dependency in dependencies: assert isinstance(dependency, str) - assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}" - dependencies.append(dependency[6:].split("<")[0]) - return tuple(dependencies) + maybe_typeshed_dependency = Requirement(dependency).name + if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping: + typeshed.append(pypi_name_to_typeshed_name_mapping[maybe_typeshed_dependency]) + else: + external.append(dependency) + return PackageDependencies(tuple(typeshed), tuple(external)) -def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> list[str]: - seen = seen if seen is not None else {package_name} - for dependency in filterfalse(seen.__contains__, read_dependencies(package_name)): - seen.update(get_recursive_requirements(dependency, seen)) - return sorted(seen | {package_name}) +@cache +def get_recursive_requirements(package_name: str) -> PackageDependencies: + """Recursively gather dependencies for a single stubs package. + + For example, if the stubs for `caldav` + declare a dependency on typeshed's stubs for `requests`, + and the stubs for requests declare a dependency on typeshed's stubs for `urllib3`, + `get_recursive_requirements("caldav")` will determine that the stubs for `caldav` + have both `requests` and `urllib3` as typeshed-internal dependencies. + """ + typeshed: set[str] = set() + external: set[str] = set() + non_recursive_requirements = read_dependencies(package_name) + typeshed.update(non_recursive_requirements.typeshed_pkgs) + external.update(non_recursive_requirements.external_pkgs) + for pkg in non_recursive_requirements.typeshed_pkgs: + reqs = get_recursive_requirements(pkg) + typeshed.update(reqs.typeshed_pkgs) + external.update(reqs.external_pkgs) + return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external))) + + +# ==================================================================== +# Dynamic venv creation +# ==================================================================== + + +class VenvInfo(NamedTuple): + pip_exe: Annotated[str, "A path to the venv's pip executable"] + python_exe: Annotated[str, "A path to the venv's python executable"] + + @staticmethod + def of_existing_venv(venv_dir: Path) -> VenvInfo: + if sys.platform == "win32": + pip = venv_dir / "Scripts" / "pip.exe" + python = venv_dir / "Scripts" / "python.exe" + else: + pip = venv_dir / "bin" / "pip" + python = venv_dir / "bin" / "python" + + return VenvInfo(str(pip), str(python)) + + +def make_venv(venv_dir: Path) -> VenvInfo: + try: + venv.create(venv_dir, with_pip=True, clear=True) + except subprocess.CalledProcessError as e: + if "ensurepip" in e.cmd: + print_error( + "stubtest requires a Python installation with ensurepip. " + "If on Linux, you may need to install the python3-venv package." + ) + raise + + return VenvInfo.of_existing_venv(venv_dir) + + +@cache +def get_mypy_req() -> str: + with open("requirements-tests.txt", encoding="UTF-8") as f: + return next(line.strip() for line in f if "mypy" in line) # ==================================================================== @@ -83,6 +175,10 @@ class PackageInfo(NamedTuple): name: str test_case_directory: Path + @property + def is_stdlib(self) -> bool: + return self.name == "stdlib" + def testcase_dir_from_package_name(package_name: str) -> Path: return Path("stubs", package_name, "@tests/test_cases")