Remove dependency on pkg_resources from setuptools (#1536)

Avoid using `importlib.util.find_spec()` to avoid actually importing parent packages.

Author: jacobtylerwalls

ChangeLog

@@ -15,6 +15,10 @@ Release date: TBA
 
   Closes #1512
 
+* Remove dependency on ``pkg_resources`` from ``setuptools``.
+
+  Closes #1103
+
 * Allowed ``AstroidManager.clear_cache`` to reload necessary brain plugins.
 
 * Fixed incorrect inferences after rebuilding the builtins module, e.g. by calling

astroid/interpreter/_import/spec.py

@@ -10,6 +10,7 @@
 import importlib.machinery
 import importlib.util
 import os
+import pathlib
 import sys
 import zipimport
 from collections.abc import Sequence
@@ -147,7 +148,7 @@ def contribute_to_path(self, spec, processed):
             # Builtin.
             return None
 
-        if _is_setuptools_namespace(spec.location):
+        if _is_setuptools_namespace(Path(spec.location)):
             # extend_path is called, search sys.path for module/packages
             # of this name see pkgutil.extend_path documentation
             path = [
@@ -179,7 +180,7 @@ def contribute_to_path(self, spec, processed):
 
 
 class ExplicitNamespacePackageFinder(ImportlibFinder):
-    """A finder for the explicit namespace packages, generated through pkg_resources."""
+    """A finder for the explicit namespace packages."""
 
     def find_module(self, modname, module_parts, processed, submodule_path):
         if processed:
@@ -253,12 +254,12 @@ def contribute_to_path(self, spec, processed):
 )
 
 
-def _is_setuptools_namespace(location):
+def _is_setuptools_namespace(location: pathlib.Path) -> bool:
     try:
-        with open(os.path.join(location, "__init__.py"), "rb") as stream:
+        with open(location / "__init__.py", "rb") as stream:
             data = stream.read(4096)
     except OSError:
-        return None
+        return False
     else:
         extend_path = b"pkgutil" in data and b"extend_path" in data
         declare_namespace = (

astroid/interpreter/_import/util.py

@@ -2,15 +2,35 @@
 # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE
 # Copyright (c) https://github.com/PyCQA/astroid/blob/main/CONTRIBUTORS.txt
 
-try:
-    import pkg_resources
-except ImportError:
-    pkg_resources = None  # type: ignore[assignment]
+import sys
+from functools import lru_cache
+from importlib.util import _find_spec_from_path
 
 
-def is_namespace(modname):
-    return (
-        pkg_resources is not None
-        and hasattr(pkg_resources, "_namespace_packages")
-        and modname in pkg_resources._namespace_packages
-    )
+@lru_cache(maxsize=4096)
+def is_namespace(modname: str) -> bool:
+    if modname in sys.builtin_module_names:
+        return False
+
+    found_spec = None
+
+    # find_spec() attempts to import parent packages when given dotted paths.
+    # That's unacceptable here, so we fallback to _find_spec_from_path(), which does
+    # not, but requires instead that each single parent ('astroid', 'nodes', etc.)
+    # be specced from left to right.
+    processed_components = []
+    last_parent = None
+    for component in modname.split("."):
+        processed_components.append(component)
+        working_modname = ".".join(processed_components)
+        try:
+            found_spec = _find_spec_from_path(working_modname, last_parent)
+        except ValueError:
+            # executed .pth files may not have __spec__
+            return True
+        last_parent = working_modname
+
+    if found_spec is None:
+        return False
+
+    return found_spec.origin is None

astroid/manager.py

@@ -18,7 +18,7 @@
 
 from astroid.const import BRAIN_MODULES_DIRECTORY
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
-from astroid.interpreter._import import spec
+from astroid.interpreter._import import spec, util
 from astroid.modutils import (
     NoSourceFile,
     _cache_normalize_path_,
@@ -384,6 +384,7 @@ def clear_cache(self) -> None:
         for lru_cache in (
             LookupMixIn.lookup,
             _cache_normalize_path_,
+            util.is_namespace,
             ObjectModel.attributes,
         ):
             lru_cache.cache_clear()

setup.cfg

@@ -39,7 +39,6 @@ packages = find:
 install_requires =
     lazy_object_proxy>=1.4.0
     wrapt>=1.11,<2
-    setuptools>=20.0
     typed-ast>=1.4.0,<2.0;implementation_name=="cpython" and python_version<"3.8"
     typing-extensions>=3.10;python_version<"3.10"
 python_requires = >=3.7.2

tests/unittest_manager.py

@@ -10,12 +10,11 @@
 from collections.abc import Iterator
 from contextlib import contextmanager
 
-import pkg_resources
-
 import astroid
 from astroid import manager, test_utils
 from astroid.const import IS_JYTHON
 from astroid.exceptions import AstroidBuildingError, AstroidImportError
+from astroid.interpreter._import import util
 from astroid.modutils import is_standard_module
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef
@@ -111,6 +110,16 @@ def test_ast_from_namespace_pkgutil(self) -> None:
     def test_ast_from_namespace_pkg_resources(self) -> None:
         self._test_ast_from_old_namespace_package_protocol("pkg_resources")
 
+    def test_identify_old_namespace_package_protocol(self) -> None:
+        # Like the above cases, this package follows the old namespace package protocol
+        # astroid currently assumes such packages are in sys.modules, so import it
+        # pylint: disable-next=import-outside-toplevel
+        import tests.testdata.python3.data.path_pkg_resources_1.package.foo as _  # noqa
+
+        self.assertTrue(
+            util.is_namespace("tests.testdata.python3.data.path_pkg_resources_1")
+        )
+
     def test_implicit_namespace_package(self) -> None:
         data_dir = os.path.dirname(resources.find("data/namespace_pep_420"))
         contribute = os.path.join(data_dir, "contribute_to_namespace")
@@ -131,7 +140,6 @@ def test_implicit_namespace_package(self) -> None:
     def test_namespace_package_pth_support(self) -> None:
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = []
 
         try:
             module = self.manager.ast_from_module_name("foogle.fax")
@@ -141,18 +149,14 @@ def test_namespace_package_pth_support(self) -> None:
             with self.assertRaises(AstroidImportError):
                 self.manager.ast_from_module_name("foogle.moogle")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def test_nested_namespace_import(self) -> None:
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = ["foogle.crank"]
-        pkg_resources._namespace_packages["foogle.crank"] = []
         try:
             self.manager.ast_from_module_name("foogle.crank")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def test_namespace_and_file_mismatch(self) -> None:
@@ -161,12 +165,10 @@ def test_namespace_and_file_mismatch(self) -> None:
         self.assertEqual(ast.name, "unittest")
         pth = "foogle_fax-0.12.5-py2.7-nspkg.pth"
         site.addpackage(resources.RESOURCE_PATH, pth, [])
-        pkg_resources._namespace_packages["foogle"] = []
         try:
             with self.assertRaises(AstroidImportError):
                 self.manager.ast_from_module_name("unittest.foogle.fax")
         finally:
-            del pkg_resources._namespace_packages["foogle"]
             sys.modules.pop("foogle")
 
     def _test_ast_from_zip(self, archive: str) -> None:
@@ -323,6 +325,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
         lrus = (
             astroid.nodes.node_classes.LookupMixIn.lookup,
             astroid.modutils._cache_normalize_path_,
+            util.is_namespace,
             astroid.interpreter.objectmodel.ObjectModel.attributes,
         )
 
@@ -332,6 +335,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None:
         # Generate some hits and misses
         ClassDef().lookup("garbage")
         is_standard_module("unittest", std_path=["garbage_path"])
+        util.is_namespace("unittest")
         astroid.interpreter.objectmodel.ObjectModel().attributes()
 
         # Did the hits or misses actually happen?