diff --git a/news/ebe79ad8-48c0-11ea-b9de-4f2009146da0.trivial b/news/ebe79ad8-48c0-11ea-b9de-4f2009146da0.trivial new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyproject.toml b/pyproject.toml index 01fae701523..5ddc81b8967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ drop = [ "bin/", # interpreter and OS specific msgpack libs "msgpack/*.so", + # importlib-metadata carries a bunch of test data we don't need + "importlib_metadata/docs/", + "importlib_metadata/tests/", # unneeded parts of setuptools "easy_install.py", "setuptools", diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py new file mode 100644 index 00000000000..04508b3471f --- /dev/null +++ b/src/pip/_internal/metadata/__init__.py @@ -0,0 +1,28 @@ +""" +This package wraps the vendored importlib_metadata and pkg_resources to +provide a (mostly) compatible shim. + +The pkg_resources implementation is used on Python 2, otherwise we use +importlib_metadata. +""" + +import sys + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if sys.version_info >= (3, 5): + from pip._internal.metadata import _importlib_metadata as impl +else: + from pip._internal.metadata import _pkg_resources as impl + +if MYPY_CHECK_RUNNING: + from typing import Union + from pip._internal.metadata import _importlib_metadata as _i + from pip._internal.metadata import _pkg_resources as _p + + Distribution = Union[_i.Distribution, _p.Distribution] + + +get_file_lines = impl.get_file_lines + +get_metadata = impl.get_metadata diff --git a/src/pip/_internal/metadata/_importlib_metadata.py b/src/pip/_internal/metadata/_importlib_metadata.py new file mode 100644 index 00000000000..4c832c3c577 --- /dev/null +++ b/src/pip/_internal/metadata/_importlib_metadata.py @@ -0,0 +1,19 @@ +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from email.message import Message + from typing import Iterator, Optional + from pip._vendor.importlib_metadata import Distribution + + +def get_metadata(dist): + # type: (Distribution) -> Message + return dist.metadata + + +def get_file_lines(dist, name): + # type: (Distribution, str) -> Optional[Iterator[str]] + content = dist.read_text("INSTALLER") + if not content: + return None + return content.splitlines() diff --git a/src/pip/_internal/metadata/_pkg_resources.py b/src/pip/_internal/metadata/_pkg_resources.py new file mode 100644 index 00000000000..ab5b0e85db2 --- /dev/null +++ b/src/pip/_internal/metadata/_pkg_resources.py @@ -0,0 +1,52 @@ +from __future__ import absolute_import + +import logging +from email.parser import FeedParser + +from pip._vendor import pkg_resources + +from pip._internal.exceptions import NoneMetadataError +from pip._internal.utils.misc import display_path +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterator, Optional + from email.message import Message + from pip._vendor.pkg_resources import Distribution + + +logger = logging.getLogger(__name__) + + +def get_metadata(dist): + # type: (Distribution) -> Message + """ + :raises NoneMetadataError: if the distribution reports `has_metadata()` + True but `get_metadata()` returns None. + """ + metadata_name = 'METADATA' + if (isinstance(dist, pkg_resources.DistInfoDistribution) and + dist.has_metadata(metadata_name)): + metadata = dist.get_metadata(metadata_name) + elif dist.has_metadata('PKG-INFO'): + metadata_name = 'PKG-INFO' + metadata = dist.get_metadata(metadata_name) + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' + + if metadata is None: + raise NoneMetadataError(dist, metadata_name) + + feed_parser = FeedParser() + # The following line errors out if with a "NoneType" TypeError if + # passed metadata=None. + feed_parser.feed(metadata) + return feed_parser.close() + + +def get_file_lines(dist, name): + # type: (Distribution, str) -> Optional[Iterator[str]] + if not dist.has_metadata(name): + return None + return dist.get_metadata_lines(name) diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 68aa86edbf0..c17d2e4c6d6 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -1,22 +1,11 @@ -from __future__ import absolute_import - -import logging -from email.parser import FeedParser - -from pip._vendor import pkg_resources from pip._vendor.packaging import specifiers, version -from pip._internal.exceptions import NoneMetadataError -from pip._internal.utils.misc import display_path +from pip._internal.metadata import get_file_lines, get_metadata from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Optional, Tuple - from email.message import Message - from pip._vendor.pkg_resources import Distribution - - -logger = logging.getLogger(__name__) + from pip._internal.metadata import Distribution def check_requires_python(requires_python, version_info): @@ -41,35 +30,8 @@ def check_requires_python(requires_python, version_info): return python_version in requires_python_specifier -def get_metadata(dist): - # type: (Distribution) -> Message - """ - :raises NoneMetadataError: if the distribution reports `has_metadata()` - True but `get_metadata()` returns None. - """ - metadata_name = 'METADATA' - if (isinstance(dist, pkg_resources.DistInfoDistribution) and - dist.has_metadata(metadata_name)): - metadata = dist.get_metadata(metadata_name) - elif dist.has_metadata('PKG-INFO'): - metadata_name = 'PKG-INFO' - metadata = dist.get_metadata(metadata_name) - else: - logger.warning("No metadata found in %s", display_path(dist.location)) - metadata = '' - - if metadata is None: - raise NoneMetadataError(dist, metadata_name) - - feed_parser = FeedParser() - # The following line errors out if with a "NoneType" TypeError if - # passed metadata=None. - feed_parser.feed(metadata) - return feed_parser.close() - - def get_requires_python(dist): - # type: (pkg_resources.Distribution) -> Optional[str] + # type: (Distribution) -> Optional[str] """ Return the "Requires-Python" metadata for a distribution, or None if not present. @@ -87,8 +49,10 @@ def get_requires_python(dist): def get_installer(dist): # type: (Distribution) -> str - if dist.has_metadata('INSTALLER'): - for line in dist.get_metadata_lines('INSTALLER'): - if line.strip(): - return line.strip() + lines = get_file_lines(dist, 'INSTALLER') + if lines is None: + return '' + for line in lines: + if line.strip(): + return line.strip() return '' diff --git a/src/pip/_vendor/importlib_metadata/__init__.py b/src/pip/_vendor/importlib_metadata/__init__.py new file mode 100644 index 00000000000..50fcea99225 --- /dev/null +++ b/src/pip/_vendor/importlib_metadata/__init__.py @@ -0,0 +1,588 @@ +from __future__ import unicode_literals, absolute_import + +import io +import os +import re +import abc +import csv +import sys +from pip._vendor import zipp +import operator +import functools +import itertools +import posixpath +import collections + +from ._compat import ( + install, + NullFinder, + ConfigParser, + suppress, + map, + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + pathlib, + ModuleNotFoundError, + MetaPathFinder, + email_message_from_string, + PyPy_repr, + ) +from importlib import import_module +from itertools import starmap + + +__metaclass__ = type + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'requires', + 'version', + ] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + +class EntryPoint( + PyPy_repr, + collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + `_ + for more information. + """ + + pattern = re.compile( + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+))?\s*' + r'(?P\[.*\])?\s*$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def extras(self): + match = self.pattern.match(self.value) + return list(re.finditer(r'\w+', match.group('extras') or '')) + + @classmethod + def _from_config(cls, config): + return [ + cls(name, value, group) + for group in config.sections() + for name, value in config.items(group) + ] + + @classmethod + def _from_text(cls, text): + config = ConfigParser(delimiters='=') + # case sensitive: https://stackoverflow.com/q/1611799/812183 + config.optionxform = str + try: + config.read_string(text) + except AttributeError: # pragma: nocover + # Python 2 has no read_string + config.readfp(io.StringIO(text)) + return EntryPoint._from_config(config) + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints easily. + """ + return iter((self.name, self)) + + def __reduce__(self): + return ( + self.__class__, + (self.name, self.value, self.group), + ) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return ''.format(self.mode, self.value) + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(DistributionFinder.Context(name=name)) + dist = next(dists, None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) + for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) + for finder in sys.meta_path + ) + return filter(None, declared) + + @property + def metadata(self): + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + return email_message_from_string(text) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoint._from_text(self.read_text('entry_points.txt')) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is + missing. + Result may be empty if the metadata exists but is empty. + """ + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and list(starmap(make_file, csv.reader(file_lines))) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return source and self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + section_pairs = cls._read_sections(source.splitlines()) + sections = { + section: list(map(operator.itemgetter('line'), results)) + for section, results in + itertools.groupby(section_pairs, operator.itemgetter('section')) + } + return cls._convert_egg_info_reqs_to_simple_reqs(sections) + + @staticmethod + def _read_sections(lines): + section = None + for line in filter(None, lines): + section_match = re.match(r'\[(.*)\]$', line) + if section_match: + section = section_match.group(1) + continue + yield locals() + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + def make_condition(name): + return name and 'extra == "{name}"'.format(name=name) + + def parse_condition(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = '({markers})'.format(markers=markers) + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + for section, deps in sections.items(): + for dep in deps: + yield dep + parse_condition(section) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The path that a distribution finder should search. + + Typically refers to Python package paths and defaults + to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + """ + + def __init__(self, root): + self.root = root + self.base = os.path.basename(root).lower() + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipp.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return ( + posixpath.split(child)[0] + for child in names + ) + + def is_egg(self, search): + base = self.base + return ( + base == search.versionless_egg_name + or base.startswith(search.prefix) + and base.endswith('.egg')) + + def search(self, name): + for child in self.children(): + n_low = child.lower() + if (n_low in name.exact_matches + or n_low.startswith(name.prefix) + and n_low.endswith(name.suffixes) + # legacy case: + or self.is_egg(name) and n_low == 'egg-info'): + yield self.joinpath(child) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + normalized = '' + prefix = '' + suffixes = '.dist-info', '.egg-info' + exact_matches = [''][:0] + versionless_egg_name = '' + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = name.lower().replace('-', '_') + self.prefix = self.normalized + '-' + self.exact_matches = [ + self.normalized + suffix for suffix in self.suffixes] + self.versionless_egg_name = self.normalized + '.egg' + + +@install +class MetadataPathFinder(NullFinder, DistributionFinder): + """A degenerate finder for distribution packages on the file system. + + This finder supplies only a find_distributions() method for versions + of Python that do not have a PathFinder find_distributions(). + """ + + def find_distributions(self, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = self._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + return itertools.chain.from_iterable( + path.search(Prepared(name)) + for path in map(FastPath, paths) + ) + + +class PathDistribution(Distribution): + def __init__(self, path): + """Construct a distribution from a path to the metadata directory. + + :param path: A pathlib.Path or similar object supporting + .joinpath(), __div__, .parent, and .read_text(). + """ + self._path = path + + def read_text(self, filename): + with suppress(FileNotFoundError, IsADirectoryError, KeyError, + NotADirectoryError, PermissionError): + return self._path.joinpath(filename).read_text(encoding='utf-8') + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name): + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: An email.Message containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +def entry_points(): + """Return EntryPoint objects for all installed packages. + + :return: EntryPoint objects for all installed packages. + """ + eps = itertools.chain.from_iterable( + dist.entry_points for dist in distributions()) + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return { + group: tuple(eps) + for group, eps in grouped + } + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires diff --git a/src/pip/_vendor/importlib_metadata/_compat.py b/src/pip/_vendor/importlib_metadata/_compat.py new file mode 100644 index 00000000000..ed752b39667 --- /dev/null +++ b/src/pip/_vendor/importlib_metadata/_compat.py @@ -0,0 +1,131 @@ +from __future__ import absolute_import + +import io +import abc +import sys +import email + + +if sys.version_info > (3,): # pragma: nocover + import builtins + from configparser import ConfigParser + from contextlib import suppress + FileNotFoundError = builtins.FileNotFoundError + IsADirectoryError = builtins.IsADirectoryError + NotADirectoryError = builtins.NotADirectoryError + PermissionError = builtins.PermissionError + map = builtins.map +else: # pragma: nocover + from backports.configparser import ConfigParser + from itertools import imap as map # type: ignore + from contextlib2 import suppress # noqa + FileNotFoundError = IOError, OSError + IsADirectoryError = IOError, OSError + NotADirectoryError = IOError, OSError + PermissionError = IOError, OSError + +if sys.version_info > (3, 5): # pragma: nocover + import pathlib +else: # pragma: nocover + import pathlib2 as pathlib + +try: + ModuleNotFoundError = builtins.FileNotFoundError +except (NameError, AttributeError): # pragma: nocover + ModuleNotFoundError = ImportError # type: ignore + + +if sys.version_info >= (3,): # pragma: nocover + from importlib.abc import MetaPathFinder +else: # pragma: nocover + class MetaPathFinder(object): + __metaclass__ = abc.ABCMeta + + +__metaclass__ = type +__all__ = [ + 'install', 'NullFinder', 'MetaPathFinder', 'ModuleNotFoundError', + 'pathlib', 'ConfigParser', 'map', 'suppress', 'FileNotFoundError', + 'NotADirectoryError', 'email_message_from_string', + ] + + +def install(cls): + """ + Class decorator for installation on sys.meta_path. + + Adds the backport DistributionFinder to sys.meta_path and + attempts to disable the finder functionality of the stdlib + DistributionFinder. + """ + sys.meta_path.append(cls()) + disable_stdlib_finder() + return cls + + +def disable_stdlib_finder(): + """ + Give the backport primacy for discovering path-based distributions + by monkey-patching the stdlib O_O. + + See #91 for more background for rationale on this sketchy + behavior. + """ + def matches(finder): + return ( + getattr(finder, '__module__', None) == '_frozen_importlib_external' + and hasattr(finder, 'find_distributions') + ) + for finder in filter(matches, sys.meta_path): # pragma: nocover + del finder.find_distributions + + +class NullFinder: + """ + A "Finder" (aka "MetaClassFinder") that never finds any modules, + but may find distributions. + """ + @staticmethod + def find_spec(*args, **kwargs): + return None + + # In Python 2, the import system requires finders + # to have a find_module() method, but this usage + # is deprecated in Python 3 in favor of find_spec(). + # For the purposes of this finder (i.e. being present + # on sys.meta_path but having no other import + # system functionality), the two methods are identical. + find_module = find_spec + + +def py2_message_from_string(text): # nocoverpy3 + # Work around https://bugs.python.org/issue25545 where + # email.message_from_string cannot handle Unicode on Python 2. + io_buffer = io.StringIO(text) + return email.message_from_file(io_buffer) + + +email_message_from_string = ( + py2_message_from_string + if sys.version_info < (3,) else + email.message_from_string + ) + + +class PyPy_repr: + """ + Override repr for EntryPoint objects on PyPy to avoid __iter__ access. + Ref #97, #102. + """ + affected = hasattr(sys, 'pypy_version_info') + + def __compat_repr__(self): # pragma: nocover + def make_param(name): + value = getattr(self, name) + return '{name}={value!r}'.format(**locals()) + params = ', '.join(map(make_param, self._fields)) + return 'EntryPoint({params})'.format(**locals()) + + if affected: # pragma: nocover + __repr__ = __compat_repr__ + del affected diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index cbc2830ac09..3fa49e859c2 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -5,6 +5,9 @@ contextlib2==0.6.0 distlib==0.3.0 distro==1.4.0 html5lib==1.0.1 +importlib-metadata==1.5.0 + zipp==2.1.0 + # Don't include Python 2 dependencies; we use pkg_resources there. ipaddress==1.0.23 # Only needed on 2.6 and 2.7 msgpack==0.6.2 packaging==20.1 diff --git a/src/pip/_vendor/zipp.py b/src/pip/_vendor/zipp.py new file mode 100644 index 00000000000..46165daca07 --- /dev/null +++ b/src/pip/_vendor/zipp.py @@ -0,0 +1,260 @@ +import io +import posixpath +import zipfile +import functools +import itertools +import contextlib +from collections import OrderedDict + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + """ + path = path.rstrip(posixpath.sep) + while path and path != posixpath.sep: + yield path + path, tail = posixpath.split(path) + + +class CompleteDirs(zipfile.ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + # Deduplicate entries in original order + implied_dirs = OrderedDict.fromkeys( + p + posixpath.sep for p in parents + # Cast names to a set for O(1) lookups + if p + posixpath.sep not in set(names) + ) + return implied_dirs + + def namelist(self): + names = super(CompleteDirs, self).namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, zipfile.ZipFile): + return cls(_pathlib_compat(source)) + + # Only allow for FastPath when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + res = cls.__new__(cls) + vars(res).update(vars(source)) + return res + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super(FastLookup, self).namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super(FastLookup, self)._name_set() + return self.__lookup + + +def _pathlib_compat(path): + """ + For path-like objects, convert to a filename for compatibility + on Python 3.6.1 and earlier. + """ + try: + return path.__fspath__() + except AttributeError: + return str(path) + + +class Path: + """ + A pathlib-compatible interface for zip files. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = zipfile.ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> root = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = root.iterdir() + >>> a + Path('abcde.zip', 'a.txt') + >>> b + Path('abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text() + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> str(c) + 'abcde.zip/b/c.txt' + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + self.root = FastLookup.make(root) + self.at = at + + @property + def open(self): + return functools.partial(self.root.open, self.at) + + @property + def name(self): + return posixpath.basename(self.at.rstrip("/")) + + def read_text(self, *args, **kwargs): + with self.open() as strm: + return io.TextIOWrapper(strm, *args, **kwargs).read() + + def read_bytes(self): + with self.open() as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return Path(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return not self.is_dir() + + def exists(self): + return self.at in self.root._name_set() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self.root.namelist()) + return filter(self._is_child, subs) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, add): + next = posixpath.join(self.at, _pathlib_compat(add)) + return self._next(self.root.resolve_dir(next)) + + __truediv__ = joinpath + + @property + def parent(self): + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) diff --git a/tools/automation/vendoring/patches/importlib-metadata.patch b/tools/automation/vendoring/patches/importlib-metadata.patch new file mode 100644 index 00000000000..33c12dc9b6e --- /dev/null +++ b/tools/automation/vendoring/patches/importlib-metadata.patch @@ -0,0 +1,20 @@ +diff --git a/src/pip/_vendor/importlib_metadata/__init__.py b/src/pip/_vendor/importlib_metadata/__init__.py +index 089fca97..50fcea99 100644 +--- a/src/pip/_vendor/importlib_metadata/__init__.py ++++ b/src/pip/_vendor/importlib_metadata/__init__.py +@@ -6,7 +6,7 @@ import re + import abc + import csv + import sys +-import zipp ++from pip._vendor import zipp + import operator + import functools + import itertools +@@ -586,6 +586,3 @@ def requires(distribution_name): + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires +- +- +-__version__ = version(__name__)