Implement "lazy sequence" to avoid Internet

find_matches() is modified to return a special type that implements
the sequence protocol (instead of a plain list). This special sequence
type tries to use the installed candidate as the first element if
possible, and only access indexes when the installed candidate is
considered unsatisfactory.

Author: uranusjr

src/pip/_internal/resolution/resolvelib/factory.py

@@ -1,7 +1,5 @@
-import collections
 import logging
 
-from pip._vendor import six
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.exceptions import (
@@ -30,6 +28,7 @@
     LinkCandidate,
     RequiresPythonCandidate,
 )
+from .found_candidates import FoundCandidates
 from .requirements import (
     ExplicitRequirement,
     RequiresPythonRequirement,
@@ -41,6 +40,7 @@
         Dict,
         FrozenSet,
         Iterable,
+        Iterator,
         List,
         Optional,
         Sequence,
@@ -100,7 +100,7 @@ def __init__(
         if not ignore_installed:
             self._installed_dists = {
                 canonicalize_name(dist.project_name): dist
-                for dist in get_installed_distributions()
+                for dist in get_installed_distributions(local_only=False)
             }
         else:
             self._installed_dists = {}
@@ -154,6 +154,7 @@ def _iter_found_candidates(
         ireqs,  # type: Sequence[InstallRequirement]
         specifier,  # type: SpecifierSet
         hashes,  # type: Hashes
+        prefers_installed,  # type: bool
     ):
         # type: (...) -> Iterable[Candidate]
         if not ireqs:
@@ -172,54 +173,49 @@ def _iter_found_candidates(
             hashes &= ireq.hashes(trust_internet=False)
             extras |= frozenset(ireq.extras)
 
-        # We use this to ensure that we only yield a single candidate for
-        # each version (the finder's preferred one for that version). The
-        # requirement needs to return only one candidate per version, so we
-        # implement that logic here so that requirements using this helper
-        # don't all have to do the same thing later.
-        candidates = collections.OrderedDict()  # type: VersionCandidates
-
         # Get the installed version, if it matches, unless the user
         # specified `--force-reinstall`, when we want the version from
         # the index instead.
-        installed_version = None
         installed_candidate = None
         if not self._force_reinstall and name in self._installed_dists:
             installed_dist = self._installed_dists[name]
-            installed_version = installed_dist.parsed_version
-            if specifier.contains(installed_version, prereleases=True):
+            if specifier.contains(installed_dist.version, prereleases=True):
                 installed_candidate = self._make_candidate_from_dist(
                     dist=installed_dist,
                     extras=extras,
                     template=template,
                 )
 
-        found = self._finder.find_best_candidate(
-            project_name=name,
-            specifier=specifier,
-            hashes=hashes,
-        )
-        for ican in found.iter_applicable():
-            if ican.version == installed_version and installed_candidate:
-                candidate = installed_candidate
-            else:
-                candidate = self._make_candidate_from_link(
+        def iter_index_candidates():
+            # type: () -> Iterator[Candidate]
+            result = self._finder.find_best_candidate(
+                project_name=name,
+                specifier=specifier,
+                hashes=hashes,
+            )
+            # PackageFinder returns earlier versions first, so we reverse.
+            for ican in reversed(list(result.iter_applicable())):
+                yield self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
                     template=template,
                     name=name,
                     version=ican.version,
                 )
-            candidates[ican.version] = candidate
-
-        # Yield the installed version even if it is not found on the index.
-        if installed_version and installed_candidate:
-            candidates[installed_version] = installed_candidate
 
-        return six.itervalues(candidates)
+        return FoundCandidates(
+            iter_index_candidates,
+            installed_candidate,
+            prefers_installed,
+        )
 
-    def find_candidates(self, requirements, constraint):
-        # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate]
+    def find_candidates(
+        self,
+        requirements,  # type: Sequence[Requirement]
+        constraint,  # type: Constraint
+        prefers_installed,  # type: bool
+    ):
+        # type: (...) -> Iterable[Candidate]
         explicit_candidates = set()  # type: Set[Candidate]
         ireqs = []  # type: List[InstallRequirement]
         for req in requirements:
@@ -236,6 +232,7 @@ def find_candidates(self, requirements, constraint):
                 ireqs,
                 constraint.specifier,
                 constraint.hashes,
+                prefers_installed,
             )
 
         if constraint:

src/pip/_internal/resolution/resolvelib/found_candidates.py

@@ -0,0 +1,122 @@
+from pip._vendor.six.moves import collections_abc
+
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Callable, Iterator, Optional, Set
+
+    from pip._vendor.packaging.version import _BaseVersion
+
+    from .base import Candidate
+
+
+class _InstalledFirstCandidatesIterator(collections_abc.Iterator):
+    """Iterator for ``FoundCandidates``.
+
+    This iterator is used when the resolver prefers to keep the version of an
+    already-installed package. The already-installed candidate is always
+    returned first. Candidates from index are accessed only when the resolver
+    wants them, and the already-installed version is excluded from them.
+    """
+    def __init__(
+        self,
+        get_others,  # type: Callable[[], Iterator[Candidate]]
+        installed,  # type: Optional[Candidate]
+    ):
+        self._installed = installed
+        self._get_others = get_others
+        self._others = None  # type: Optional[Iterator[Candidate]]
+        self._returned = set()  # type: Set[_BaseVersion]
+
+    def __next__(self):
+        # type: () -> Candidate
+        if self._installed and self._installed.version not in self._returned:
+            self._returned.add(self._installed.version)
+            return self._installed
+        if self._others is None:
+            self._others = self._get_others()
+        cand = next(self._others)
+        while cand.version in self._returned:
+            cand = next(self._others)
+        self._returned.add(cand.version)
+        return cand
+
+    next = __next__  # XXX: Python 2.
+
+
+class _InstalledReplacesCandidatesIterator(collections_abc.Iterator):
+    """Iterator for ``FoundCandidates``.
+
+    This iterator is used when the resolver prefers to upgrade an
+    already-installed package. Candidates from index are returned in their
+    normal ordering, except replaced when the version is already installed.
+    """
+    def __init__(
+        self,
+        get_others,  # type: Callable[[], Iterator[Candidate]]
+        installed,  # type: Optional[Candidate]
+    ):
+        self._installed = installed
+        self._get_others = get_others
+        self._others = None  # type: Optional[Iterator[Candidate]]
+        self._returned = set()  # type: Set[_BaseVersion]
+
+    def __next__(self):
+        # type: () -> Candidate
+        if self._others is None:
+            self._others = self._get_others()
+        cand = next(self._others)
+        while cand.version in self._returned:
+            cand = next(self._others)
+        if self._installed and cand.version == self._installed.version:
+            cand = self._installed
+        self._returned.add(cand.version)
+        return cand
+
+    next = __next__  # XXX: Python 2.
+
+
+class FoundCandidates(collections_abc.Sequence):
+    """A lazy sequence to provide candidates to the resolver.
+
+    The intended usage is to return this from `find_matches()` so the resolver
+    can iterate through the sequence multiple times, but only access the index
+    page when remote packages are actually needed. This improve performances
+    when suitable candidates are already installed on disk.
+    """
+    def __init__(
+        self,
+        get_others,  # type: Callable[[], Iterator[Candidate]]
+        installed,  # type: Optional[Candidate]
+        prefers_installed,  # type: bool
+    ):
+        self._get_others = get_others
+        self._installed = installed
+        self._prefers_installed = prefers_installed
+
+    def __getitem__(self, index):
+        # type: (int) -> Candidate
+        for i, value in enumerate(self):
+            if index == i:
+                return value
+        raise IndexError(index)
+
+    def __iter__(self):
+        # type: () -> Iterator[Candidate]
+        if self._prefers_installed:
+            klass = _InstalledFirstCandidatesIterator
+        else:
+            klass = _InstalledReplacesCandidatesIterator
+        return klass(self._get_others, self._installed)
+
+    def __len__(self):
+        # type: () -> int
+        return sum(1 for _ in self)
+
+    def __bool__(self):
+        # type: () -> bool
+        if self._prefers_installed and self._installed:
+            return True
+        return any(self)
+
+    __nonzero__ = __bool__  # XXX: Python 2.

src/pip/_internal/resolution/resolvelib/provider.py

@@ -45,30 +45,26 @@ def __init__(
         self._upgrade_strategy = upgrade_strategy
         self._user_requested = user_requested
 
-    def _sort_matches(self, matches):
-        # type: (Iterable[Candidate]) -> Sequence[Candidate]
-
-        # The requirement is responsible for returning a sequence of potential
-        # candidates, one per version. The provider handles the logic of
-        # deciding the order in which these candidates should be passed to
-        # the resolver.
-
-        # The `matches` argument is a sequence of candidates, one per version,
-        # which are potential options to be installed. The requirement will
-        # have already sorted out whether to give us an already-installed
-        # candidate or a version from PyPI (i.e., it will deal with options
-        # like --force-reinstall and --ignore-installed).
-
-        # We now work out the correct order.
-        #
-        # 1. If no other considerations apply, later versions take priority.
-        # 2. An already installed distribution is preferred over any other,
-        #    unless the user has requested an upgrade.
-        #    Upgrades are allowed when:
-        #    * The --upgrade flag is set, and
-        #      - The project was specified on the command line, or
-        #      - The project is a dependency and the "eager" upgrade strategy
-        #        was requested.
+    def identify(self, dependency):
+        # type: (Union[Requirement, Candidate]) -> str
+        return dependency.name
+
+    def get_preference(
+        self,
+        resolution,  # type: Optional[Candidate]
+        candidates,  # type: Sequence[Candidate]
+        information  # type: Sequence[Tuple[Requirement, Candidate]]
+    ):
+        # type: (...) -> Any
+        transitive = all(parent is not None for _, parent in information)
+        return (transitive, bool(candidates))
+
+    def find_matches(self, requirements):
+        # type: (Sequence[Requirement]) -> Iterable[Candidate]
+        if not requirements:
+            return []
+        name = requirements[0].name
+
         def _eligible_for_upgrade(name):
             # type: (str) -> bool
             """Are upgrades allowed for this project?
@@ -87,56 +83,11 @@ def _eligible_for_upgrade(name):
                 return (name in self._user_requested)
             return False
 
-        def sort_key(c):
-            # type: (Candidate) -> int
-            """Return a sort key for the matches.
-
-            The highest priority should be given to installed candidates that
-            are not eligible for upgrade. We use the integer value in the first
-            part of the key to sort these before other candidates.
-
-            We only pull the installed candidate to the bottom (i.e. most
-            preferred), but otherwise keep the ordering returned by the
-            requirement. The requirement is responsible for returning a list
-            otherwise sorted for the resolver, taking account for versions
-            and binary preferences as specified by the user.
-            """
-            if c.is_installed and not _eligible_for_upgrade(c.name):
-                return 1
-            return 0
-
-        return sorted(matches, key=sort_key)
-
-    def identify(self, dependency):
-        # type: (Union[Requirement, Candidate]) -> str
-        return dependency.name
-
-    def get_preference(
-        self,
-        resolution,  # type: Optional[Candidate]
-        candidates,  # type: Sequence[Candidate]
-        information  # type: Sequence[Tuple[Requirement, Optional[Candidate]]]
-    ):
-        # type: (...) -> Any
-        """Return a sort key to determine what dependency to look next.
-
-        A smaller value makes a dependency higher priority. We put direct
-        (user-requested) dependencies first since they may contain useful
-        user-specified version ranges. Users tend to expect us to catch
-        problems in them early as well.
-        """
-        transitive = all(parent is not None for _, parent in information)
-        return (transitive, len(candidates))
-
-    def find_matches(self, requirements):
-        # type: (Sequence[Requirement]) -> Iterable[Candidate]
-        if not requirements:
-            return []
-        constraint = self._constraints.get(
-            requirements[0].name, Constraint.empty(),
+        return self._factory.find_candidates(
+            requirements,
+            constraint=self._constraints.get(name, Constraint.empty()),
+            prefers_installed=(not _eligible_for_upgrade(name)),
         )
-        candidates = self._factory.find_candidates(requirements, constraint)
-        return reversed(self._sort_matches(candidates))
 
     def is_satisfied_by(self, requirement, candidate):
         # type: (Requirement, Candidate) -> bool

tests/unit/resolution_resolvelib/test_requirement.py

@@ -58,7 +58,9 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory):
     """Requirements should return the correct number of candidates"""
     for spec, _, match_count in test_cases:
         req = factory.make_requirement_from_spec(spec, comes_from=None)
-        matches = factory.find_candidates([req], Constraint.empty())
+        matches = factory.find_candidates(
+            [req], Constraint.empty(), prefers_installed=False,
+        )
         assert len(list(matches)) == match_count
 
 
@@ -67,7 +69,10 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory):
     """
     for spec, _, _ in test_cases:
         req = factory.make_requirement_from_spec(spec, comes_from=None)
-        for c in factory.find_candidates([req], Constraint.empty()):
+        candidates = factory.find_candidates(
+            [req], Constraint.empty(), prefers_installed=False,
+        )
+        for c in candidates:
             assert isinstance(c, Candidate)
             assert req.is_satisfied_by(c)