Merge pull request #8267 from uranusjr/new-resolver-candidate-order

Upgrade ResolveLib to 0.4.0 and implement the new Provider.find_matches() interface

Author: pfmoore

src/pip/_internal/resolution/resolvelib/base.py

@@ -3,15 +3,20 @@
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Iterable, Optional, Sequence, Set
+    from typing import FrozenSet, Iterable, Optional, Tuple
 
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._vendor.packaging.specifiers import SpecifierSet
     from pip._vendor.packaging.version import _BaseVersion
 
+    from pip._internal.req.req_install import InstallRequirement
+
+    CandidateLookup = Tuple[
+        Optional["Candidate"],
+        Optional[InstallRequirement],
+    ]
+
 
 def format_name(project, extras):
-    # type: (str, Set[str]) -> str
+    # type: (str, FrozenSet[str]) -> str
     if not extras:
         return project
     canonical_extras = sorted(canonicalize_name(e) for e in extras)
@@ -24,14 +29,14 @@ def name(self):
         # type: () -> str
         raise NotImplementedError("Subclass should override")
 
-    def find_matches(self, constraint):
-        # type: (SpecifierSet) -> Sequence[Candidate]
-        raise NotImplementedError("Subclass should override")
-
     def is_satisfied_by(self, candidate):
         # type: (Candidate) -> bool
         return False
 
+    def get_candidate_lookup(self):
+        # type: () -> CandidateLookup
+        raise NotImplementedError("Subclass should override")
+
 
 class Candidate(object):
     @property

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -17,7 +17,7 @@
 from .base import Candidate, format_name
 
 if MYPY_CHECK_RUNNING:
-    from typing import Any, Iterable, Optional, Set, Tuple, Union
+    from typing import Any, FrozenSet, Iterable, Optional, Tuple, Union
 
     from pip._vendor.packaging.version import _BaseVersion
     from pip._vendor.pkg_resources import Distribution
@@ -132,6 +132,10 @@ def __repr__(self):
             link=str(self.link),
         )
 
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.__class__, self.link))
+
     def __eq__(self, other):
         # type: (Any) -> bool
         if isinstance(other, self.__class__):
@@ -313,6 +317,10 @@ def __repr__(self):
             distribution=self.dist,
         )
 
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.__class__, self.name, self.version))
+
     def __eq__(self, other):
         # type: (Any) -> bool
         if isinstance(other, self.__class__):
@@ -371,7 +379,7 @@ class ExtrasCandidate(Candidate):
     def __init__(
         self,
         base,  # type: BaseCandidate
-        extras,  # type: Set[str]
+        extras,  # type: FrozenSet[str]
     ):
         # type: (...) -> None
         self.base = base
@@ -385,6 +393,10 @@ def __repr__(self):
             extras=self.extras,
         )
 
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.base, self.extras))
+
     def __eq__(self, other):
         # type: (Any) -> bool
         if isinstance(other, self.__class__):

src/pip/_internal/resolution/resolvelib/factory.py

@@ -9,6 +9,7 @@
     UnsupportedPythonVersion,
 )
 from pip._internal.utils.compatibility_tags import get_supported
+from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.misc import (
     dist_in_site_packages,
     dist_in_usersite,
@@ -31,7 +32,17 @@
 )
 
 if MYPY_CHECK_RUNNING:
-    from typing import Dict, Iterable, Iterator, Optional, Set, Tuple, TypeVar
+    from typing import (
+        FrozenSet,
+        Dict,
+        Iterable,
+        List,
+        Optional,
+        Sequence,
+        Set,
+        Tuple,
+        TypeVar,
+    )
 
     from pip._vendor.packaging.specifiers import SpecifierSet
     from pip._vendor.packaging.version import _BaseVersion
@@ -71,7 +82,7 @@ def __init__(
     ):
         # type: (...) -> None
 
-        self.finder = finder
+        self._finder = finder
         self.preparer = preparer
         self._wheel_cache = wheel_cache
         self._python_candidate = RequiresPythonCandidate(py_version_info)
@@ -94,7 +105,7 @@ def __init__(
     def _make_candidate_from_dist(
         self,
         dist,  # type: Distribution
-        extras,  # type: Set[str]
+        extras,  # type: FrozenSet[str]
         parent,  # type: InstallRequirement
     ):
         # type: (...) -> Candidate
@@ -106,7 +117,7 @@ def _make_candidate_from_dist(
     def _make_candidate_from_link(
         self,
         link,  # type: Link
-        extras,  # type: Set[str]
+        extras,  # type: FrozenSet[str]
         parent,  # type: InstallRequirement
         name,  # type: Optional[str]
         version,  # type: Optional[_BaseVersion]
@@ -130,9 +141,28 @@ def _make_candidate_from_link(
             return ExtrasCandidate(base, extras)
         return base
 
-    def iter_found_candidates(self, ireq, extras):
-        # type: (InstallRequirement, Set[str]) -> Iterator[Candidate]
-        name = canonicalize_name(ireq.req.name)
+    def _iter_found_candidates(
+        self,
+        ireqs,  # type: Sequence[InstallRequirement]
+        specifier,  # type: SpecifierSet
+    ):
+        # type: (...) -> Iterable[Candidate]
+        if not ireqs:
+            return ()
+
+        # The InstallRequirement implementation requires us to give it a
+        # "parent", which doesn't really fit with graph-based resolution.
+        # Here we just choose the first requirement to represent all of them.
+        # Hopefully the Project model can correct this mismatch in the future.
+        parent = ireqs[0]
+        name = canonicalize_name(parent.req.name)
+
+        hashes = Hashes()
+        extras = frozenset()  # type: FrozenSet[str]
+        for ireq in ireqs:
+            specifier &= ireq.req.specifier
+            hashes |= ireq.hashes(trust_internet=False)
+            extras |= frozenset(ireq.extras)
 
         # We use this to ensure that we only yield a single candidate for
         # each version (the finder's preferred one for that version). The
@@ -148,43 +178,68 @@ def iter_found_candidates(self, ireq, extras):
         if not self._force_reinstall and name in self._installed_dists:
             installed_dist = self._installed_dists[name]
             installed_version = installed_dist.parsed_version
-            if ireq.req.specifier.contains(
-                installed_version,
-                prereleases=True
-            ):
+            if specifier.contains(installed_version, prereleases=True):
                 candidate = self._make_candidate_from_dist(
                     dist=installed_dist,
                     extras=extras,
-                    parent=ireq,
+                    parent=parent,
                 )
                 candidates[installed_version] = candidate
 
-        found = self.finder.find_best_candidate(
-            project_name=ireq.req.name,
-            specifier=ireq.req.specifier,
-            hashes=ireq.hashes(trust_internet=False),
+        found = self._finder.find_best_candidate(
+            project_name=name,
+            specifier=specifier,
+            hashes=hashes,
         )
         for ican in found.iter_applicable():
             if ican.version == installed_version:
                 continue
             candidate = self._make_candidate_from_link(
                 link=ican.link,
                 extras=extras,
-                parent=ireq,
+                parent=parent,
                 name=name,
                 version=ican.version,
             )
             candidates[ican.version] = candidate
 
         return six.itervalues(candidates)
 
+    def find_candidates(self, requirements, constraint):
+        # type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate]
+        explicit_candidates = set()  # type: Set[Candidate]
+        ireqs = []  # type: List[InstallRequirement]
+        for req in requirements:
+            cand, ireq = req.get_candidate_lookup()
+            if cand is not None:
+                explicit_candidates.add(cand)
+            if ireq is not None:
+                ireqs.append(ireq)
+
+        # If none of the requirements want an explicit candidate, we can ask
+        # the finder for candidates.
+        if not explicit_candidates:
+            return self._iter_found_candidates(ireqs, constraint)
+
+        if constraint:
+            name = explicit_candidates.pop().name
+            raise InstallationError(
+                "Could not satisfy constraints for {!r}: installation from "
+                "path or url cannot be constrained to a version".format(name)
+            )
+
+        return (
+            c for c in explicit_candidates
+            if all(req.is_satisfied_by(c) for req in requirements)
+        )
+
     def make_requirement_from_install_req(self, ireq):
         # type: (InstallRequirement) -> Requirement
         if not ireq.link:
-            return SpecifierRequirement(ireq, factory=self)
+            return SpecifierRequirement(ireq)
         cand = self._make_candidate_from_link(
             ireq.link,
-            extras=set(ireq.extras),
+            extras=frozenset(ireq.extras),
             parent=ireq,
             name=canonicalize_name(ireq.name) if ireq.name else None,
             version=None,

src/pip/_internal/resolution/resolvelib/provider.py

@@ -4,7 +4,16 @@
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Optional, Sequence, Set, Tuple, Union
+    from typing import (
+        Any,
+        Dict,
+        Iterable,
+        Optional,
+        Sequence,
+        Set,
+        Tuple,
+        Union,
+    )
 
     from .base import Requirement, Candidate
     from .factory import Factory
@@ -45,7 +54,7 @@ def __init__(
         self.user_requested = user_requested
 
     def _sort_matches(self, matches):
-        # type: (Sequence[Candidate]) -> Sequence[Candidate]
+        # type: (Iterable[Candidate]) -> Sequence[Candidate]
 
         # The requirement is responsible for returning a sequence of potential
         # candidates, one per version. The provider handles the logic of
@@ -68,7 +77,6 @@ def _sort_matches(self, matches):
         #      - The project was specified on the command line, or
         #      - The project is a dependency and the "eager" upgrade strategy
         #        was requested.
-
         def _eligible_for_upgrade(name):
             # type: (str) -> bool
             """Are upgrades allowed for this project?
@@ -121,11 +129,15 @@ def get_preference(
         # Use the "usual" value for now
         return len(candidates)
 
-    def find_matches(self, requirement):
-        # type: (Requirement) -> Sequence[Candidate]
-        constraint = self._constraints.get(requirement.name, SpecifierSet())
-        matches = requirement.find_matches(constraint)
-        return self._sort_matches(matches)
+    def find_matches(self, requirements):
+        # type: (Sequence[Requirement]) -> Iterable[Candidate]
+        if not requirements:
+            return []
+        constraint = self._constraints.get(
+            requirements[0].name, SpecifierSet(),
+        )
+        candidates = self._factory.find_candidates(requirements, constraint)
+        return reversed(self._sort_matches(candidates))
 
     def is_satisfied_by(self, requirement, candidate):
         # type: (Requirement, Candidate) -> bool