Merge pull request #10032 from uranusjr/new-resolver-order-projects-by-depth

Rework resolution ordering to consider "depth"

Author: pradyunsg

news/9455.feature.rst

@@ -0,0 +1,2 @@
+New resolver: The order of dependencies resolution has been tweaked to traverse
+the dependency graph in a more breadth-first approach.

src/pip/_internal/resolution/resolvelib/provider.py

@@ -1,3 +1,5 @@
+import collections
+import math
 from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union
 
 from pip._vendor.resolvelib.providers import AbstractProvider
@@ -60,6 +62,7 @@ def __init__(
         self._ignore_dependencies = ignore_dependencies
         self._upgrade_strategy = upgrade_strategy
         self._user_requested = user_requested
+        self._known_depths: Dict[str, float] = collections.defaultdict(lambda: math.inf)
 
     def identify(self, requirement_or_candidate):
         # type: (Union[Requirement, Candidate]) -> str
@@ -79,48 +82,43 @@ def get_preference(
 
         Currently pip considers the followings in order:
 
-        * Prefer if any of the known requirements points to an explicit URL.
-        * If equal, prefer if any requirements contain ``===`` and ``==``.
-        * If equal, prefer if requirements include version constraints, e.g.
-          ``>=`` and ``<``.
-        * If equal, prefer user-specified (non-transitive) requirements, and
-          order user-specified requirements by the order they are specified.
+        * Prefer if any of the known requirements is "direct", e.g. points to an
+          explicit URL.
+        * If equal, prefer if any requirement is "pinned", i.e. contains
+          operator ``===`` or ``==``.
+        * If equal, calculate an approximate "depth" and resolve requirements
+          closer to the user-specified requirements first.
+        * Order user-specified requirements by the order they are specified.
+        * If equal, prefers "non-free" requirements, i.e. contains at least one
+          operator, such as ``>=`` or ``<``.
         * If equal, order alphabetically for consistency (helps debuggability).
         """
-
-        def _get_restrictive_rating(requirements):
-            # type: (Iterable[Requirement]) -> int
-            """Rate how restrictive a set of requirements are.
-
-            ``Requirement.get_candidate_lookup()`` returns a 2-tuple for
-            lookup. The first element is ``Optional[Candidate]`` and the
-            second ``Optional[InstallRequirement]``.
-
-            * If the requirement is an explicit one, the explicitly-required
-              candidate is returned as the first element.
-            * If the requirement is based on a PEP 508 specifier, the backing
-              ``InstallRequirement`` is returned as the second element.
-
-            We use the first element to check whether there is an explicit
-            requirement, and the second for equality operator.
-            """
-            lookups = (r.get_candidate_lookup() for r in requirements)
-            cands, ireqs = zip(*lookups)
-            if any(cand is not None for cand in cands):
-                return 0
-            spec_sets = (ireq.specifier for ireq in ireqs if ireq)
-            operators = [
-                specifier.operator for spec_set in spec_sets for specifier in spec_set
-            ]
-            if any(op in ("==", "===") for op in operators):
-                return 1
-            if operators:
-                return 2
-            # A "bare" requirement without any version requirements.
-            return 3
-
-        rating = _get_restrictive_rating(r for r, _ in information[identifier])
-        order = self._user_requested.get(identifier, float("inf"))
+        lookups = (r.get_candidate_lookup() for r, _ in information[identifier])
+        candidate, ireqs = zip(*lookups)
+        operators = [
+            specifier.operator
+            for specifier_set in (ireq.specifier for ireq in ireqs if ireq)
+            for specifier in specifier_set
+        ]
+
+        direct = candidate is not None
+        pinned = any(op[:2] == "==" for op in operators)
+        unfree = bool(operators)
+
+        try:
+            requested_order: Union[int, float] = self._user_requested[identifier]
+        except KeyError:
+            requested_order = math.inf
+            parent_depths = (
+                self._known_depths[parent.name] if parent is not None else 0.0
+                for _, parent in information[identifier]
+            )
+            inferred_depth = min(d for d in parent_depths) + 1.0
+            self._known_depths[identifier] = inferred_depth
+        else:
+            inferred_depth = 1.0
+
+        requested_order = self._user_requested.get(identifier, math.inf)
 
         # Requires-Python has only one candidate and the check is basically
         # free, so we always do it first to avoid needless work if it fails.
@@ -136,7 +134,16 @@ def _get_restrictive_rating(requirements):
         # while we work on "proper" branch pruning techniques.
         delay_this = identifier == "setuptools"
 
-        return (not requires_python, delay_this, rating, order, identifier)
+        return (
+            not requires_python,
+            delay_this,
+            not direct,
+            not pinned,
+            inferred_depth,
+            requested_order,
+            not unfree,
+            identifier,
+        )
 
     def find_matches(
         self,