(joe) requirement creation is very expensive, and at least in my test case there were only ~200 unique requirement objects created in ~5-10 minutes of resolution time

Author: jbylund

news/10550.feature.rst

@@ -0,0 +1 @@
+Improve performance of dependency resolution.

src/pip/_internal/req/constructors.py

@@ -8,6 +8,7 @@
 InstallRequirement.
 """
 
+import functools
 import logging
 import os
 import re
@@ -39,6 +40,11 @@
 operators = Specifier._operators.keys()
 
 
+@functools.lru_cache(maxsize=None)
+def get_or_create_requirement(req_string: str) -> Requirement:
+    return Requirement(req_string)
+
+
 def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
     m = re.match(r"^(.+)(\[[^\]]+\])$", path)
     extras = None
@@ -54,7 +60,7 @@ def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
 def convert_extras(extras: Optional[str]) -> Set[str]:
     if not extras:
         return set()
-    return Requirement("placeholder" + extras.lower()).extras
+    return get_or_create_requirement("placeholder" + extras.lower()).extras
 
 
 def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
@@ -83,7 +89,7 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
             return (
                 package_name,
                 url_no_extras,
-                Requirement("placeholder" + extras.lower()).extras,
+                get_or_create_requirement("placeholder" + extras.lower()).extras,
             )
         else:
             return package_name, url_no_extras, set()
@@ -309,7 +315,7 @@ def with_source(text: str) -> str:
 
     def _parse_req_string(req_as_string: str) -> Requirement:
         try:
-            req = Requirement(req_as_string)
+            req = get_or_create_requirement(req_as_string)
         except InvalidRequirement:
             if os.path.sep in req_as_string:
                 add_msg = "It looks like a path."
@@ -386,7 +392,7 @@ def install_req_from_req_string(
     user_supplied: bool = False,
 ) -> InstallRequirement:
     try:
-        req = Requirement(req_string)
+        req = get_or_create_requirement(req_string)
     except InvalidRequirement:
         raise InstallationError(f"Invalid requirement: '{req_string}'")
 

src/pip/_internal/resolution/resolvelib/factory.py

@@ -18,8 +18,9 @@
     cast,
 )
 
-from pip._vendor.packaging.requirements import InvalidRequirement
-from pip._vendor.packaging.requirements import Requirement as PackagingRequirement
+from pip._vendor.packaging.requirements import (
+    InvalidRequirement,
+)
 from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.resolvelib import ResolutionImpossible
@@ -38,7 +39,7 @@
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
 from pip._internal.operations.prepare import RequirementPreparer
-from pip._internal.req.constructors import install_req_from_link_and_ireq
+from pip._internal.req.constructors import get_or_create_requirement, install_req_from_link_and_ireq
 from pip._internal.req.req_install import (
     InstallRequirement,
     check_invalid_constraint_type,
@@ -365,7 +366,7 @@ def find_candidates(
         # If the current identifier contains extras, add explicit candidates
         # from entries from extra-less identifier.
         with contextlib.suppress(InvalidRequirement):
-            parsed_requirement = PackagingRequirement(identifier)
+            parsed_requirement = get_or_create_requirement(identifier)
             explicit_candidates.update(
                 self._iter_explicit_candidates_from_base(
                     requirements.get(parsed_requirement.name, ()),

tests/unit/test_req.py

@@ -658,6 +658,14 @@ def test_parse_editable_local_extras(
     )
 
 
+def test_get_or_create_caching() -> None:
+    """test caching of get_or_create requirement"""
+    teststr = "affinegap==1.10"
+    assert get_or_create_requirement(teststr) == Requirement(teststr)
+    assert not (get_or_create_requirement(teststr) is Requirement(teststr))
+    assert get_or_create_requirement(teststr) is get_or_create_requirement(teststr)
+
+
 def test_exclusive_environment_markers() -> None:
     """Make sure RequirementSet accepts several excluding env markers"""
     eq36 = install_req_from_line("Django>=1.6.10,<1.7 ; python_version == '3.6'")