diff --git a/news/10550.feature.rst b/news/10550.feature.rst new file mode 100644 index 00000000000..942fe58bcb1 --- /dev/null +++ b/news/10550.feature.rst @@ -0,0 +1 @@ +Cache requirement objects, to improve performance reducing reparses of requirement strings. diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 0a6b3367e1e..5cf923515d7 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -26,6 +26,7 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.filetypes import is_archive_file from pip._internal.utils.misc import is_installable_dir +from pip._internal.utils.packaging import get_requirement from pip._internal.utils.urls import path_to_url from pip._internal.vcs import is_url, vcs @@ -54,7 +55,7 @@ def _strip_extras(path: str) -> Tuple[str, Optional[str]]: def convert_extras(extras: Optional[str]) -> Set[str]: if not extras: return set() - return Requirement("placeholder" + extras.lower()).extras + return get_requirement("placeholder" + extras.lower()).extras def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: @@ -83,7 +84,7 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: return ( package_name, url_no_extras, - Requirement("placeholder" + extras.lower()).extras, + get_requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, set() @@ -309,7 +310,7 @@ def with_source(text: str) -> str: def _parse_req_string(req_as_string: str) -> Requirement: try: - req = Requirement(req_as_string) + req = get_requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." @@ -386,7 +387,7 @@ def install_req_from_req_string( user_supplied: bool = False, ) -> InstallRequirement: try: - req = Requirement(req_string) + req = get_requirement(req_string) except InvalidRequirement: raise InstallationError(f"Invalid requirement: '{req_string}'") diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index e96764d31ea..766dc26c0f9 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -19,7 +19,6 @@ ) from pip._vendor.packaging.requirements import InvalidRequirement -from pip._vendor.packaging.requirements import Requirement as PackagingRequirement from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name from pip._vendor.resolvelib import ResolutionImpossible @@ -46,6 +45,7 @@ from pip._internal.resolution.base import InstallRequirementProvider from pip._internal.utils.compatibility_tags import get_supported from pip._internal.utils.hashes import Hashes +from pip._internal.utils.packaging import get_requirement from pip._internal.utils.virtualenv import running_under_virtualenv from .base import Candidate, CandidateVersion, Constraint, Requirement @@ -365,7 +365,7 @@ def find_candidates( # If the current identifier contains extras, add explicit candidates # from entries from extra-less identifier. with contextlib.suppress(InvalidRequirement): - parsed_requirement = PackagingRequirement(identifier) + parsed_requirement = get_requirement(identifier) explicit_candidates.update( self._iter_explicit_candidates_from_base( requirements.get(parsed_requirement.name, ()), diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 9687c86aa3c..f100473e647 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -1,3 +1,4 @@ +import functools import logging from email.message import Message from email.parser import FeedParser @@ -5,6 +6,7 @@ from pip._vendor import pkg_resources from pip._vendor.packaging import specifiers, version +from pip._vendor.packaging.requirements import Requirement from pip._vendor.pkg_resources import Distribution from pip._internal.exceptions import NoneMetadataError @@ -69,3 +71,14 @@ def get_installer(dist: Distribution) -> str: if line.strip(): return line.strip() return "" + + +@functools.lru_cache(maxsize=512) +def get_requirement(req_string: str) -> Requirement: + """Construct a packaging.Requirement object with caching""" + # Parsing requirement strings is expensive, and is also expected to happen + # with a low diversity of different arguments (at least relative the number + # constructed). This method adds a cache to requirement object creation to + # minimize repeated parsing of the same string to construct equivalent + # Requirement objects. + return Requirement(req_string) diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 8750ca85338..88277448c2c 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -2,8 +2,9 @@ import pytest from pip._vendor.packaging import specifiers +from pip._vendor.packaging.requirements import Requirement -from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.packaging import check_requires_python, get_requirement @pytest.mark.parametrize( @@ -27,3 +28,17 @@ def test_check_requires_python__invalid() -> None: """ with pytest.raises(specifiers.InvalidSpecifier): check_requires_python("invalid", (3, 6, 5)) + + +def test_get_or_create_caching() -> None: + """test caching of get_or_create requirement""" + teststr = "affinegap==1.10" + from_helper = get_requirement(teststr) + freshly_made = Requirement(teststr) + + # Requirement doesn't have an equality operator (yet) so test + # equality of attribute for list of attributes + for iattr in ["name", "url", "extras", "specifier", "marker"]: + assert getattr(from_helper, iattr) == getattr(freshly_made, iattr) + assert get_requirement(teststr) is not Requirement(teststr) + assert get_requirement(teststr) is get_requirement(teststr)