Merge pull request #8910 from hugovk/pip-cache-http

Include http subdirectory in 'pip cache info' and 'pip cache purge'

Author: pradyunsg

news/8892.feature

@@ -0,0 +1 @@
+Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands.

src/pip/_internal/commands/cache.py

@@ -102,19 +102,30 @@ def get_cache_info(self, options, args):
         if args:
             raise CommandError('Too many arguments')
 
+        num_http_files = len(self._find_http_files(options))
         num_packages = len(self._find_wheels(options, '*'))
 
-        cache_location = self._wheels_cache_dir(options)
-        cache_size = filesystem.format_directory_size(cache_location)
+        http_cache_location = self._cache_dir(options, 'http')
+        wheels_cache_location = self._cache_dir(options, 'wheels')
+        http_cache_size = filesystem.format_directory_size(http_cache_location)
+        wheels_cache_size = filesystem.format_directory_size(
+            wheels_cache_location
+        )
 
         message = textwrap.dedent("""
-            Location: {location}
-            Size: {size}
+            Package index page cache location: {http_cache_location}
+            Package index page cache size: {http_cache_size}
+            Number of HTTP files: {num_http_files}
+            Wheels location: {wheels_cache_location}
+            Wheels size: {wheels_cache_size}
             Number of wheels: {package_count}
         """).format(
-            location=cache_location,
+            http_cache_location=http_cache_location,
+            http_cache_size=http_cache_size,
+            num_http_files=num_http_files,
+            wheels_cache_location=wheels_cache_location,
             package_count=num_packages,
-            size=cache_size,
+            wheels_cache_size=wheels_cache_size,
         ).strip()
 
         logger.info(message)
@@ -169,6 +180,11 @@ def remove_cache_items(self, options, args):
             raise CommandError('Please provide a pattern')
 
         files = self._find_wheels(options, args[0])
+
+        # Only fetch http files if no specific pattern given
+        if args[0] == '*':
+            files += self._find_http_files(options)
+
         if not files:
             raise CommandError('No matching packages')
 
@@ -184,13 +200,18 @@ def purge_cache(self, options, args):
 
         return self.remove_cache_items(options, ['*'])
 
-    def _wheels_cache_dir(self, options):
-        # type: (Values) -> str
-        return os.path.join(options.cache_dir, 'wheels')
+    def _cache_dir(self, options, subdir):
+        # type: (Values, str) -> str
+        return os.path.join(options.cache_dir, subdir)
+
+    def _find_http_files(self, options):
+        # type: (Values) -> List[str]
+        http_dir = self._cache_dir(options, 'http')
+        return filesystem.find_files(http_dir, '*')
 
     def _find_wheels(self, options, pattern):
         # type: (Values, str) -> List[str]
-        wheel_dir = self._wheels_cache_dir(options)
+        wheel_dir = self._cache_dir(options, 'wheels')
 
         # The wheel filename format, as specified in PEP 427, is:
         #     {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl

tests/functional/test_cache.py

@@ -15,11 +15,30 @@ def cache_dir(script):
     return result.stdout.strip()
 
 
+@pytest.fixture
+def http_cache_dir(cache_dir):
+    return os.path.normcase(os.path.join(cache_dir, 'http'))
+
+
 @pytest.fixture
 def wheel_cache_dir(cache_dir):
     return os.path.normcase(os.path.join(cache_dir, 'wheels'))
 
 
+@pytest.fixture
+def http_cache_files(http_cache_dir):
+    destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname')
+
+    if not os.path.exists(destination):
+        return []
+
+    filenames = glob(os.path.join(destination, '*'))
+    files = []
+    for filename in filenames:
+        files.append(os.path.join(destination, filename))
+    return files
+
+
 @pytest.fixture
 def wheel_cache_files(wheel_cache_dir):
     destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname')
@@ -34,6 +53,24 @@ def wheel_cache_files(wheel_cache_dir):
     return files
 
 
+@pytest.fixture
+def populate_http_cache(http_cache_dir):
+    destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname')
+    os.makedirs(destination)
+
+    files = [
+        ('aaaaaaaaa', os.path.join(destination, 'aaaaaaaaa')),
+        ('bbbbbbbbb', os.path.join(destination, 'bbbbbbbbb')),
+        ('ccccccccc', os.path.join(destination, 'ccccccccc')),
+    ]
+
+    for _name, filename in files:
+        with open(filename, 'w'):
+            pass
+
+    return files
+
+
 @pytest.fixture
 def populate_wheel_cache(wheel_cache_dir):
     destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname')
@@ -83,6 +120,29 @@ def list_matches_wheel_abspath(wheel_name, result):
                and os.path.exists(l), lines))
 
 
+@pytest.fixture
+def remove_matches_http(http_cache_dir):
+    """Returns True if any line in `result`, which should be the output of
+    a `pip cache purge` call, matches `http_filename`.
+
+    E.g., If http_filename is `aaaaaaaaa`, it searches for a line equal to
+    `Removed <http files cache dir>/arbitrary/pathname/aaaaaaaaa`.
+    """
+
+    def _remove_matches_http(http_filename, result):
+        lines = result.stdout.splitlines()
+
+        # The "/arbitrary/pathname/" bit is an implementation detail of how
+        # the `populate_http_cache` fixture is implemented.
+        path = os.path.join(
+            http_cache_dir, 'arbitrary', 'pathname', http_filename,
+        )
+        expected = 'Removed {}'.format(path)
+        return expected in lines
+
+    return _remove_matches_http
+
+
 @pytest.fixture
 def remove_matches_wheel(wheel_cache_dir):
     """Returns True if any line in `result`, which should be the output of
@@ -124,11 +184,17 @@ def test_cache_dir_too_many_args(script, cache_dir):
     assert 'ERROR: Too many arguments' in result.stderr.splitlines()
 
 
-@pytest.mark.usefixtures("populate_wheel_cache")
-def test_cache_info(script, wheel_cache_dir, wheel_cache_files):
+@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache")
+def test_cache_info(
+        script, http_cache_dir, wheel_cache_dir, wheel_cache_files
+):
     result = script.pip('cache', 'info')
 
-    assert 'Location: {}'.format(wheel_cache_dir) in result.stdout
+    assert (
+        'Package index page cache location: {}'.format(http_cache_dir)
+        in result.stdout
+    )
+    assert 'Wheels location: {}'.format(wheel_cache_dir) in result.stdout
     num_wheels = len(wheel_cache_files)
     assert 'Number of wheels: {}'.format(num_wheels) in result.stdout
 
@@ -265,21 +331,28 @@ def test_cache_remove_name_and_version_match(script, remove_matches_wheel):
     assert not remove_matches_wheel('zzz-7.8.9', result)
 
 
-@pytest.mark.usefixtures("populate_wheel_cache")
-def test_cache_purge(script, remove_matches_wheel):
-    """Running `pip cache purge` should remove all cached wheels."""
+@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache")
+def test_cache_purge(script, remove_matches_http, remove_matches_wheel):
+    """Running `pip cache purge` should remove all cached http files and
+    wheels."""
     result = script.pip('cache', 'purge', '--verbose')
 
+    assert remove_matches_http('aaaaaaaaa', result)
+    assert remove_matches_http('bbbbbbbbb', result)
+    assert remove_matches_http('ccccccccc', result)
+
     assert remove_matches_wheel('yyy-1.2.3', result)
     assert remove_matches_wheel('zzz-4.5.6', result)
     assert remove_matches_wheel('zzz-4.5.7', result)
     assert remove_matches_wheel('zzz-7.8.9', result)
 
 
-@pytest.mark.usefixtures("populate_wheel_cache")
-def test_cache_purge_too_many_args(script, wheel_cache_files):
+@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache")
+def test_cache_purge_too_many_args(
+        script, http_cache_files, wheel_cache_files
+):
     """Running `pip cache purge aaa` should raise an error and remove no
-    cached wheels."""
+    cached http files or wheels."""
     result = script.pip('cache', 'purge', 'aaa', '--verbose',
                         expect_error=True)
     assert result.stdout == ''
@@ -289,7 +362,7 @@ def test_cache_purge_too_many_args(script, wheel_cache_files):
     assert 'ERROR: Too many arguments' in result.stderr.splitlines()
 
     # Make sure nothing was deleted.
-    for filename in wheel_cache_files:
+    for filename in http_cache_files + wheel_cache_files:
         assert os.path.exists(filename)