try to build wheels for directories/vcs

news/4501.feature

@@ -0,0 +1,2 @@
+pip now builds a wheel when installing from a local directory or a VCS URL.
+Wheels from these sources are not cached.

src/pip/_internal/cache.py

@@ -11,6 +11,7 @@
 from pip._internal import index
 from pip._internal.compat import expanduser
 from pip._internal.download import path_to_url
+from pip._internal.utils import temp_dir
 from pip._internal.wheel import InvalidWheelFilename, Wheel
 
 logger = logging.getLogger(__name__)
@@ -32,6 +33,9 @@ def __init__(self, cache_dir, format_control, allowed_formats):
         self.cache_dir = expanduser(cache_dir) if cache_dir else None
         self.format_control = format_control
         self.allowed_formats = allowed_formats
+        # Ephemeral cache: store wheels just for this run
+        self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache")
+        self._ephem_cache_dir.create()
 
         _valid_formats = {"source", "binary"}
         assert self.allowed_formats.union(_valid_formats) == _valid_formats
@@ -102,6 +106,10 @@ def _link_for_candidate(self, link, candidate):
 
         return index.Link(path_to_url(path))
 
+    def cleanup(self):
+        """Remove the ephermal caches created temporarily to build wheels"""
+        self._ephem_cache_dir.cleanup()
+
 
 class WheelCache(Cache):
     """A cache of wheels for future installs.
@@ -131,6 +139,18 @@ def get_path_for_link(self, link):
         # join them all together.
         return os.path.join(self.cache_dir, "wheels", *parts)
 
+    def get_ephem_path_for_link(self, link):
+        """Return a directory to store cached wheels for link
+
+        Unlike get_path_for_link, the directory will be removed
+        before pip exits.
+        """
+        parts = self._get_cache_path_parts(link)
+
+        # Inside of the base location for cached wheels, expand our parts and
+        # join them all together.
+        return os.path.join(self._ephem_cache_dir.path, "wheels", *parts)
+
     def get(self, link, package_name):
         candidates = []
 

src/pip/_internal/commands/freeze.py

@@ -88,5 +88,8 @@ def run(self, options, args):
             skip=skip,
             exclude_editable=options.exclude_editable)
 
-        for line in freeze(**freeze_kwargs):
-            sys.stdout.write(line + '\n')
+        try:
+            for line in freeze(**freeze_kwargs):
+                sys.stdout.write(line + '\n')
+        finally:
+            wheel_cache.cleanup()

src/pip/_internal/commands/install.py

@@ -247,18 +247,19 @@ def run(self, options, args):
                     use_user_site=options.use_user_site,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=None,
-                    progress_bar=options.progress_bar,
-                )
                 try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=None,
+                        progress_bar=options.progress_bar,
+                    )
+
                     resolver = Resolver(
                         preparer=preparer,
                         finder=finder,
@@ -350,6 +351,7 @@ def run(self, options, args):
                     # Clean up
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()
 
         if options.target_dir:
             self._handle_target_dir(

src/pip/_internal/commands/wheel.py

@@ -146,35 +146,35 @@ def run(self, options, args):
                     require_hashes=options.require_hashes,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
+                try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
 
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=options.wheel_dir,
-                    progress_bar=options.progress_bar,
-                )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=options.wheel_dir,
+                        progress_bar=options.progress_bar,
+                    )
 
-                resolver = Resolver(
-                    preparer=preparer,
-                    finder=finder,
-                    session=session,
-                    wheel_cache=wheel_cache,
-                    use_user_site=False,
-                    upgrade_strategy="to-satisfy-only",
-                    force_reinstall=False,
-                    ignore_dependencies=options.ignore_dependencies,
-                    ignore_requires_python=options.ignore_requires_python,
-                    ignore_installed=True,
-                    isolated=options.isolated_mode,
-                )
-                resolver.resolve(requirement_set)
+                    resolver = Resolver(
+                        preparer=preparer,
+                        finder=finder,
+                        session=session,
+                        wheel_cache=wheel_cache,
+                        use_user_site=False,
+                        upgrade_strategy="to-satisfy-only",
+                        force_reinstall=False,
+                        ignore_dependencies=options.ignore_dependencies,
+                        ignore_requires_python=options.ignore_requires_python,
+                        ignore_installed=True,
+                        isolated=options.isolated_mode,
+                    )
+                    resolver.resolve(requirement_set)
 
-                try:
                     # build wheels
                     wb = WheelBuilder(
                         requirement_set,
@@ -196,3 +196,4 @@ def run(self, options, args):
                 finally:
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()

src/pip/_internal/wheel.py

@@ -741,6 +741,7 @@ def build(self, session, autobuilding=False):
 
         buildset = []
         for req in reqset:
+            use_ephem_cache = False
             if req.constraint:
                 continue
             if req.is_wheel:
@@ -750,41 +751,46 @@ def build(self, session, autobuilding=False):
             elif autobuilding and req.editable:
                 pass
             elif autobuilding and req.link and not req.link.is_artifact:
-                pass
+                # VCS checkout. Build wheel just for this run.
+                use_ephem_cache = True
             elif autobuilding and not req.source_dir:
                 pass
             else:
                 if autobuilding:
                     link = req.link
                     base, ext = link.splitext()
                     if index.egg_info_matches(base, None, link) is None:
-                        # Doesn't look like a package - don't autobuild a wheel
-                        # because we'll have no way to lookup the result sanely
-                        continue
-                    if "binary" not in index.fmt_ctl_formats(
+                        # E.g. local directory. Build wheel just for this run.
+                        use_ephem_cache = True
+                    elif "binary" not in index.fmt_ctl_formats(
                             self.finder.format_control,
                             canonicalize_name(req.name)):
                         logger.info(
                             "Skipping bdist_wheel for %s, due to binaries "
                             "being disabled for it.", req.name)
                         continue
-                buildset.append(req)
+                buildset.append((req, use_ephem_cache))
 
         if not buildset:
             return True
 
         # Build the wheels.
         logger.info(
             'Building wheels for collected packages: %s',
-            ', '.join([req.name for req in buildset]),
+            ', '.join([req.name for (req, _) in buildset]),
         )
         with indent_log():
             build_success, build_failure = [], []
-            for req in buildset:
+            for req, use_ephem_cache in buildset:
                 python_tag = None
                 if autobuilding:
                     python_tag = pep425tags.implementation_tag
-                    output_dir = self.wheel_cache.get_path_for_link(req.link)
+                    if use_ephem_cache:
+                        output_dir = self.wheel_cache.get_ephem_path_for_link(
+                            req.link)
+                    else:
+                        output_dir = self.wheel_cache.get_path_for_link(
+                            req.link)
                     try:
                         ensure_dir(output_dir)
                     except OSError as e:

tests/functional/test_install.py

@@ -983,16 +983,14 @@ def test_install_builds_wheels(script, data, common_wheels):
     # and built wheels for upper and wheelbroken
     assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
     # wheelbroken has to run install
     # into the cache
-    assert wheels != [], str(res)
     # and installed from the wheel
     assert "Running setup.py install for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # wheelbroken has to run install
     assert "Running setup.py install for wheelb" in str(res), str(res)
     # We want to make sure we used the correct implementation tag
@@ -1016,13 +1014,12 @@ def test_install_no_binary_disables_building_wheels(
     assert expected in str(res), str(res)
     # and built wheels for wheelbroken only
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+    # Wheels are built for local directories, but not cached
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
     # Nor upper, which was blacklisted
     assert "Running setup.py bdist_wheel for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # Wheels are built for local directories, but not cached
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # And these two fell back to sdist based installed.
     assert "Running setup.py install for wheelb" in str(res), str(res)
     assert "Running setup.py install for upper" in str(res), str(res)

tests/functional/test_install_cleanup.py

@@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
     build = script.base_path / 'pip-build'
     script.pip(
         'install', '--no-clean', '--no-index', '--build', build,
-        '--find-links=%s' % data.find_links, 'simple',
+        '--find-links=%s' % data.find_links, 'simple', expect_temp=True,
     )
     assert exists(build)
 
@@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
     result = script.pip(
         'install', '-f', data.find_links, '--no-index', 'simple',
         '--build', build,
-        expect_error=True,
+        expect_error=True, expect_temp=True,
     )
 
     assert result.returncode == PREVIOUS_BUILD_DIR_ERROR

tests/functional/test_install_user.py

@@ -68,24 +68,27 @@ def test_install_subversion_usersite_editable_with_distribute(
         )
         result.assert_installed('INITools', use_user_site=True)
 
-    def test_install_curdir_usersite(self, script, virtualenv, data):
+    @pytest.mark.network
+    def test_install_curdir_usersite(self, script, virtualenv, data,
+                                     common_wheels):
         """
         Test installing current directory ('.') into usersite
         """
         virtualenv.system_site_packages = True
+        script.pip('install', 'wheel', '--no-index', '-f', common_wheels)
         run_from = data.packages.join("FSPkg")
         result = script.pip(
             'install', '-vvv', '--user', curdir,
             cwd=run_from,
             expect_error=False,
         )
         fspkg_folder = script.user_site / 'fspkg'
-        egg_info_folder = (
-            script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion
+        dist_info_folder = (
+            script.user_site / 'FSPkg-0.1.dev0.dist-info'
         )
         assert fspkg_folder in result.files_created, result.stdout
 
-        assert egg_info_folder in result.files_created
+        assert dist_info_folder in result.files_created
 
     def test_install_user_venv_nositepkgs_fails(self, script, data):
         """

tests/functional/test_wheel.py

@@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(
     result = script.pip(
         'wheel', '--no-index', '--find-links=%s' % data.find_links,
         '--build', script.venv_path / 'build',
-        'simple==3.0', expect_error=True,
+        'simple==3.0', expect_error=True, expect_temp=True,
     )
 
     # Then I see that the error code is the right one