diff --git a/news/4501.feature b/news/4501.feature new file mode 100644 index 00000000000..a737cc6d10b --- /dev/null +++ b/news/4501.feature @@ -0,0 +1,2 @@ +pip now builds a wheel when installing from a local directory or a VCS URL. +Wheels from these sources are not cached. diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index bd4324b0211..49afefa9bbf 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -11,6 +11,7 @@ from pip._internal import index from pip._internal.compat import expanduser from pip._internal.download import path_to_url +from pip._internal.utils import temp_dir from pip._internal.wheel import InvalidWheelFilename, Wheel logger = logging.getLogger(__name__) @@ -32,6 +33,9 @@ def __init__(self, cache_dir, format_control, allowed_formats): self.cache_dir = expanduser(cache_dir) if cache_dir else None self.format_control = format_control self.allowed_formats = allowed_formats + # Ephemeral cache: store wheels just for this run + self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache") + self._ephem_cache_dir.create() _valid_formats = {"source", "binary"} assert self.allowed_formats.union(_valid_formats) == _valid_formats @@ -102,6 +106,10 @@ def _link_for_candidate(self, link, candidate): return index.Link(path_to_url(path)) + def cleanup(self): + """Remove the ephermal caches created temporarily to build wheels""" + self._ephem_cache_dir.cleanup() + class WheelCache(Cache): """A cache of wheels for future installs. @@ -131,6 +139,18 @@ def get_path_for_link(self, link): # join them all together. return os.path.join(self.cache_dir, "wheels", *parts) + def get_ephem_path_for_link(self, link): + """Return a directory to store cached wheels for link + + Unlike get_path_for_link, the directory will be removed + before pip exits. + """ + parts = self._get_cache_path_parts(link) + + # Inside of the base location for cached wheels, expand our parts and + # join them all together. + return os.path.join(self._ephem_cache_dir.path, "wheels", *parts) + def get(self, link, package_name): candidates = [] diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 2ac0bf9ca7e..7707894b091 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -88,5 +88,8 @@ def run(self, options, args): skip=skip, exclude_editable=options.exclude_editable) - for line in freeze(**freeze_kwargs): - sys.stdout.write(line + '\n') + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index b7575e52620..e439aacc639 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -247,18 +247,19 @@ def run(self, options, args): use_user_site=options.use_user_site, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -350,6 +351,7 @@ def run(self, options, args): # Clean up if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index e03983cd124..413830c585d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -146,35 +146,35 @@ def run(self, options, args): require_hashes=options.require_hashes, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=options.wheel_dir, + progress_bar=options.progress_bar, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = Resolver( + preparer=preparer, + finder=finder, + session=session, + wheel_cache=wheel_cache, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=options.ignore_dependencies, + ignore_requires_python=options.ignore_requires_python, + ignore_installed=True, + isolated=options.isolated_mode, + ) + resolver.resolve(requirement_set) - try: # build wheels wb = WheelBuilder( requirement_set, @@ -196,3 +196,4 @@ def run(self, options, args): finally: if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index a38e16eb949..35d96153d31 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -741,6 +741,7 @@ def build(self, session, autobuilding=False): buildset = [] for req in reqset: + use_ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -750,7 +751,8 @@ def build(self, session, autobuilding=False): elif autobuilding and req.editable: pass elif autobuilding and req.link and not req.link.is_artifact: - pass + # VCS checkout. Build wheel just for this run. + use_ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -758,17 +760,16 @@ def build(self, session, autobuilding=False): link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: - # Doesn't look like a package - don't autobuild a wheel - # because we'll have no way to lookup the result sanely - continue - if "binary" not in index.fmt_ctl_formats( + # E.g. local directory. Build wheel just for this run. + use_ephem_cache = True + elif "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue - buildset.append(req) + buildset.append((req, use_ephem_cache)) if not buildset: return True @@ -776,15 +777,20 @@ def build(self, session, autobuilding=False): # Build the wheels. logger.info( 'Building wheels for collected packages: %s', - ', '.join([req.name for req in buildset]), + ', '.join([req.name for (req, _) in buildset]), ) with indent_log(): build_success, build_failure = [], [] - for req in buildset: + for req, use_ephem_cache in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag - output_dir = self.wheel_cache.get_path_for_link(req.link) + if use_ephem_cache: + output_dir = self.wheel_cache.get_ephem_path_for_link( + req.link) + else: + output_dir = self.wheel_cache.get_path_for_link( + req.link) try: ensure_dir(output_dir) except OSError as e: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index dee56c7d7ce..c8f600118a0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -983,16 +983,14 @@ def test_install_builds_wheels(script, data, common_wheels): # and built wheels for upper and wheelbroken assert "Running setup.py bdist_wheel for upper" in str(res), str(res) assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache - assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) # We want to make sure we used the correct implementation tag @@ -1016,13 +1014,12 @@ def test_install_no_binary_disables_building_wheels( assert expected in str(res), str(res) # and built wheels for wheelbroken only assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # Nor upper, which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. assert "Running setup.py install for wheelb" in str(res), str(res) assert "Running setup.py install for upper" in str(res), str(res) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index a2c2644fd2c..606b32cac70 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): build = script.base_path / 'pip-build' script.pip( 'install', '--no-clean', '--no-index', '--build', build, - '--find-links=%s' % data.find_links, 'simple', + '--find-links=%s' % data.find_links, 'simple', expect_temp=True, ) assert exists(build) @@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data): result = script.pip( 'install', '-f', data.find_links, '--no-index', 'simple', '--build', build, - expect_error=True, + expect_error=True, expect_temp=True, ) assert result.returncode == PREVIOUS_BUILD_DIR_ERROR diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index cf0cb513773..fe748600a3d 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -68,11 +68,14 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) - def test_install_curdir_usersite(self, script, virtualenv, data): + @pytest.mark.network + def test_install_curdir_usersite(self, script, virtualenv, data, + common_wheels): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True + script.pip('install', 'wheel', '--no-index', '-f', common_wheels) run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, @@ -80,12 +83,12 @@ def test_install_curdir_usersite(self, script, virtualenv, data): expect_error=False, ) fspkg_folder = script.user_site / 'fspkg' - egg_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion + dist_info_folder = ( + script.user_site / 'FSPkg-0.1.dev0.dist-info' ) assert fspkg_folder in result.files_created, result.stdout - assert egg_info_folder in result.files_created + assert dist_info_folder in result.files_created def test_install_user_venv_nositepkgs_fails(self, script, data): """ diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index e3e30173c41..58b7283fdad 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir( result = script.pip( 'wheel', '--no-index', '--find-links=%s' % data.find_links, '--build', script.venv_path / 'build', - 'simple==3.0', expect_error=True, + 'simple==3.0', expect_error=True, expect_temp=True, ) # Then I see that the error code is the right one