From 3517df5011e2a00eeff68ed4fa1f72849066f7ca Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:01:10 +0100 Subject: [PATCH 01/30] Try to build wheels for directories/VCS Follow up from gh-4144 To allow build system abstractions, we want installation to go through wheels in more cases. In particular, installing packages from a local directory or a VCS URL currently runs 'setup.py install'. The aim of this PR is to have it build a wheel, which is stored in an ephemeral cache directory, used for installation, and then discarded. We can't cache it permanently based on the path/URL, because the code there might change, but our cache wouldn't be invalidated. --- src/pip/_internal/commands/wheel.py | 52 ++++++++++++++--------------- src/pip/_internal/req/req_set.py | 3 ++ src/pip/_internal/wheel.py | 18 +++++----- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index e03983cd124..44ab05c4ac9 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -146,35 +146,35 @@ def run(self, options, args): require_hashes=options.require_hashes, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=options.wheel_dir, + progress_bar=options.progress_bar, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = Resolver( + preparer=preparer, + finder=finder, + session=session, + wheel_cache=wheel_cache, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=options.ignore_dependencies, + ignore_requires_python=options.ignore_requires_python, + ignore_installed=True, + isolated=options.isolated_mode, + ) + resolver.resolve(requirement_set) - try: # build wheels wb = WheelBuilder( requirement_set, diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index 3c8e23510fd..3cc1fa6e56f 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -169,6 +169,9 @@ def cleanup_files(self): for req in self.reqs_to_cleanup: req.remove_temporary_source() + if self._wheel_cache: + self._wheel_cache.cleanup() + def _to_install(self): """Create the installation order. diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index a38e16eb949..44023732515 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -741,6 +741,7 @@ def build(self, session, autobuilding=False): buildset = [] for req in reqset: + ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -750,7 +751,8 @@ def build(self, session, autobuilding=False): elif autobuilding and req.editable: pass elif autobuilding and req.link and not req.link.is_artifact: - pass + # VCS checkout. Build wheel just for this run. + ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -758,17 +760,16 @@ def build(self, session, autobuilding=False): link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: - # Doesn't look like a package - don't autobuild a wheel - # because we'll have no way to lookup the result sanely - continue - if "binary" not in index.fmt_ctl_formats( + # E.g. local directory. Build wheel just for this run. + ephem_cache = True + elif "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue - buildset.append(req) + buildset.append((req, ephem_cache)) if not buildset: return True @@ -780,11 +781,12 @@ def build(self, session, autobuilding=False): ) with indent_log(): build_success, build_failure = [], [] - for req in buildset: + for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag - output_dir = self.wheel_cache.get_path_for_link(req.link) + output_dir = self.wheel_cache.get_path_for_link(req.link, + ephem=ephem) try: ensure_dir(output_dir) except OSError as e: From b241c227937cd381058d51843a7a8387d1df7f43 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:06:07 +0100 Subject: [PATCH 02/30] Add news file --- news/4501.feature | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/4501.feature diff --git a/news/4501.feature b/news/4501.feature new file mode 100644 index 00000000000..475a1317224 --- /dev/null +++ b/news/4501.feature @@ -0,0 +1,3 @@ +Installing from a local directory or a VCS URL now builds a wheel to install, +rather than running ``setup.py install``. Wheels from these sources are not +cached. From fa927a60340d70bd3c9dff0f2b0f4059cd20197b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:08:45 +0100 Subject: [PATCH 03/30] Fix list comp --- src/pip/_internal/wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 44023732515..f49b86ac33d 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -777,7 +777,7 @@ def build(self, session, autobuilding=False): # Build the wheels. logger.info( 'Building wheels for collected packages: %s', - ', '.join([req.name for req in buildset]), + ', '.join([req.name for (req, _) in buildset]), ) with indent_log(): build_success, build_failure = [], [] From 48ee5d76c002ddcaa26383b54a4aa8af10f7160f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:26:06 +0100 Subject: [PATCH 04/30] Allow temp files for another test When there's a previous build directory, no_clean is set --- tests/functional/test_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index e3e30173c41..58b7283fdad 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir( result = script.pip( 'wheel', '--no-index', '--find-links=%s' % data.find_links, '--build', script.venv_path / 'build', - 'simple==3.0', expect_error=True, + 'simple==3.0', expect_error=True, expect_temp=True, ) # Then I see that the error code is the right one From b229fd40be34675208d09b112192703387112717 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:41:51 +0100 Subject: [PATCH 05/30] Fix cleanup on 'pip install' with no args --- src/pip/_internal/commands/install.py | 33 +++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index b7575e52620..5a7b192099b 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -247,17 +247,6 @@ def run(self, options, args): use_user_site=options.use_user_site, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) try: resolver = Resolver( preparer=preparer, @@ -274,10 +263,24 @@ def run(self, options, args): ) resolver.resolve(requirement_set) - # on -d don't do complex things like building - # wheels, and don't try to build wheels when wheel is - # not installed. - if wheel and options.cache_dir: + self.populate_requirement_set( + requirement_set, args, options, finder, session, self.name, + wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + + if (not wheel or not options.cache_dir): + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + requirement_set.prepare_files(finder) + else: # build wheels before install. wb = WheelBuilder( requirement_set, From c5297e803541a7f053bce88bef361461aefe4bd2 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:02:44 +0100 Subject: [PATCH 06/30] Cleanup wheel cache after 'pip freeze' --- src/pip/_internal/commands/freeze.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 2ac0bf9ca7e..7707894b091 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -88,5 +88,8 @@ def run(self, options, args): skip=skip, exclude_editable=options.exclude_editable) - for line in freeze(**freeze_kwargs): - sys.stdout.write(line + '\n') + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() From 8fbd8f44fad7d85522b85ed92d94a8e2353c7e66 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:06:20 +0100 Subject: [PATCH 07/30] Expect temp files in another test --- tests/functional/test_install_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index a2c2644fd2c..834ccb06802 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): build = script.base_path / 'pip-build' script.pip( 'install', '--no-clean', '--no-index', '--build', build, - '--find-links=%s' % data.find_links, 'simple', + '--find-links=%s' % data.find_links, 'simple', expect_temp=True, ) assert exists(build) From e58e50fd28a639ee624e1920f31a4f32174bcbde Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:10:32 +0100 Subject: [PATCH 08/30] Update test of install output --- tests/functional/test_install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index dee56c7d7ce..4ebceb72715 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -983,16 +983,15 @@ def test_install_builds_wheels(script, data, common_wheels): # and built wheels for upper and wheelbroken assert "Running setup.py bdist_wheel for upper" in str(res), str(res) assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) # We want to make sure we used the correct implementation tag From 2c67a916683ba459d6a10240db2cde7e4da5e385 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:11:06 +0100 Subject: [PATCH 09/30] Expect temp files on another test --- tests/functional/test_install_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 834ccb06802..606b32cac70 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data): result = script.pip( 'install', '-f', data.find_links, '--no-index', 'simple', '--build', build, - expect_error=True, + expect_error=True, expect_temp=True, ) assert result.returncode == PREVIOUS_BUILD_DIR_ERROR From ef1a3ae868d3ba1cdf547a8e9f479dae48cf3407 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:12:28 +0100 Subject: [PATCH 10/30] Update another install output test --- tests/functional/test_install.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 4ebceb72715..2dc55502bd0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1015,13 +1015,15 @@ def test_install_no_binary_disables_building_wheels( assert expected in str(res), str(res) # and built wheels for wheelbroken only assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # Nor upper, which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # wheelbroken has to run install + # into the cache + assert wheels != [], str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. assert "Running setup.py install for wheelb" in str(res), str(res) assert "Running setup.py install for upper" in str(res), str(res) From a7b0e4f44b178c880721b98e0ae35634771c243b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 21:49:36 +0100 Subject: [PATCH 11/30] Update test to check for dist-info --- tests/functional/test_install_user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index cf0cb513773..749e1663ecb 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -80,12 +80,12 @@ def test_install_curdir_usersite(self, script, virtualenv, data): expect_error=False, ) fspkg_folder = script.user_site / 'fspkg' - egg_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion + dist_info_folder = ( + script.user_site / 'FSPkg-0.1.dev0.dist-info' % pyversion ) assert fspkg_folder in result.files_created, result.stdout - assert egg_info_folder in result.files_created + assert dist_info_folder in result.files_created def test_install_user_venv_nositepkgs_fails(self, script, data): """ From 58f69176f427000d949a518a2b293be9952e87f4 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 22:21:29 +0100 Subject: [PATCH 12/30] Remove string formatting operation --- tests/functional/test_install_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 749e1663ecb..ce3b2fd2d0c 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -81,7 +81,7 @@ def test_install_curdir_usersite(self, script, virtualenv, data): ) fspkg_folder = script.user_site / 'fspkg' dist_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0.dist-info' % pyversion + script.user_site / 'FSPkg-0.1.dev0.dist-info' ) assert fspkg_folder in result.files_created, result.stdout From 711c99aadda305c5a45dd3c1bd52a6ef1db82f3a Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 20 May 2017 09:43:51 +0100 Subject: [PATCH 13/30] Install 'wheel' for test that now requires it --- tests/functional/test_install_user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index ce3b2fd2d0c..f3ab4b70674 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -68,11 +68,13 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) + @pytest.mark.network def test_install_curdir_usersite(self, script, virtualenv, data): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True + script.pip("install", "wheel") run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, From 25d55460c05354c7dd53803af56d06f2f4528e02 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 13:15:32 -0500 Subject: [PATCH 14/30] fix merge issues --- src/pip/_internal/commands/install.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 5a7b192099b..0bd6c7c590a 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -248,6 +248,11 @@ def run(self, options, args): ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -275,12 +280,10 @@ def run(self, options, args): progress_bar=options.progress_bar, ) - if (not wheel or not options.cache_dir): - # on -d don't do complex things like building - # wheels, and don't try to build wheels when wheel is - # not installed. - requirement_set.prepare_files(finder) - else: + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + if wheel and options.cache_dir: # build wheels before install. wb = WheelBuilder( requirement_set, From 882c8b9f62fe945e0eace8f2145a5930aeeb1af9 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 26 Jun 2017 07:33:11 -0400 Subject: [PATCH 15/30] flake8 --- src/pip/_internal/commands/install.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 0bd6c7c590a..4ea8dd91bf6 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -248,11 +248,6 @@ def run(self, options, args): ) try: - self.populate_requirement_set( - requirement_set, args, options, finder, session, - self.name, wheel_cache - ) - resolver = Resolver( preparer=preparer, finder=finder, From ade439cac26c7f346c8da6e9a2128120ee4512a0 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:08:37 -0400 Subject: [PATCH 16/30] fix --- src/pip/_internal/commands/install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 4ea8dd91bf6..3c100493a68 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -248,6 +248,18 @@ def run(self, options, args): ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, self.name, + wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -263,18 +275,6 @@ def run(self, options, args): ) resolver.resolve(requirement_set) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) - # on -d don't do complex things like building # wheels, and don't try to build wheels when wheel is # not installed. From 396abfe7fbcd1552dfca753e3b5d2b9f85bad89d Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:12:38 -0400 Subject: [PATCH 17/30] not a thing anymore --- src/pip/_internal/req/req_set.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index 3cc1fa6e56f..af20e3cc687 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -16,9 +16,6 @@ def __init__(self, require_hashes=False, target_dir=None, use_user_site=False, pycompile=True): """Create a RequirementSet. - - :param wheel_cache: The pip wheel cache, for passing to - InstallRequirement. """ self.requirements = OrderedDict() @@ -169,9 +166,6 @@ def cleanup_files(self): for req in self.reqs_to_cleanup: req.remove_temporary_source() - if self._wheel_cache: - self._wheel_cache.cleanup() - def _to_install(self): """Create the installation order. From dd326a7507eb4ed9ce43b180f4ef9b9f80778c5f Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:15:21 -0400 Subject: [PATCH 18/30] these are probably needed --- src/pip/_internal/commands/install.py | 1 + src/pip/_internal/commands/wheel.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 3c100493a68..f4c45771abe 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -351,6 +351,7 @@ def run(self, options, args): # Clean up if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 44ab05c4ac9..413830c585d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -196,3 +196,4 @@ def run(self, options, args): finally: if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() From 951dfdd8f5b5a575f76e7e4cfabe9f3076dd089a Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:16:06 -0400 Subject: [PATCH 19/30] flake8 --- src/pip/_internal/commands/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index f4c45771abe..e439aacc639 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -249,8 +249,8 @@ def run(self, options, args): try: self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache + requirement_set, args, options, finder, session, + self.name, wheel_cache ) preparer = RequirementPreparer( build_dir=directory.path, From e168cbb5b1cb86509eca0f2564764f9a405d3297 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 13:27:38 -0500 Subject: [PATCH 20/30] cache: implement epherm cache directory and cleanup --- src/pip/_internal/cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index bd4324b0211..5d44a4c3826 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -12,6 +12,7 @@ from pip._internal.compat import expanduser from pip._internal.download import path_to_url from pip._internal.wheel import InvalidWheelFilename, Wheel +from pip._internal.utils import temp_dir logger = logging.getLogger(__name__) @@ -32,6 +33,9 @@ def __init__(self, cache_dir, format_control, allowed_formats): self.cache_dir = expanduser(cache_dir) if cache_dir else None self.format_control = format_control self.allowed_formats = allowed_formats + # Ephemeral cache: store wheels just for this run + self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache") + self._ephem_cache_dir.create() _valid_formats = {"source", "binary"} assert self.allowed_formats.union(_valid_formats) == _valid_formats @@ -102,6 +106,9 @@ def _link_for_candidate(self, link, candidate): return index.Link(path_to_url(path)) + def cleanup(self): + """Remove the ephermal caches created temporarily to build wheels""" + self._ephem_cache_dir.cleanup() class WheelCache(Cache): """A cache of wheels for future installs. From dbcd1a7322fde7fdef5bad7975283a2e5ab5863c Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 13:31:23 -0500 Subject: [PATCH 21/30] cache: add epem parameter --- src/pip/_internal/cache.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 5d44a4c3826..9938c46a74a 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -117,7 +117,7 @@ class WheelCache(Cache): def __init__(self, cache_dir, format_control): super(WheelCache, self).__init__(cache_dir, format_control, {"binary"}) - def get_path_for_link(self, link): + def get_path_for_link(self, link, ephem=False): """Return a directory to store cached wheels for link Because there are M wheels for any one sdist, we provide a directory @@ -134,9 +134,14 @@ def get_path_for_link(self, link): """ parts = self._get_cache_path_parts(link) + if not ephem: + cache_dir = self.cache_dir + else: + cache_dir = self._ephem_cache_dir.path + # Inside of the base location for cached wheels, expand our parts and # join them all together. - return os.path.join(self.cache_dir, "wheels", *parts) + return os.path.join(cache_dir, "wheels", *parts) def get(self, link, package_name): candidates = [] From c28d6140d635e80005a292c4199e8c440e37a439 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 14:26:35 -0500 Subject: [PATCH 22/30] tests: install: fix merge errors --- tests/functional/test_install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 2dc55502bd0..ba3cd0d1e11 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -987,7 +987,6 @@ def test_install_builds_wheels(script, data, common_wheels): assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache - assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) # Wheels are built for local directories, but not cached. From bd31280784c6bb06b9e603b12c36cd57e8aa7a93 Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 14:28:50 -0500 Subject: [PATCH 23/30] :art: --- src/pip/_internal/cache.py | 1 + src/pip/_internal/wheel.py | 4 ++-- tests/functional/test_install.py | 3 --- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 9938c46a74a..845559f4b99 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -110,6 +110,7 @@ def cleanup(self): """Remove the ephermal caches created temporarily to build wheels""" self._ephem_cache_dir.cleanup() + class WheelCache(Cache): """A cache of wheels for future installs. """ diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index f49b86ac33d..8d0d75b1160 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -785,8 +785,8 @@ def build(self, session, autobuilding=False): python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag - output_dir = self.wheel_cache.get_path_for_link(req.link, - ephem=ephem) + output_dir = self.wheel_cache.get_path_for_link( + req.link, ephem=ephem) try: ensure_dir(output_dir) except OSError as e: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index ba3cd0d1e11..c8f600118a0 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1018,9 +1018,6 @@ def test_install_no_binary_disables_building_wheels( assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # Nor upper, which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) - # wheelbroken has to run install - # into the cache - assert wheels != [], str(res) # Wheels are built for local directories, but not cached assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. From b6377d47e37cbaa1111142b6322e5964ec8efafd Mon Sep 17 00:00:00 2001 From: xoviat Date: Thu, 7 Sep 2017 14:32:35 -0500 Subject: [PATCH 24/30] :art: --- src/pip/_internal/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 845559f4b99..2d1af17f7af 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -11,8 +11,8 @@ from pip._internal import index from pip._internal.compat import expanduser from pip._internal.download import path_to_url -from pip._internal.wheel import InvalidWheelFilename, Wheel from pip._internal.utils import temp_dir +from pip._internal.wheel import InvalidWheelFilename, Wheel logger = logging.getLogger(__name__) From edcc6de4a092b47b3ef8aface289a20806823fb6 Mon Sep 17 00:00:00 2001 From: xoviat Date: Fri, 8 Sep 2017 10:58:57 -0500 Subject: [PATCH 25/30] tests: user: use common_wheels fixture --- tests/functional/test_install_user.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index f3ab4b70674..fe748600a3d 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -69,12 +69,13 @@ def test_install_subversion_usersite_editable_with_distribute( result.assert_installed('INITools', use_user_site=True) @pytest.mark.network - def test_install_curdir_usersite(self, script, virtualenv, data): + def test_install_curdir_usersite(self, script, virtualenv, data, + common_wheels): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True - script.pip("install", "wheel") + script.pip('install', 'wheel', '--no-index', '-f', common_wheels) run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, From 97c667b3d2b4f1e2d1814e42320182802b6f81d2 Mon Sep 17 00:00:00 2001 From: xoviat Date: Fri, 8 Sep 2017 11:03:24 -0500 Subject: [PATCH 26/30] wheel: rename ephen_cache flag --- src/pip/_internal/wheel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 8d0d75b1160..ff6ba9485ea 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -741,7 +741,7 @@ def build(self, session, autobuilding=False): buildset = [] for req in reqset: - ephem_cache = False + use_ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -752,7 +752,7 @@ def build(self, session, autobuilding=False): pass elif autobuilding and req.link and not req.link.is_artifact: # VCS checkout. Build wheel just for this run. - ephem_cache = True + use_ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -761,7 +761,7 @@ def build(self, session, autobuilding=False): base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. - ephem_cache = True + use_ephem_cache = True elif "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): @@ -769,7 +769,7 @@ def build(self, session, autobuilding=False): "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue - buildset.append((req, ephem_cache)) + buildset.append((req, use_ephem_cache)) if not buildset: return True From 43eb3c6f87198d70d14d0fe8c7fca3074719d43c Mon Sep 17 00:00:00 2001 From: xoviat Date: Fri, 8 Sep 2017 11:12:29 -0500 Subject: [PATCH 27/30] :newspaper: --- news/4501.feature | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/news/4501.feature b/news/4501.feature index 475a1317224..a737cc6d10b 100644 --- a/news/4501.feature +++ b/news/4501.feature @@ -1,3 +1,2 @@ -Installing from a local directory or a VCS URL now builds a wheel to install, -rather than running ``setup.py install``. Wheels from these sources are not -cached. +pip now builds a wheel when installing from a local directory or a VCS URL. +Wheels from these sources are not cached. From 00281b32bd0c66c0a7a0e45c5ed836ac93547353 Mon Sep 17 00:00:00 2001 From: xoviat Date: Fri, 8 Sep 2017 11:16:37 -0500 Subject: [PATCH 28/30] cache: move ephem flag into two methods --- src/pip/_internal/cache.py | 19 +++++++++++++------ src/pip/_internal/wheel.py | 10 +++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 2d1af17f7af..3046a68f10e 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -118,7 +118,7 @@ class WheelCache(Cache): def __init__(self, cache_dir, format_control): super(WheelCache, self).__init__(cache_dir, format_control, {"binary"}) - def get_path_for_link(self, link, ephem=False): + def get_path_for_link(self, link): """Return a directory to store cached wheels for link Because there are M wheels for any one sdist, we provide a directory @@ -135,14 +135,21 @@ def get_path_for_link(self, link, ephem=False): """ parts = self._get_cache_path_parts(link) - if not ephem: - cache_dir = self.cache_dir - else: - cache_dir = self._ephem_cache_dir.path + # Inside of the base location for cached wheels, expand our parts and + # join them all together. + return os.path.join(self.cache_dir, "wheels", *parts) + + def get_ephem_path_for_link(self, link): + """Return a driectory to store cached wheels for link + + Unlike get_path_for_link, the directory will be removed + before pip exits. + """ + parts = self._get_cache_path_parts(link) # Inside of the base location for cached wheels, expand our parts and # join them all together. - return os.path.join(cache_dir, "wheels", *parts) + return os.path.join(self._ephem_cache_dir.path, "wheels", *parts) def get(self, link, package_name): candidates = [] diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index ff6ba9485ea..35d96153d31 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -781,12 +781,16 @@ def build(self, session, autobuilding=False): ) with indent_log(): build_success, build_failure = [], [] - for req, ephem in buildset: + for req, use_ephem_cache in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag - output_dir = self.wheel_cache.get_path_for_link( - req.link, ephem=ephem) + if use_ephem_cache: + output_dir = self.wheel_cache.get_ephem_path_for_link( + req.link) + else: + output_dir = self.wheel_cache.get_path_for_link( + req.link) try: ensure_dir(output_dir) except OSError as e: From 02a5d5874308642bfb02fe2a357a32b823ead305 Mon Sep 17 00:00:00 2001 From: xoviat Date: Sat, 9 Sep 2017 14:52:47 -0500 Subject: [PATCH 29/30] req_set: revert 396abfe7fbcd1552dfca753e3b5d2b9f85bad89d. --- src/pip/_internal/req/req_set.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index af20e3cc687..3c8e23510fd 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -16,6 +16,9 @@ def __init__(self, require_hashes=False, target_dir=None, use_user_site=False, pycompile=True): """Create a RequirementSet. + + :param wheel_cache: The pip wheel cache, for passing to + InstallRequirement. """ self.requirements = OrderedDict() From fb09a604082b9ff021579be7a10639d53bd5c1bf Mon Sep 17 00:00:00 2001 From: xoviat Date: Sun, 24 Sep 2017 19:35:20 -0500 Subject: [PATCH 30/30] cache: fix spelling error --- src/pip/_internal/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 3046a68f10e..49afefa9bbf 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -140,7 +140,7 @@ def get_path_for_link(self, link): return os.path.join(self.cache_dir, "wheels", *parts) def get_ephem_path_for_link(self, link): - """Return a driectory to store cached wheels for link + """Return a directory to store cached wheels for link Unlike get_path_for_link, the directory will be removed before pip exits.