Change the editable VCS directory location for pip and Pipenv

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Changed the location of repositories for editable VCS dependencies when using pip and Pipenv, to improve build performance and match the behaviour when using Poetry. ([#1753](https://github.com/heroku/heroku-buildpack-python/pull/1753))
 
 ## [v277] - 2025-02-17
 

bin/compile

@@ -178,8 +178,6 @@ cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK}" "${cached_python_full_ve
 
 # The directory for the .profile.d scripts.
 mkdir -p "$(dirname "$PROFILE_PATH")"
-# The directory for editable VCS dependencies.
-mkdir -p /app/.heroku/src
 
 # On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
 # they occur in a temp directory. Because Python is not portable, we must create
@@ -191,7 +189,6 @@ if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then
 	# python expects to reside in /app, so set up symlinks
 	# we will not remove these later so subsequent buildpacks can still invoke it
 	ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python
-	# Note: .heroku/src is copied in later.
 fi
 
 python::install "${BUILD_DIR}" "${STACK}" "${python_full_version}" "${python_major_version}" "${python_version_origin}"
@@ -250,15 +247,6 @@ nltk_downloader_start_time=$(nowms)
 sub_env "${BUILDPACK_DIR}/bin/steps/nltk"
 meta_time "nltk_downloader_duration" "${nltk_downloader_start_time}"
 
-# Support for editable installations.
-# In CI, $BUILD_DIR is /app.
-# Realpath is used to support use-cases where one of the locations is a symlink to the other.
-# shellcheck disable=SC2312 # TODO: Invoke this command separately to avoid masking its return value.
-if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then
-	rm -rf "$BUILD_DIR/.heroku/src"
-	deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
-fi
-
 # Django collectstatic support.
 # The buildpack automatically runs collectstatic for Django applications.
 collectstatic_start_time=$(nowms)

bin/utils

@@ -8,22 +8,6 @@ shopt -s nullglob
 
 source "${BUILDPACK_DIR:?}/vendor/buildpack-stdlib_v8.sh"
 
-# Does some serious copying.
-deep-cp() {
-	declare source="$1" target="$2"
-
-	mkdir -p "$target"
-
-	# cp doesn't like being called without source params,
-	# so make sure they expand to something first.
-	# subshell to avoid surprising caller with shopts.
-	(
-		shopt -s nullglob dotglob
-		set -- "$source"/!(tmp|.|..)
-		[[ $# == 0 ]] || cp -a "$@" "$target"
-	)
-}
-
 # Measure the size of the Python installation.
 measure-size() {
 	{ du -s .heroku/python 2>/dev/null || echo 0; } | awk '{print $1}'

lib/cache.sh

@@ -129,7 +129,6 @@ function cache::restore() {
 			"${cache_dir}/.heroku/python-poetry" \
 			"${cache_dir}/.heroku/python-stack" \
 			"${cache_dir}/.heroku/python-version" \
-			"${cache_dir}/.heroku/src" \
 			"${cache_dir}/.heroku/requirements.txt"
 
 		meta_set "cache_status" "discarded"
@@ -143,17 +142,13 @@ function cache::restore() {
 		# TODO: Compare the performance of moving the directory vs copying files.
 		cp -R "${cache_dir}/.heroku/python" "${build_dir}/.heroku/" &>/dev/null || true
 
-		# Editable VCS code repositories, used by pip/pipenv.
-		if [[ -d "${cache_dir}/.heroku/src" ]]; then
-			cp -R "${cache_dir}/.heroku/src" "${build_dir}/.heroku/" &>/dev/null || true
-		fi
-
 		meta_set "cache_status" "reused"
 	fi
 
 	# Remove any legacy cache contents written by older buildpack versions.
 	rm -rf \
 		"${cache_dir}/.heroku/python-sqlite3-version" \
+		"${cache_dir}/.heroku/src" \
 		"${cache_dir}/.heroku/vendor"
 
 	meta_time "cache_restore_duration" "${cache_restore_start_time}"
@@ -175,13 +170,6 @@ function cache::save() {
 	rm -rf "${cache_dir}/.heroku/python"
 	cp -R "${build_dir}/.heroku/python" "${cache_dir}/.heroku/"
 
-	# Editable VCS code repositories, used by pip/pipenv.
-	rm -rf "${cache_dir}/.heroku/src"
-	if [[ -d "${build_dir}/.heroku/src" ]]; then
-		# TODO: Investigate why errors are ignored and ideally stop doing so.
-		cp -R "${build_dir}/.heroku/src" "${cache_dir}/.heroku/" &>/dev/null || true
-	fi
-
 	# Metadata used by subsequent builds to determine whether the cache can be reused.
 	# These are written/consumed via separate files and not the metadata store for compatibility
 	# with buildpack versions prior to the metadata store existing (which was only added in v252).

lib/pip.sh

@@ -110,7 +110,7 @@ function pip::install_dependencies() {
 			--no-cache-dir \
 			--no-input \
 			--progress-bar off \
-			--src='/app/.heroku/src' \
+			--src='/app/.heroku/python/src' \
 			|& tee "${WARNINGS_LOG:?}" \
 			|& sed --unbuffered --expression '/Requirement already satisfied/d' \
 			|& output::indent

lib/pipenv.sh

@@ -81,7 +81,7 @@ function pipenv::install_dependencies() {
 	# shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled.
 	if ! {
 		"${pipenv_install_command[@]}" \
-			--extra-pip-args='--src=/app/.heroku/src' \
+			--extra-pip-args='--src=/app/.heroku/python/src' \
 			--system \
 			|& tee "${WARNINGS_LOG:?}" \
 			|& output::indent

spec/hatchet/pip_spec.rb

@@ -100,21 +100,21 @@
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/app/.heroku/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/app/.heroku/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -123,10 +123,10 @@
 
         # Test rewritten paths work at runtime.
         expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT)
-          easy-install.pth:/app/.heroku/src/gunicorn
+          easy-install.pth:/app/.heroku/python/src/gunicorn
           easy-install.pth:/app/packages/local_package_setup_py
           __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          gunicorn.egg-link:/app/.heroku/src/gunicorn
+          gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           local-package-setup-py.egg-link:/app/packages/local_package_setup_py
 
           Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
@@ -139,21 +139,21 @@
         app.push!
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/app/.heroku/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/app/.heroku/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!

spec/hatchet/pipenv_spec.rb

@@ -356,21 +356,21 @@
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/app/.heroku/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/app/.heroku/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
@@ -379,10 +379,10 @@
 
         # Test rewritten paths work at runtime.
         expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT)
-          easy-install.pth:/app/.heroku/src/gunicorn
+          easy-install.pth:/app/.heroku/python/src/gunicorn
           easy-install.pth:/app/packages/local_package_setup_py
           __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          gunicorn.egg-link:/app/.heroku/src/gunicorn
+          gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           local-package-setup-py.egg-link:/app/packages/local_package_setup_py
 
           Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
@@ -395,21 +395,21 @@
         app.push!
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/app/.heroku/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/app/.heroku/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py
-          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/app/.heroku/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py
+          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
+          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!