heroku · edmorley · Feb 22, 2025 · Feb 21, 2025
@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added build-time rewriting of editable VCS dependency paths (in addition to the existing run-time rewriting), to work around an upstream Pipenv bug with editable VCS dependencies not being reinstalled correctly for cached builds. ([#1756](https://github.com/heroku/heroku-buildpack-python/pull/1756))
 - Changed the location of repositories for editable VCS dependencies when using pip and Pipenv, to improve build performance and match the behaviour when using Poetry. ([#1753](https://github.com/heroku/heroku-buildpack-python/pull/1753))
 
 ## [v277] - 2025-02-17

@@ -280,13 +280,43 @@ if [[ \$HOME != "/app" ]]; then
 fi
 EOT
 
-# At runtime, rewrite paths in editable package .egg-link, .pth and finder files from the build time paths
-# (such as `/tmp/build_<hash>`) back to `/app`. This is not done during the build itself, since later
-# buildpacks still need the build time paths.
+# When dependencies are installed in editable mode, the package manager/build backend creates `.pth`
+# (and related) files in site-packages, which contain absolute path references to the actual location
+# of the packages. By default the Heroku build runs from a directory like `/tmp/build_<hash>`, which
+# changes every build and also differs from the app location at runtime (`/app`). This means any build
+# directory paths referenced in .pth and related files will no longer exist at runtime or during cached
+# rebuilds, unless we rewrite the paths.
+#
+# Ideally, we would be able to rewrite all paths to use the `/app/.heroku/python/` symlink trick we use
+# when invoking Python, since then the same path would work across the current build, runtime and cached
+# rebuilds. However, this trick only works for paths under that directory (since it's not possible to
+# symlink `/app` or other directories we don't own), and when apps use path-based editable dependencies
+# the paths will be outside of that (such as a subdirectory of the app source, or even the root of the
+# build directory). We also can't just rewrite all paths now ready for runtime, since other buildpacks
+# might run after this one that make use of the editable dependencies. As such, we have to perform path
+# rewriting for path-based editable dependencies at app boot instead.
+#
+# For VCS editable dependencies, we can use the symlink trick and so configure the repo checkout location
+# as `/app/.heroku/python/src/`, which in theory should mean the `.pth` files use that path. However,
+# some build backends (such as setuptools' PEP660 implementation) call realpath on it causing the
+# `/tmp/build_*` location to be written instead, meaning VCS src paths need to be rewritten regardless.
+#
+# In addition to ensuring dependencies work for subsequent buildpacks and at runtime, they must also
+# work for cached rebuilds. Most package managers will reinstall editable dependencies regardless on
+# next install, which means we can avoid having to rewrite paths on cache restore from the old build
+# directory to the new location (`/tmp/build_<different-hash>`). However, Pipenv has a bug when using
+# PEP660 style editable VCS dependencies where it won't reinstall if it's missing (or in our case, the
+# path has changed), which means we must make sure that VCS src paths stored in the cache do use the
+# symlink path. See: https://github.com/pypa/pipenv/issues/6348
+#
+# As such, we have to perform two rewrites:
+# 1. At build time, of just the VCS editable paths (which we can safely change to /app paths early).
+# 2. At runtime, to rewrite the remaining path-based editable dependency paths.
 if [[ "${BUILD_DIR}" != "/app" ]]; then
-	cat <<EOT >>"$PROFILE_PATH"
-find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+
-EOT
+	find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e "s#${BUILD_DIR}/.heroku/python#/app/.heroku/python#" {} \+
+	cat <<-EOT >>"${PROFILE_PATH}"
+		find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+
+	EOT
 fi
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.

@@ -102,6 +102,10 @@ function cache::restore() {
 				elif [[ "${cached_pipenv_version}" != "${PIPENV_VERSION:?}" ]]; then
 					cache_invalidation_reasons+=("The Pipenv version has changed from ${cached_pipenv_version} to ${PIPENV_VERSION}")
 				fi
+				# TODO: Remove this next time the Pipenv version is bumped (since it will trigger cache invalidation of its own)
+				if [[ -d "${cache_dir}/.heroku/src" ]]; then
+					cache_invalidation_reasons+=("The editable VCS repository location has changed (and Pipenv doesn't handle this correctly)")
+				fi
 				;;
 			poetry)
 				local cached_poetry_version

@@ -4,7 +4,7 @@ set -euo pipefail
 
 cd .heroku/python/lib/python*/site-packages/
 
-# List any path like strings in .egg-link, .pth, and finder files in site-packages.
+# List any path like strings in the .egg-link, .pth, and finder files in site-packages.
 grep --extended-regexp --only-matching -- '/\S+' *.egg-link *.pth __editable___*_finder.py | sort
 echo
 

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true}
+gunicorn = {git = "git+https://github.com/benoitc/gunicorn", editable = true}
 local-package-pyproject-toml = {file = "packages/local_package_pyproject_toml", editable = true}
 local-package-setup-py = {file = "packages/local_package_setup_py", editable = true}
 pipenv-editable = {file = ".", editable = true}
@@ -4,8 +4,8 @@ set -euo pipefail
 
 cd .heroku/python/lib/python*/site-packages/
 
-# List any path like strings in .egg-link, .pth, and finder files in site-packages.
-grep --extended-regexp --only-matching -- '/\S+' *.egg-link *.pth __editable___*_finder.py | sort
+# List any path like strings in the .pth and finder files in site-packages.
+grep --extended-regexp --only-matching -- '/\S+' *.pth __editable___*_finder.py | sort
 echo
 
 echo -n "Running entrypoint for the pyproject.toml-based local package: "

@@ -1,7 +1,7 @@
 [project]
 name = "pipenv-editable"
 version = "0.0.0"
-requires-python = ">=3.12"
+requires-python = ">=3.13"
 
 [build-system]
 requires = ["hatchling"]

@@ -101,20 +101,20 @@
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/app/.heroku/python/src/gunicorn
           remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/app/.heroku/python/src/gunicorn
           remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
@@ -140,20 +140,20 @@
         app.push!
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        easy-install.pth:/app/.heroku/python/src/gunicorn
           remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote:        gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
           remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: easy-install.pth:/app/.heroku/python/src/gunicorn
           remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
+          remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn
           remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!

@@ -337,6 +337,7 @@
           remote: -----> Discarding cache since:
           remote:        - The Python version has changed from 3.12.4 to #{DEFAULT_PYTHON_FULL_VERSION}
           remote:        - The Pipenv version has changed from 2023.12.1 to #{PIPENV_VERSION}
+          remote:        - The editable VCS repository location has changed \\(and Pipenv doesn't handle this correctly\\)
           remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION}
           remote: -----> Installing pip #{PIP_VERSION}
           remote: -----> Installing Pipenv #{PIPENV_VERSION}
@@ -348,53 +349,45 @@
     end
   end
 
-  # This test has to use Python 3.12 until we work around the Pipenv editable VCS dependency
-  # cache invalidation bug when using pyproject.toml / PEP517 based installs.
   context 'when Pipfile contains editable requirements' do
     let(:buildpacks) { [:default, 'heroku-community/inline'] }
     let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_editable', buildpacks:) }
 
-    it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do
+    it 'rewrites .pth and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do
       app.deploy do |app|
         expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX))
           remote: -----> Installing dependencies using 'pipenv install --deploy'
           remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'}
           remote:        _pipenv_editable.pth:/tmp/build_.+
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
-          remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
+          remote:        Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'}
           remote: _pipenv_editable.pth:/tmp/build_.+
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
-          remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
+          remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\)
         REGEX
 
         # Test rewritten paths work at runtime.
         expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT)
-          easy-install.pth:/app/.heroku/python/src/gunicorn
-          easy-install.pth:/app/packages/local_package_setup_py
+          __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          gunicorn.egg-link:/app/.heroku/python/src/gunicorn
-          local-package-setup-py.egg-link:/app/packages/local_package_setup_py
+          __editable___local_package_setup_py_0_0_1_finder.py:/app/packages/local_package_setup_py/local_package_setup_py'}
           _pipenv_editable.pth:/app
 
           Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           Running entrypoint for the setup.py-based local package: Hello setup.py!
-          Running entrypoint for the VCS package: gunicorn (version 20.1.0)
+          Running entrypoint for the VCS package: gunicorn (version 23.0.0)
         OUTPUT
 
         # Test that the cached .pth files work correctly.
@@ -404,27 +397,23 @@
           remote: -----> Installing dependencies using 'pipenv install --deploy'
           remote:        Installing dependencies from Pipfile.lock \\(.+\\)...
           remote: -----> Running bin/post_compile hook
-          remote:        easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote:        easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote:        __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote:        gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote:        local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote:        __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'}
           remote:        _pipenv_editable.pth:/tmp/build_.+
           remote:        
           remote:        Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote:        Running entrypoint for the setup.py-based local package: Hello setup.py!
-          remote:        Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
+          remote:        Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\)
           remote: -----> Inline app detected
-          remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'}
           remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'}
-          remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn
-          remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py
+          remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'}
           remote: _pipenv_editable.pth:/tmp/build_.+
           remote: 
           remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml!
           remote: Running entrypoint for the setup.py-based local package: Hello setup.py!
-          remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\)
+          remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\)
         REGEX
       end
     end