Merge pull request #8212 from sbidoul/pep517-editable-sbi

PEP 660 (build_editable) support

Author: sbidoul

news/8212.feature.rst

@@ -0,0 +1,2 @@
+Support editable installs for projects that have a ``pyproject.toml`` and use a
+build backend that supports :pep:`660`.

src/pip/_internal/commands/install.py

@@ -306,6 +306,12 @@ def run(self, options: Values, args: List[str]) -> int:
         try:
             reqs = self.get_requirements(args, options, finder, session)
 
+            # Only when installing is it permitted to use PEP 660.
+            # In other circumstances (pip wheel, pip download) we generate
+            # regular (i.e. non editable) metadata and wheels.
+            for req in reqs:
+                req.permit_editable_wheels = True
+
             reject_location_related_install_options(reqs, options.install_options)
 
             preparer = self.make_requirement_preparer(
@@ -361,22 +367,22 @@ def run(self, options: Values, args: List[str]) -> int:
                 global_options=[],
             )
 
-            # If we're using PEP 517, we cannot do a direct install
+            # If we're using PEP 517, we cannot do a legacy setup.py install
             # so we fail here.
             pep517_build_failure_names: List[str] = [
                 r.name for r in build_failures if r.use_pep517  # type: ignore
             ]
             if pep517_build_failure_names:
                 raise InstallationError(
-                    "Could not build wheels for {} which use"
-                    " PEP 517 and cannot be installed directly".format(
+                    "Could not build wheels for {}, which is required to "
+                    "install pyproject.toml-based projects".format(
                         ", ".join(pep517_build_failure_names)
                     )
                 )
 
             # For now, we just warn about failures building legacy
-            # requirements, as we'll fall through to a direct
-            # install for those.
+            # requirements, as we'll fall through to a setup.py install for
+            # those.
             for r in build_failures:
                 if not r.use_pep517:
                     r.legacy_install_reason = 8368

src/pip/_internal/distributions/sdist.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Set, Tuple
+from typing import Iterable, Set, Tuple
 
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
@@ -37,23 +37,17 @@ def prepare_distribution_metadata(
         self.req.prepare_metadata()
 
     def _setup_isolation(self, finder: PackageFinder) -> None:
-        def _raise_conflicts(
-            conflicting_with: str, conflicting_reqs: Set[Tuple[str, str]]
-        ) -> None:
-            format_string = (
-                "Some build dependencies for {requirement} "
-                "conflict with {conflicting_with}: {description}."
-            )
-            error_message = format_string.format(
-                requirement=self.req,
-                conflicting_with=conflicting_with,
-                description=", ".join(
-                    f"{installed} is incompatible with {wanted}"
-                    for installed, wanted in sorted(conflicting)
-                ),
-            )
-            raise InstallationError(error_message)
+        self._prepare_build_backend(finder)
+        # Install any extra build dependencies that the backend requests.
+        # This must be done in a second pass, as the pyproject.toml
+        # dependencies must be installed before we can call the backend.
+        if self.req.editable and self.req.permit_editable_wheels:
+            build_reqs = self._get_build_requires_editable()
+        else:
+            build_reqs = self._get_build_requires_wheel()
+        self._install_build_reqs(finder, build_reqs)
 
+    def _prepare_build_backend(self, finder: PackageFinder) -> None:
         # Isolate in a BuildEnvironment and install the build-time
         # requirements.
         pyproject_requires = self.req.pyproject_requires
@@ -67,7 +61,7 @@ def _raise_conflicts(
             self.req.requirements_to_check
         )
         if conflicting:
-            _raise_conflicts("PEP 517/518 supported requirements", conflicting)
+            self._raise_conflicts("PEP 517/518 supported requirements", conflicting)
         if missing:
             logger.warning(
                 "Missing build requirements in pyproject.toml for %s.",
@@ -78,19 +72,46 @@ def _raise_conflicts(
                 "pip cannot fall back to setuptools without %s.",
                 " and ".join(map(repr, sorted(missing))),
             )
-        # Install any extra build dependencies that the backend requests.
-        # This must be done in a second pass, as the pyproject.toml
-        # dependencies must be installed before we can call the backend.
+
+    def _get_build_requires_wheel(self) -> Iterable[str]:
         with self.req.build_env:
             runner = runner_with_spinner_message("Getting requirements to build wheel")
             backend = self.req.pep517_backend
             assert backend is not None
             with backend.subprocess_runner(runner):
-                reqs = backend.get_requires_for_build_wheel()
+                return backend.get_requires_for_build_wheel()
 
+    def _get_build_requires_editable(self) -> Iterable[str]:
+        with self.req.build_env:
+            runner = runner_with_spinner_message(
+                "Getting requirements to build editable"
+            )
+            backend = self.req.pep517_backend
+            assert backend is not None
+            with backend.subprocess_runner(runner):
+                return backend.get_requires_for_build_editable()
+
+    def _install_build_reqs(self, finder: PackageFinder, reqs: Iterable[str]) -> None:
         conflicting, missing = self.req.build_env.check_requirements(reqs)
         if conflicting:
-            _raise_conflicts("the backend dependencies", conflicting)
+            self._raise_conflicts("the backend dependencies", conflicting)
         self.req.build_env.install_requirements(
             finder, missing, "normal", "Installing backend dependencies"
         )
+
+    def _raise_conflicts(
+        self, conflicting_with: str, conflicting_reqs: Set[Tuple[str, str]]
+    ) -> None:
+        format_string = (
+            "Some build dependencies for {requirement} "
+            "conflict with {conflicting_with}: {description}."
+        )
+        error_message = format_string.format(
+            requirement=self.req,
+            conflicting_with=conflicting_with,
+            description=", ".join(
+                f"{installed} is incompatible with {wanted}"
+                for installed, wanted in sorted(conflicting_reqs)
+            ),
+        )
+        raise InstallationError(error_message)

src/pip/_internal/operations/build/metadata.py

@@ -23,7 +23,9 @@ def generate_metadata(build_env: BuildEnvironment, backend: Pep517HookCaller) ->
         # Note that Pep517HookCaller implements a fallback for
         # prepare_metadata_for_build_wheel, so we don't have to
         # consider the possibility that this hook doesn't exist.
-        runner = runner_with_spinner_message("Preparing wheel metadata")
+        runner = runner_with_spinner_message(
+            "Preparing wheel metadata (pyproject.toml)"
+        )
         with backend.subprocess_runner(runner):
             distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
 

src/pip/_internal/operations/build/metadata_editable.py

@@ -0,0 +1,34 @@
+"""Metadata generation logic for source distributions.
+"""
+
+import os
+
+from pip._vendor.pep517.wrappers import Pep517HookCaller
+
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+def generate_editable_metadata(
+    build_env: BuildEnvironment, backend: Pep517HookCaller
+) -> str:
+    """Generate metadata using mechanisms described in PEP 660.
+
+    Returns the generated metadata directory.
+    """
+    metadata_tmpdir = TempDirectory(kind="modern-metadata", globally_managed=True)
+
+    metadata_dir = metadata_tmpdir.path
+
+    with build_env:
+        # Note that Pep517HookCaller implements a fallback for
+        # prepare_metadata_for_build_wheel/editable, so we don't have to
+        # consider the possibility that this hook doesn't exist.
+        runner = runner_with_spinner_message(
+            "Preparing editable metadata (pyproject.toml)"
+        )
+        with backend.subprocess_runner(runner):
+            distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
+
+    return os.path.join(metadata_dir, distinfo_dir)

src/pip/_internal/operations/build/metadata_legacy.py

@@ -5,6 +5,7 @@
 import os
 
 from pip._internal.build_env import BuildEnvironment
+from pip._internal.cli.spinners import open_spinner
 from pip._internal.exceptions import InstallationError
 from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
 from pip._internal.utils.subprocess import call_subprocess
@@ -54,11 +55,13 @@ def generate_metadata(
     )
 
     with build_env:
-        call_subprocess(
-            args,
-            cwd=source_dir,
-            command_desc="python setup.py egg_info",
-        )
+        with open_spinner("Preparing metadata (setup.py)") as spinner:
+            call_subprocess(
+                args,
+                cwd=source_dir,
+                command_desc="python setup.py egg_info",
+                spinner=spinner,
+            )
 
     # Return the .egg-info directory.
     return _find_egg_info(egg_info_dir)

src/pip/_internal/operations/build/wheel.py

@@ -23,7 +23,9 @@ def build_wheel_pep517(
     try:
         logger.debug("Destination directory: %s", tempd)
 
-        runner = runner_with_spinner_message(f"Building wheel for {name} (PEP 517)")
+        runner = runner_with_spinner_message(
+            f"Building wheel for {name} (pyproject.toml)"
+        )
         with backend.subprocess_runner(runner):
             wheel_name = backend.build_wheel(
                 tempd,

src/pip/_internal/operations/build/wheel_editable.py

@@ -0,0 +1,46 @@
+import logging
+import os
+from typing import Optional
+
+from pip._vendor.pep517.wrappers import HookMissing, Pep517HookCaller
+
+from pip._internal.utils.subprocess import runner_with_spinner_message
+
+logger = logging.getLogger(__name__)
+
+
+def build_wheel_editable(
+    name: str,
+    backend: Pep517HookCaller,
+    metadata_directory: str,
+    tempd: str,
+) -> Optional[str]:
+    """Build one InstallRequirement using the PEP 660 build process.
+
+    Returns path to wheel if successfully built. Otherwise, returns None.
+    """
+    assert metadata_directory is not None
+    try:
+        logger.debug("Destination directory: %s", tempd)
+
+        runner = runner_with_spinner_message(
+            f"Building editable for {name} (pyproject.toml)"
+        )
+        with backend.subprocess_runner(runner):
+            try:
+                wheel_name = backend.build_editable(
+                    tempd,
+                    metadata_directory=metadata_directory,
+                )
+            except HookMissing as e:
+                logger.error(
+                    "Cannot build editable %s because the build "
+                    "backend does not have the %s hook",
+                    name,
+                    e,
+                )
+                return None
+    except Exception:
+        logger.error("Failed building editable for %s", name)
+        return None
+    return os.path.join(tempd, wheel_name)

src/pip/_internal/req/constructors.py

@@ -22,7 +22,6 @@
 from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
-from pip._internal.pyproject import make_pyproject_path
 from pip._internal.req.req_file import ParsedRequirement
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.filetypes import is_archive_file
@@ -75,21 +74,6 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
     url_no_extras, extras = _strip_extras(url)
 
     if os.path.isdir(url_no_extras):
-        setup_py = os.path.join(url_no_extras, "setup.py")
-        setup_cfg = os.path.join(url_no_extras, "setup.cfg")
-        if not os.path.exists(setup_py) and not os.path.exists(setup_cfg):
-            msg = (
-                'File "setup.py" or "setup.cfg" not found. Directory cannot be '
-                "installed in editable mode: {}".format(os.path.abspath(url_no_extras))
-            )
-            pyproject_path = make_pyproject_path(url_no_extras)
-            if os.path.isfile(pyproject_path):
-                msg += (
-                    '\n(A "pyproject.toml" file was found, but editable '
-                    "mode currently requires a setuptools-based build.)"
-                )
-            raise InstallationError(msg)
-
         # Treating it as code that has already been checked out
         url_no_extras = path_to_url(url_no_extras)
 
@@ -197,6 +181,7 @@ def install_req_from_editable(
     options: Optional[Dict[str, Any]] = None,
     constraint: bool = False,
     user_supplied: bool = False,
+    permit_editable_wheels: bool = False,
 ) -> InstallRequirement:
 
     parts = parse_req_from_editable(editable_req)
@@ -206,6 +191,7 @@ def install_req_from_editable(
         comes_from=comes_from,
         user_supplied=user_supplied,
         editable=True,
+        permit_editable_wheels=permit_editable_wheels,
         link=parts.link,
         constraint=constraint,
         use_pep517=use_pep517,