diff --git a/docs/html/reference/build-system/pyproject-toml.md b/docs/html/reference/build-system/pyproject-toml.md index 7959a2d081f..d2ec0323e6a 100644 --- a/docs/html/reference/build-system/pyproject-toml.md +++ b/docs/html/reference/build-system/pyproject-toml.md @@ -106,6 +106,16 @@ This is considered a stopgap solution until setuptools adds support for regular {ref}`deprecation policy `. ``` +### Backend Configuration + +Build backends have the ability to accept configuration settings, which can +change the way the build is handled. These settings take the form of a +series of `key=value` pairs. The user can supply configuration settings +using the `--config-settings` command line option (which can be supplied +multiple times, in order to specify multiple settings). + +The supplied configuration settings are passed to every backend hook call. + ## Build output It is the responsibility of the build backend to ensure that the output is diff --git a/docs/html/reference/requirements-file-format.md b/docs/html/reference/requirements-file-format.md index cf1d434eb6a..75e6d0b1e6b 100644 --- a/docs/html/reference/requirements-file-format.md +++ b/docs/html/reference/requirements-file-format.md @@ -111,6 +111,7 @@ The options which can be applied to individual requirements are: - {ref}`--install-option ` - {ref}`--global-option ` +- {ref}`--config-settings ` - `--hash` (for {ref}`Hash-checking mode`) ## Referring to other requirements files diff --git a/news/11059.feature.rst b/news/11059.feature.rst new file mode 100644 index 00000000000..3cc19b89804 --- /dev/null +++ b/news/11059.feature.rst @@ -0,0 +1 @@ +Add a user interface for supplying config settings to build backends. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index cd1e8a00f63..6a705edec37 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -794,6 +794,33 @@ def _handle_no_use_pep517( help=SUPPRESS_HELP, ) + +def _handle_config_settings( + option: Option, opt_str: str, value: str, parser: OptionParser +) -> None: + key, sep, val = value.partition("=") + if sep != "=": + parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL") # noqa + dest = getattr(parser.values, option.dest) + if dest is None: + dest = {} + setattr(parser.values, option.dest, dest) + dest[key] = val + + +config_settings: Callable[..., Option] = partial( + Option, + "--config-settings", + dest="config_settings", + type=str, + action="callback", + callback=_handle_config_settings, + metavar="settings", + help="Configuration settings to be passed to the PEP 517 build backend. " + "Settings take the form KEY=VALUE. Use multiple --config-settings options " + "to pass multiple keys to the backend.", +) + install_options: Callable[..., Option] = partial( Option, "--install-option", diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 2c2db00dec6..aab177002d4 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -325,6 +325,7 @@ def make_resolver( install_req_from_req_string, isolated=options.isolated_mode, use_pep517=use_pep517, + config_settings=getattr(options, "config_settings", None), ) suppress_build_failures = cls.determine_build_failure_suppression(options) resolver_variant = cls.determine_resolver_variant(options) @@ -397,6 +398,7 @@ def get_requirements( isolated=options.isolated_mode, use_pep517=options.use_pep517, user_supplied=True, + config_settings=getattr(options, "config_settings", None), ) requirements.append(req_to_add) @@ -406,6 +408,7 @@ def get_requirements( user_supplied=True, isolated=options.isolated_mode, use_pep517=options.use_pep517, + config_settings=getattr(options, "config_settings", None), ) requirements.append(req_to_add) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 75c8b143c3e..55edb280c96 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -190,6 +190,7 @@ def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.use_pep517()) self.cmd_opts.add_option(cmdoptions.no_use_pep517()) + self.cmd_opts.add_option(cmdoptions.config_settings()) self.cmd_opts.add_option(cmdoptions.install_options()) self.cmd_opts.add_option(cmdoptions.global_options()) diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 4fefe8111dd..05b91d4733d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -73,6 +73,7 @@ def add_options(self) -> None: help="Don't verify if built wheel is valid.", ) + self.cmd_opts.add_option(cmdoptions.config_settings()) self.cmd_opts.add_option(cmdoptions.build_options()) self.cmd_opts.add_option(cmdoptions.global_options()) diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 25bfb391d88..a879e94ea1a 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -207,6 +207,7 @@ def install_req_from_editable( constraint: bool = False, user_supplied: bool = False, permit_editable_wheels: bool = False, + config_settings: Optional[Dict[str, str]] = None, ) -> InstallRequirement: parts = parse_req_from_editable(editable_req) @@ -224,6 +225,7 @@ def install_req_from_editable( install_options=options.get("install_options", []) if options else [], global_options=options.get("global_options", []) if options else [], hash_options=options.get("hashes", {}) if options else {}, + config_settings=config_settings, extras=parts.extras, ) @@ -380,6 +382,7 @@ def install_req_from_line( constraint: bool = False, line_source: Optional[str] = None, user_supplied: bool = False, + config_settings: Optional[Dict[str, str]] = None, ) -> InstallRequirement: """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. @@ -399,6 +402,7 @@ def install_req_from_line( install_options=options.get("install_options", []) if options else [], global_options=options.get("global_options", []) if options else [], hash_options=options.get("hashes", {}) if options else {}, + config_settings=config_settings, constraint=constraint, extras=parts.extras, user_supplied=user_supplied, @@ -411,6 +415,7 @@ def install_req_from_req_string( isolated: bool = False, use_pep517: Optional[bool] = None, user_supplied: bool = False, + config_settings: Optional[Dict[str, str]] = None, ) -> InstallRequirement: try: req = get_requirement(req_string) @@ -440,6 +445,7 @@ def install_req_from_req_string( isolated=isolated, use_pep517=use_pep517, user_supplied=user_supplied, + config_settings=config_settings, ) @@ -448,6 +454,7 @@ def install_req_from_parsed_requirement( isolated: bool = False, use_pep517: Optional[bool] = None, user_supplied: bool = False, + config_settings: Optional[Dict[str, str]] = None, ) -> InstallRequirement: if parsed_req.is_editable: req = install_req_from_editable( @@ -457,6 +464,7 @@ def install_req_from_parsed_requirement( constraint=parsed_req.constraint, isolated=isolated, user_supplied=user_supplied, + config_settings=config_settings, ) else: @@ -469,6 +477,7 @@ def install_req_from_parsed_requirement( constraint=parsed_req.constraint, line_source=parsed_req.line_source, user_supplied=user_supplied, + config_settings=config_settings, ) return req @@ -487,4 +496,5 @@ def install_req_from_link_and_ireq( install_options=ireq.install_options, global_options=ireq.global_options, hash_options=ireq.hash_options, + config_settings=ireq.config_settings, ) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 637b6ce10af..ef969dee6db 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -46,6 +46,7 @@ ) from pip._internal.utils.hashes import Hashes from pip._internal.utils.misc import ( + ConfiguredPep517HookCaller, ask_path_exists, backup_dir, display_path, @@ -80,6 +81,7 @@ def __init__( install_options: Optional[List[str]] = None, global_options: Optional[List[str]] = None, hash_options: Optional[Dict[str, List[str]]] = None, + config_settings: Optional[Dict[str, str]] = None, constraint: bool = False, extras: Collection[str] = (), user_supplied: bool = False, @@ -138,6 +140,7 @@ def __init__( self.install_options = install_options if install_options else [] self.global_options = global_options if global_options else [] self.hash_options = hash_options if hash_options else {} + self.config_settings = config_settings # Set to True after successful preparation of this requirement self.prepared = False # User supplied requirement are explicitly requested for installation @@ -469,7 +472,8 @@ def load_pyproject_toml(self) -> None: requires, backend, check, backend_path = pyproject_toml_data self.requirements_to_check = check self.pyproject_requires = requires - self.pep517_backend = Pep517HookCaller( + self.pep517_backend = ConfiguredPep517HookCaller( + self, self.unpacked_source_directory, backend, backend_path=backend_path, diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 9b8450e86b8..d1470ecbf4e 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -69,6 +69,7 @@ def make_install_req_from_link( global_options=template.global_options, hashes=template.hash_options, ), + config_settings=template.config_settings, ) ireq.original_link = template.original_link ireq.link = link @@ -92,6 +93,7 @@ def make_install_req_from_editable( global_options=template.global_options, hashes=template.hash_options, ), + config_settings=template.config_settings, ) @@ -116,6 +118,7 @@ def _make_install_req_from_dist( global_options=template.global_options, hashes=template.hash_options, ), + config_settings=template.config_settings, ) ireq.satisfied_by = dist return ireq diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 9142cacbf52..a8f4cb5cf56 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -21,6 +21,7 @@ BinaryIO, Callable, ContextManager, + Dict, Generator, Iterable, Iterator, @@ -33,6 +34,7 @@ cast, ) +from pip._vendor.pep517 import Pep517HookCaller from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed from pip import __version__ @@ -55,6 +57,7 @@ "captured_stdout", "ensure_dir", "remove_auth_from_url", + "ConfiguredPep517HookCaller", ] @@ -630,3 +633,91 @@ def partition( """ t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) + + +class ConfiguredPep517HookCaller(Pep517HookCaller): + def __init__( + self, + config_holder: Any, + source_dir: str, + build_backend: str, + backend_path: Optional[str] = None, + runner: Optional[Callable[..., None]] = None, + python_executable: Optional[str] = None, + ): + super().__init__( + source_dir, build_backend, backend_path, runner, python_executable + ) + self.config_holder = config_holder + + def build_wheel( + self, + wheel_directory: str, + config_settings: Optional[Dict[str, str]] = None, + metadata_directory: Optional[str] = None, + ) -> str: + cs = self.config_holder.config_settings + return super().build_wheel( + wheel_directory, config_settings=cs, metadata_directory=metadata_directory + ) + + def build_sdist( + self, sdist_directory: str, config_settings: Optional[Dict[str, str]] = None + ) -> str: + cs = self.config_holder.config_settings + return super().build_sdist(sdist_directory, config_settings=cs) + + def build_editable( + self, + wheel_directory: str, + config_settings: Optional[Dict[str, str]] = None, + metadata_directory: Optional[str] = None, + ) -> str: + cs = self.config_holder.config_settings + return super().build_editable( + wheel_directory, config_settings=cs, metadata_directory=metadata_directory + ) + + def get_requires_for_build_wheel( + self, config_settings: Optional[Dict[str, str]] = None + ) -> List[str]: + cs = self.config_holder.config_settings + return super().get_requires_for_build_wheel(config_settings=cs) + + def get_requires_for_build_sdist( + self, config_settings: Optional[Dict[str, str]] = None + ) -> List[str]: + cs = self.config_holder.config_settings + return super().get_requires_for_build_sdist(config_settings=cs) + + def get_requires_for_build_editable( + self, config_settings: Optional[Dict[str, str]] = None + ) -> List[str]: + cs = self.config_holder.config_settings + return super().get_requires_for_build_editable(config_settings=cs) + + def prepare_metadata_for_build_wheel( + self, + metadata_directory: str, + config_settings: Optional[Dict[str, str]] = None, + _allow_fallback: bool = True, + ) -> str: + cs = self.config_holder.config_settings + return super().prepare_metadata_for_build_wheel( + metadata_directory=metadata_directory, + config_settings=cs, + _allow_fallback=_allow_fallback, + ) + + def prepare_metadata_for_build_editable( + self, + metadata_directory: str, + config_settings: Optional[Dict[str, str]] = None, + _allow_fallback: bool = True, + ) -> str: + cs = self.config_holder.config_settings + return super().prepare_metadata_for_build_editable( + metadata_directory=metadata_directory, + config_settings=cs, + _allow_fallback=_allow_fallback, + ) diff --git a/tests/functional/test_config_settings.py b/tests/functional/test_config_settings.py new file mode 100644 index 00000000000..76c667cb91d --- /dev/null +++ b/tests/functional/test_config_settings.py @@ -0,0 +1,139 @@ +import json +from typing import Tuple +from zipfile import ZipFile + +from tests.lib import PipTestEnvironment +from tests.lib.path import Path + +PYPROJECT_TOML = """\ +[build-system] +requires = [] +build-backend = "dummy_backend:main" +backend-path = ["backend"] +""" + +BACKEND_SRC = ''' +import csv +import json +import os.path +from zipfile import ZipFile +import hashlib +import base64 +import io + +WHEEL = """\ +Wheel-Version: 1.0 +Generator: dummy_backend 1.0 +Root-Is-Purelib: true +Tag: py3-none-any +""" + +METADATA = """\ +Metadata-Version: 2.1 +Name: {project} +Version: {version} +Summary: A dummy package +Author: None +Author-email: none@example.org +License: MIT +""" + +def make_wheel(z, project, version, files): + record = [] + def add_file(name, data): + data = data.encode("utf-8") + z.writestr(name, data) + digest = hashlib.sha256(data).digest() + hash = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ASCII") + record.append((name, f"sha256={hash}", len(data))) + distinfo = f"{project}-{version}.dist-info" + add_file(f"{distinfo}/WHEEL", WHEEL) + add_file(f"{distinfo}/METADATA", METADATA.format(project=project, version=version)) + for name, data in files: + add_file(name, data) + record_name = f"{distinfo}/RECORD" + record.append((record_name, "", "")) + b = io.BytesIO() + rec = io.TextIOWrapper(b, newline="", encoding="utf-8") + w = csv.writer(rec) + w.writerows(record) + z.writestr(record_name, b.getvalue()) + rec.close() + + +class Backend: + def build_wheel( + self, + wheel_directory, + config_settings=None, + metadata_directory=None + ): + if config_settings is None: + config_settings = {} + w = os.path.join(wheel_directory, "foo-1.0-py3-none-any.whl") + with open(w, "wb") as f: + with ZipFile(f, "w") as z: + make_wheel( + z, "foo", "1.0", + [("config.json", json.dumps(config_settings))] + ) + return "foo-1.0-py3-none-any.whl" + + build_editable = build_wheel + +main = Backend() +''' + + +def make_project(path: Path) -> Tuple[str, str, Path]: + name = "foo" + version = "1.0" + project_dir = path / name + backend = project_dir / "backend" + backend.mkdir(parents=True) + (project_dir / "pyproject.toml").write_text(PYPROJECT_TOML) + (backend / "dummy_backend.py").write_text(BACKEND_SRC) + return name, version, project_dir + + +def test_backend_sees_config(script: PipTestEnvironment) -> None: + name, version, project_dir = make_project(script.scratch_path) + script.pip( + "wheel", + "--config-settings", + "FOO=Hello", + project_dir, + ) + wheel_file_name = f"{name}-{version}-py3-none-any.whl" + wheel_file_path = script.cwd / wheel_file_name + with open(wheel_file_path, "rb") as f: + with ZipFile(f) as z: + output = z.read("config.json") + assert json.loads(output) == {"FOO": "Hello"} + + +def test_install_sees_config(script: PipTestEnvironment) -> None: + _, _, project_dir = make_project(script.scratch_path) + script.pip( + "install", + "--config-settings", + "FOO=Hello", + project_dir, + ) + config = script.site_packages_path / "config.json" + with open(config, "rb") as f: + assert json.load(f) == {"FOO": "Hello"} + + +def test_install_editable_sees_config(script: PipTestEnvironment) -> None: + _, _, project_dir = make_project(script.scratch_path) + script.pip( + "install", + "--config-settings", + "FOO=Hello", + "--editable", + project_dir, + ) + config = script.site_packages_path / "config.json" + with open(config, "rb") as f: + assert json.load(f) == {"FOO": "Hello"} diff --git a/tests/unit/test_pyproject_config.py b/tests/unit/test_pyproject_config.py new file mode 100644 index 00000000000..9937f3880aa --- /dev/null +++ b/tests/unit/test_pyproject_config.py @@ -0,0 +1,44 @@ +import pytest + +from pip._internal.commands import create_command + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("install", True), + ("wheel", True), + ("freeze", False), + ], +) +def test_supports_config(command: str, expected: bool) -> None: + c = create_command(command) + options, _ = c.parse_args([]) + assert hasattr(options, "config_settings") == expected + + +def test_set_config_value_true() -> None: + i = create_command("install") + # Invalid argument exits with an error + with pytest.raises(SystemExit): + options, _ = i.parse_args(["xxx", "--config-settings", "x"]) + + +def test_set_config_value() -> None: + i = create_command("install") + options, _ = i.parse_args(["xxx", "--config-settings", "x=hello"]) + assert options.config_settings == {"x": "hello"} + + +def test_set_config_empty_value() -> None: + i = create_command("install") + options, _ = i.parse_args(["xxx", "--config-settings", "x="]) + assert options.config_settings == {"x": ""} + + +def test_replace_config_value() -> None: + i = create_command("install") + options, _ = i.parse_args( + ["xxx", "--config-settings", "x=hello", "--config-settings", "x=world"] + ) + assert options.config_settings == {"x": "world"}