feat: update compile_pip_requirements to support multiple input files (#1067)

`pip-compile` can compile multiple input files into a single output
file, but `rules_python`'s `compile_pip_requirements` doesn't currently
support this.

With this change, the `requirements_in` argument to
`compile_pip_requirements` can now accept a list of strings (in addition
to the previously accepted argument types).

In order to support a variable number of input files, my coworker
(@lpulley) and I updated `dependency_resolver.py` to use the `click` CLI
library. We felt this was acceptable since `pip-compile` already
requires `click` to run, so we're not adding a new dependency.

We also made changes to the script to avoid mutating `sys.argv`, instead
opting to build a new list (`argv`) from scratch that'll be passed to
the `pip-compile` CLI. While subjective, I feel this improves
readability, since it's not immediately obvious what's in `sys.argv`,
but it's clear that `argv` begins empty, and is added to over the course
of the program's execution.

---------

Co-authored-by: Logan Pulley <[email protected]>
Co-authored-by: Ignas Anikevicius <[email protected]>
Co-authored-by: Richard Levasseur <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -35,6 +35,9 @@ A brief description of the categories of changes:
     * `3.12 -> 3.12.4`
 
 ### Fixed
+* (rules) `compile_pip_requirements` now sets the `USERPROFILE` env variable on
+  Windows to work around an issue where `setuptools` fails to locate the user's
+  home directory.
 * (rules) correctly handle absolute URLs in parse_simpleapi_html.bzl.
 * (rules) Fixes build targets linking against `@rules_python//python/cc:current_py_cc_libs`
   in host platform builds on macOS, by editing the `LC_ID_DYLIB` field of the hermetic interpreter's
@@ -73,6 +76,7 @@ A brief description of the categories of changes:
   Fixes [#1631](https://github.com/bazelbuild/rules_python/issues/1631).
 
 ### Added
+* (rules) `compile_pip_requirements` supports multiple requirements input files as `srcs`.
 * (rules) `PYTHONSAFEPATH` is inherited from the calling environment to allow
   disabling it (Requires {obj}`--bootstrap_impl=script`)
   ([#2060](https://github.com/bazelbuild/rules_python/issues/2060)).

python/private/pypi/dependency_resolver/dependency_resolver.py

@@ -80,15 +80,15 @@ def _locate(bazel_runfiles, file):
 
 
 @click.command(context_settings={"ignore_unknown_options": True})
-@click.argument("requirements_in")
+@click.option("--src", "srcs", multiple=True, required=True)
 @click.argument("requirements_txt")
 @click.argument("update_target_label")
 @click.option("--requirements-linux")
 @click.option("--requirements-darwin")
 @click.option("--requirements-windows")
 @click.argument("extra_args", nargs=-1, type=click.UNPROCESSED)
 def main(
-    requirements_in: str,
+    srcs: Tuple[str, ...],
     requirements_txt: str,
     update_target_label: str,
     requirements_linux: Optional[str],
@@ -105,7 +105,7 @@ def main(
         requirements_windows=requirements_windows,
     )
 
-    resolved_requirements_in = _locate(bazel_runfiles, requirements_in)
+    resolved_srcs = [_locate(bazel_runfiles, src) for src in srcs]
     resolved_requirements_file = _locate(bazel_runfiles, requirements_file)
 
     # Files in the runfiles directory has the following naming schema:
@@ -118,11 +118,11 @@ def main(
         : -(len(requirements_file) - len(repository_prefix))
     ]
 
-    # As requirements_in might contain references to generated files we want to
+    # As srcs might contain references to generated files we want to
     # use the runfiles file first. Thus, we need to compute the relative path
     # from the execution root.
     # Note: Windows cannot reference generated files without runfiles support enabled.
-    requirements_in_relative = requirements_in[len(repository_prefix) :]
+    srcs_relative = [src[len(repository_prefix) :] for src in srcs]
     requirements_file_relative = requirements_file[len(repository_prefix) :]
 
     # Before loading click, set the locale for its parser.
@@ -162,10 +162,9 @@ def main(
     argv.append(
         f"--output-file={requirements_file_relative if UPDATE else requirements_out}"
     )
-    argv.append(
-        requirements_in_relative
-        if Path(requirements_in_relative).exists()
-        else resolved_requirements_in
+    argv.extend(
+        (src_relative if Path(src_relative).exists() else resolved_src)
+        for src_relative, resolved_src in zip(srcs_relative, resolved_srcs)
     )
     argv.extend(extra_args)
 
@@ -200,7 +199,7 @@ def main(
                 print(
                     "pip-compile exited with code 2. This means that pip-compile found "
                     "incompatible requirements or could not find a version that matches "
-                    f"the install requirement in {requirements_in_relative}.",
+                    f"the install requirement in one of {srcs_relative}.",
                     file=sys.stderr,
                 )
                 sys.exit(1)

python/private/pypi/pip_compile.bzl

@@ -23,6 +23,7 @@ load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test")
 
 def pip_compile(
         name,
+        srcs = None,
         src = None,
         extra_args = [],
         extra_deps = [],
@@ -53,6 +54,11 @@ def pip_compile(
 
     Args:
         name: base name for generated targets, typically "requirements".
+        srcs: a list of files containing inputs to dependency resolution. If not specified,
+            defaults to `["pyproject.toml"]`. Supported formats are:
+            * a requirements text file, usually named `requirements.in`
+            * A `.toml` file, where the `project.dependencies` list is used as per
+              [PEP621](https://peps.python.org/pep-0621/).
         src: file containing inputs to dependency resolution. If not specified,
             defaults to `pyproject.toml`. Supported formats are:
             * a requirements text file, usually named `requirements.in`
@@ -63,7 +69,7 @@ def pip_compile(
         generate_hashes: whether to put hashes in the requirements_txt file.
         py_binary: the py_binary rule to be used.
         py_test: the py_test rule to be used.
-        requirements_in: file expressing desired dependencies. Deprecated, use src instead.
+        requirements_in: file expressing desired dependencies. Deprecated, use src or srcs instead.
         requirements_txt: result of "compiling" the requirements.in file.
         requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes.
         requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes.
@@ -72,10 +78,15 @@ def pip_compile(
         visibility: passed to both the _test and .update rules.
         **kwargs: other bazel attributes passed to the "_test" rule.
     """
-    if requirements_in and src:
-        fail("Only one of 'src' and 'requirements_in' attributes can be used")
+    if len([x for x in [srcs, src, requirements_in] if x != None]) > 1:
+        fail("At most one of 'srcs', 'src', and 'requirements_in' attributes may be provided")
+
+    if requirements_in:
+        srcs = [requirements_in]
+    elif src:
+        srcs = [src]
     else:
-        src = requirements_in or src or "pyproject.toml"
+        srcs = srcs or ["pyproject.toml"]
 
     requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt
 
@@ -88,16 +99,15 @@ def pip_compile(
         visibility = visibility,
     )
 
-    data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
+    data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
 
     # Use the Label constructor so this is expanded in the context of the file
     # where it appears, which is to say, in @rules_python
     pip_compile = Label("//python/private/pypi/dependency_resolver:dependency_resolver.py")
 
     loc = "$(rlocationpath {})"
 
-    args = [
-        loc.format(src),
+    args = ["--src=%s" % loc.format(src) for src in srcs] + [
         loc.format(requirements_txt),
         "//%s:%s.update" % (native.package_name(), name),
         "--resolver=backtracking",
@@ -144,12 +154,15 @@ def pip_compile(
         "visibility": visibility,
     }
 
-    # cheap way to detect the bazel version
-    _bazel_version_4_or_greater = "propeller_optimize" in dir(native)
-
-    # Bazel 4.0 added the "env" attribute to py_test/py_binary
-    if _bazel_version_4_or_greater:
-        attrs["env"] = kwargs.pop("env", {})
+    # setuptools (the default python build tool) attempts to find user
+    # configuration in the user's home direcotory. This seems to work fine on
+    # linux and macOS, but fails on Windows, so we conditionally provide a fake
+    # USERPROFILE env variable to allow setuptools to proceed without finding
+    # user-provided configuration.
+    kwargs["env"] = select({
+        "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"},
+        "//conditions:default": {},
+    }) | kwargs.get("env", {})
 
     py_binary(
         name = name + ".update",

tests/multiple_inputs/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+compile_pip_requirements(
+    name = "multiple_requirements_in",
+    srcs = [
+        "requirements_1.in",
+        "requirements_2.in",
+    ],
+    requirements_txt = "multiple_requirements_in.txt",
+)
+
+compile_pip_requirements(
+    name = "multiple_pyproject_toml",
+    srcs = [
+        "a/pyproject.toml",
+        "b/pyproject.toml",
+    ],
+    requirements_txt = "multiple_pyproject_toml.txt",
+)
+
+compile_pip_requirements(
+    name = "multiple_inputs",
+    srcs = [
+        "a/pyproject.toml",
+        "b/pyproject.toml",
+        "requirements_1.in",
+        "requirements_2.in",
+    ],
+    requirements_txt = "multiple_inputs.txt",
+)

tests/multiple_inputs/README.md

@@ -0,0 +1,3 @@
+# multiple_inputs
+
+Test that `compile_pip_requirements` works as intended when using more than one input file.

tests/multiple_inputs/a/pyproject.toml

@@ -0,0 +1,5 @@
+[project]
+name = "multiple_inputs_1"
+version = "0.0.0"
+
+dependencies = ["urllib3"]

tests/multiple_inputs/b/pyproject.toml

@@ -0,0 +1,5 @@
+[project]
+name = "multiple_inputs_2"
+version = "0.0.0"
+
+dependencies = ["attrs"]

tests/multiple_inputs/multiple_inputs.txt

@@ -0,0 +1,18 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //tests/multiple_inputs:multiple_inputs.update
+#
+attrs==23.1.0 \
+    --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \
+    --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015
+    # via
+    #   -r tests/multiple_inputs/requirements_2.in
+    #   multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml)
+urllib3==2.0.7 \
+    --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \
+    --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e
+    # via
+    #   -r tests/multiple_inputs/requirements_1.in
+    #   multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml)

tests/multiple_inputs/multiple_pyproject_toml.txt

@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //tests/multiple_inputs:multiple_pyproject_toml.update
+#
+attrs==23.1.0 \
+    --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \
+    --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015
+    # via multiple_inputs_2 (tests/multiple_inputs/b/pyproject.toml)
+urllib3==2.0.7 \
+    --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \
+    --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e
+    # via multiple_inputs_1 (tests/multiple_inputs/a/pyproject.toml)

tests/multiple_inputs/multiple_requirements_in.txt

@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //tests/multiple_inputs:multiple_requirements_in.update
+#
+attrs==23.1.0 \
+    --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \
+    --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015
+    # via -r tests/multiple_inputs/requirements_2.in
+urllib3==2.0.7 \
+    --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \
+    --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e
+    # via -r tests/multiple_inputs/requirements_1.in

tests/multiple_inputs/requirements_1.in

@@ -0,0 +1 @@
+urllib3

tests/multiple_inputs/requirements_2.in

@@ -0,0 +1 @@
+attrs