refactor!(bzlmod): introduce pypi.install extension

With this PR we are introducing a new extension for managing PyPI dependencies
and it seems that we would fix bazel-contrib#2268 in doing so. The code has been sitting
around for long enough so that we have ironed out most of the bigger bugs
already and the maintainers have agreed for long enough that pip.parse is
not the best named, so I have chosen pypi.install.

This should not actively break the code and will make some of the experimental
parameters noop. This could yield to weird failures in cases where people are
building docker containers, so I am thinking that we should potentially add
a helpful error message prompting the users to migrate. However I am not
sure how that works out when multiple module versions are at play in the same
module graph.

Author: aignas

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

CHANGELOG.md

@@ -27,6 +27,10 @@ A brief description of the categories of changes:
 ### Changed
 * (toolchains) `py_runtime.implementation_name` now defaults to `cpython`
   (previously it defaulted to None).
+* (bzlmod) **BREAKING** The `experimental_index_url` feature has been removed
+  from `pip.parse` and a new extension (`pypi.install`) has been created in
+  order to ensure that `MODULE.bazel.lock` remains working for others.
+  Fixes [#2268](https://github.com/bazelbuild/rules_python/issues/2268).
 
 ### Fixed
 * (bzlmod) The `python.override(minor_mapping)` now merges the default and the

MODULE.bazel

@@ -54,8 +54,8 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
-pip.parse(
+pypi = use_extension("//python/private/pypi:pypi.bzl", "pypi")
+pypi.install(
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -64,7 +64,7 @@ pip.parse(
         "//tools/publish:requirements_windows.txt": "windows_*",
     },
 )
-use_repo(pip, "rules_python_publish_deps")
+use_repo(pypi, "rules_python_publish_deps")
 
 # ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
 bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
@@ -85,22 +85,22 @@ dev_python.override(
     register_all_versions = True,
 )
 
-dev_pip = use_extension(
-    "//python/private/pypi:pip.bzl",
-    "pip_internal",
+dev_pypi = use_extension(
+    "//python/private/pypi:pypi.bzl",
+    "pypi",
     dev_dependency = True,
 )
-dev_pip.parse(
+dev_pypi.install(
     hub_name = "dev_pip",
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
-dev_pip.parse(
+dev_pypi.install(
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",
 )
-use_repo(dev_pip, "dev_pip", "pypiserver")
+use_repo(dev_pypi, "dev_pip", "pypiserver")
 
 # Bazel integration test setup below
 

docs/BUILD.bazel

@@ -109,6 +109,7 @@ sphinx_stardocs(
     ] if IS_BAZEL_7_OR_HIGHER else []) + ([
         # This depends on @pythons_hub, which is only created under bzlmod,
         "//python/extensions:pip_bzl",
+        "//python/extensions:pypi_bzl",
     ] if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else []),
     prefix = "api/rules_python/",
     tags = ["docs"],

examples/bzlmod/MODULE.bazel

@@ -104,11 +104,11 @@ register_toolchains("@uv_toolchains//:all")
 # See @rules_python//python/extensions:whl_mods.bzl attributes for more information
 # on each of the attributes.
 # You are able to set a hub name, so that you can have different modifications of the same
-# wheel in different pip hubs.
-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+# wheel in different pypi hubs.
+pypi = use_extension("@rules_python//python/extensions:pypi.bzl", "pypi")
 
 # Call whl_mods.create for the requests package.
-pip.whl_mods(
+pypi.whl_mods(
     # we are using the appended_build_content.BUILD file
     # to add content to the request wheel BUILD file.
     additive_build_content_file = "//whl_mods:appended_build_content.BUILD",
@@ -127,7 +127,7 @@ write_file(
 """
 
 # Call whl_mods.create for the wheel package.
-pip.whl_mods(
+pypi.whl_mods(
     additive_build_content = ADDITIVE_BUILD_CONTENT,
     copy_executables = {
         "@@//whl_mods:data/copy_executable.py": "copied_content/executable.py",
@@ -140,27 +140,24 @@ pip.whl_mods(
     hub_name = "whl_mods_hub",
     whl_name = "wheel",
 )
-use_repo(pip, "whl_mods_hub")
+use_repo(pypi, "whl_mods_hub")
 
-# To fetch pip dependencies, use pip.parse. We can pass in various options,
+# To fetch pypi dependencies, use pypi.install. We can pass in various options,
 # but typically we pass requirements and the Python version. The Python
 # version must have been configured by a corresponding `python.toolchain()`
 # call.
 # Alternatively, `python_interpreter_target` can be used to directly specify
 # the Python interpreter to run to resolve dependencies.
-pip.parse(
+pypi.install(
     # We can use `envsubst in the above
     envsubst = ["PIP_INDEX_URL"],
-    # Use the bazel downloader to query the simple API for downloading the sources
-    # Note, that we can use envsubst for this value.
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     # One can also select a particular index for a particular package.
     # This ensures that the setup is resistant against confusion attacks.
-    # experimental_index_url_overrides = {
+    # index_url_overrides = {
     #    "my_package": "https://different-index-url.com",
     # },
     # Or you can specify extra indexes like with `pip`:
-    # experimental_extra_index_urls = [
+    # extra_index_urls = [
     #    "https://different-index-url.com",
     # ],
     experimental_requirement_cycles = {
@@ -182,6 +179,9 @@ pip.parse(
         "cp39_*",
     ],
     hub_name = "pip",
+    # Use the bazel downloader to query the simple API for downloading the sources
+    # Note, that we can use envsubst for this value.
+    index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     python_version = "3.9",
     requirements_lock = "requirements_lock_3_9.txt",
     # These modifications were created above and we
@@ -192,7 +192,7 @@ pip.parse(
         "@whl_mods_hub//:wheel.json": "wheel",
     },
 )
-pip.parse(
+pypi.install(
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -240,7 +240,7 @@ pip.parse(
 # You can add patches that will be applied on the whl contents.
 #
 # The patches have to be in the unified-diff format.
-pip.override(
+pypi.override(
     file = "requests-2.25.1-py2.py3-none-any.whl",
     patch_strip = 1,
     patches = [
@@ -249,7 +249,7 @@ pip.override(
         "@//patches:requests_record.patch",
     ],
 )
-use_repo(pip, "pip")
+use_repo(pypi, "pip")
 
 bazel_dep(name = "other_module", version = "", repo_name = "our_other_module")
 local_path_override(