Skip to content

feat(musl): add musl toolchain #2402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ Other changes:
`RULES_PYTHON_REPO_TOOLCHAIN_{VERSION}_{OS}_{ARCH}` env variable setting. For
example, this allows one to use `freethreaded` python interpreter in the
`repository_rule` to build a wheel from `sdist`.
* (toolchain) The python interpreters targeting `muslc` libc have been added
for the latest toolchain versions for each minor Python version. You can control
the toolchain selection by using the
{bzl:obj}`//python/config_settings:py_linux_libc` build flag.

{#v0-0-0-removed}
### Removed
Expand Down
7 changes: 7 additions & 0 deletions docs/toolchains.md
Original file line number Diff line number Diff line change
Expand Up @@ -451,3 +451,10 @@ The toolchain() calls should be in a separate BUILD file from everything else.
This avoids Bazel having to perform unnecessary work when it discovers the list
of available toolchains.
:::

## Toolchain selection flags

Currently the following flags are used to influence toolchain selection:
* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
* {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting
the freethreaded experimental Python builds available from `3.13.0` onwards.
6 changes: 3 additions & 3 deletions python/config_settings/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ load(
"BootstrapImplFlag",
"ExecToolsToolchainFlag",
"FreeThreadedFlag",
"LibcFlag",
"PrecompileFlag",
"PrecompileSourceRetentionFlag",
)
load(
"//python/private/pypi:flags.bzl",
"UniversalWhlFlag",
"UseWhlFlag",
"WhlLibcFlag",
"define_pypi_internal_flags",
)
load(":config_settings.bzl", "construct_config_settings")
Expand Down Expand Up @@ -87,8 +87,8 @@ string_flag(
# This is used for pip and hermetic toolchain resolution.
string_flag(
name = "py_linux_libc",
build_setting_default = WhlLibcFlag.GLIBC,
values = sorted(WhlLibcFlag.__members__.values()),
build_setting_default = LibcFlag.GLIBC,
values = LibcFlag.flag_values(),
# NOTE: Only public because it is used in pip hub and toolchain repos.
visibility = ["//visibility:public"],
)
Expand Down
11 changes: 11 additions & 0 deletions python/private/flags.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,14 @@ FreeThreadedFlag = enum(
# Do not use freethreaded python toolchain and wheels.
NO = "no",
)

# Determines which libc flavor is preferred when selecting the toolchain and
# linux whl distributions.
#
# buildifier: disable=name-conventions
LibcFlag = FlagEnum(
# Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
GLIBC = "glibc",
# Prefer musl wheels (e.g. musllinux_2_17_x86_64)
MUSL = "musl",
)
13 changes: 7 additions & 6 deletions python/private/pypi/config_settings.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ Note, that here the specialization of musl vs manylinux wheels is the same in
order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
"""

load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag")
load("//python/private:flags.bzl", "LibcFlag")
load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag")

FLAGS = struct(
**{
Expand Down Expand Up @@ -251,14 +252,14 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):

elif os == "linux":
for os_prefix, linux_libc in {
os: WhlLibcFlag.GLIBC,
"many" + os: WhlLibcFlag.GLIBC,
"musl" + os: WhlLibcFlag.MUSL,
os: LibcFlag.GLIBC,
"many" + os: LibcFlag.GLIBC,
"musl" + os: LibcFlag.MUSL,
}.items():
if linux_libc == WhlLibcFlag.GLIBC:
if linux_libc == LibcFlag.GLIBC:
libc_versions = glibc_versions
libc_flag = FLAGS.pip_whl_glibc_version
elif linux_libc == WhlLibcFlag.MUSL:
elif linux_libc == LibcFlag.MUSL:
libc_versions = muslc_versions
libc_flag = FLAGS.pip_whl_muslc_version
else:
Expand Down
10 changes: 0 additions & 10 deletions python/private/pypi/flags.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,6 @@ UniversalWhlFlag = enum(
UNIVERSAL = "universal",
)

# Determines which libc flavor is preferred when selecting the linux whl distributions.
#
# buildifier: disable=name-conventions
WhlLibcFlag = enum(
# Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
GLIBC = "glibc",
# Prefer musl wheels (e.g. musllinux_2_17_x86_64)
MUSL = "musl",
)

INTERNAL_FLAGS = [
"dist",
"whl_plat",
Expand Down
17 changes: 17 additions & 0 deletions python/versions.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "193dc7f0284e4917d52b17a077924474882ee172872f2257cfe3375d6d468ed9",
"x86_64-pc-windows-msvc": "5069008a237b90f6f7a86956903f2a0221b90d471daa6e4a94831eaa399e3993",
"x86_64-unknown-linux-gnu": "c20ee831f7f46c58fa57919b75a40eb2b6a31e03fd29aaa4e8dab4b9c4b60d5d",
"x86_64-unknown-linux-musl": "5c1cc348e317fe7af1acd6a7f665b46eccb554b20d6533f0e76c53f44d4556cc",
},
"strip_prefix": "python",
},
Expand Down Expand Up @@ -367,6 +368,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "90b46dfb1abd98d45663c7a2a8c45d3047a59391d8586d71b459cec7b75f662b",
"x86_64-pc-windows-msvc": "e48952619796c66ec9719867b87be97edca791c2ef7fbf87d42c417c3331609e",
"x86_64-unknown-linux-gnu": "3db2171e03c1a7acdc599fba583c1b92306d3788b375c9323077367af1e9d9de",
"x86_64-unknown-linux-musl": "ed519c47d9620eb916a6f95ec2875396e7b1a9ab993ee40b2f31b837733f318c",
},
"strip_prefix": "python",
},
Expand Down Expand Up @@ -481,6 +483,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "1e23ffe5bc473e1323ab8f51464da62d77399afb423babf67f8e13c82b69c674",
"x86_64-pc-windows-msvc": "647b66ff4552e70aec3bf634dd470891b4a2b291e8e8715b3bdb162f577d4c55",
"x86_64-unknown-linux-gnu": "8b50a442b04724a24c1eebb65a36a0c0e833d35374dbdf9c9470d8a97b164cd9",
"x86_64-unknown-linux-musl": "d36fc77a8dd76155a7530f6235999a693b9e7c48aa11afeb5610a091cae5aa6f",
},
"strip_prefix": "python",
},
Expand Down Expand Up @@ -559,6 +562,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "60c5271e7edc3c2ab47440b7abf4ed50fbc693880b474f74f05768f5b657045a",
"x86_64-pc-windows-msvc": "f05531bff16fa77b53be0776587b97b466070e768e6d5920894de988bdcd547a",
"x86_64-unknown-linux-gnu": "43576f7db1033dd57b900307f09c2e86f371152ac8a2607133afa51cbfc36064",
"x86_64-unknown-linux-musl": "5ed4a4078db3cbac563af66403aaa156cd6e48831d90382a1820db2b120627b5",
},
"strip_prefix": "python",
},
Expand All @@ -572,6 +576,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "cff1b7e7cd26f2d47acac1ad6590e27d29829776f77e8afa067e9419f2f6ce77",
"x86_64-pc-windows-msvc": "b25926e8ce4164cf103bacc4f4d154894ea53e07dd3fdd5ebb16fb1a82a7b1a0",
"x86_64-unknown-linux-gnu": "2c8cb15c6a2caadaa98af51df6fe78a8155b8471cb3dd7b9836038e0d3657fb4",
"x86_64-unknown-linux-musl": "2f61ee3b628a56aceea63b46c7afe2df3e22a61da706606b0c8efda57f953cf4",
"aarch64-apple-darwin-freethreaded": "efc2e71c0e05bc5bedb7a846e05f28dd26491b1744ded35ed82f8b49ccfa684b",
"aarch64-unknown-linux-gnu-freethreaded": "59b50df9826475d24bb7eff781fa3949112b5e9c92adb29e96a09cdf1216d5bd",
"ppc64le-unknown-linux-gnu-freethreaded": "1217efa5f4ce67fcc9f7eb64165b1bd0912b2a21bc25c1a7e2cb174a21a5df7e",
Expand All @@ -588,6 +593,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "python",
"x86_64-pc-windows-msvc": "python",
"x86_64-unknown-linux-gnu": "python",
"x86_64-unknown-linux-musl": "python",
"aarch64-apple-darwin-freethreaded": "python/install",
"aarch64-unknown-linux-gnu-freethreaded": "python/install",
"ppc64le-unknown-linux-gnu-freethreaded": "python/install",
Expand Down Expand Up @@ -727,6 +733,17 @@ def _generate_platforms():
# Matches the value in @platforms//cpu package
arch = "x86_64",
),
"x86_64-unknown-linux-musl": struct(
compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
flag_values = {
libc: "musl",
},
os_name = LINUX_NAME,
arch = "x86_64",
),
}

freethreaded = Label("//python/config_settings:py_freethreaded")
Expand Down