Skip to content

feat: use limited API to build PyPI wheels #254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

feat: use limited API to build PyPI wheels #254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b0f076b
feat: use limited API to build PyPI wheels
mayeut Oct 2, 2024
71c878b
address review comments
mayeut Mar 8, 2025
ca6ac38
Merge branch 'main' into limited-api
parthea Mar 17, 2025
bf19681
Merge branch 'main' into limited-api
parthea Mar 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[build-system]
# Minimum requirements for the build system to execute.
requires = ["setuptools", "wheel"] # PEP 508 specifications.
requires = ["setuptools"] # PEP 508 specifications.
48 changes: 7 additions & 41 deletions scripts/manylinux/build_on_centos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,48 +39,14 @@ ${MAIN_PYTHON_BIN}/cmake \
..
make all install

PYTHON_VERSIONS=""
if [[ -z ${BUILD_PYTHON} ]]; then
# Collect all target Python versions.
for PYTHON_BIN in /opt/python/*/bin; do
# H/T: https://stackoverflow.com/a/229606/1068170
if [[ "${PYTHON_BIN}" == *"39"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
continue
elif [[ "${PYTHON_BIN}" == *"310"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
continue
elif [[ "${PYTHON_BIN}" == *"311"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
continue
elif [[ "${PYTHON_BIN}" == *"312"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
continue
elif [[ "${PYTHON_BIN}" == *"313"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
continue
else
echo "Ignoring unsupported version: ${PYTHON_BIN}"
echo "====================================="
fi
done
else
STRIPPED_PYTHON=$(echo ${BUILD_PYTHON} | sed -e "s/\.//g" | sed -e "s/-dev$//")
for PYTHON_BIN in /opt/python/*/bin; do
if [[ "${PYTHON_BIN}" == *"${STRIPPED_PYTHON}"* ]]; then
PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}"
fi
done
fi

# Build the wheels.
# Build the wheel.
export CRC32C_PURE_PYTHON=0
export CRC32C_LIMITED_API=1
cd ${REPO_ROOT}
for PYTHON_BIN in ${PYTHON_VERSIONS}; do
${PYTHON_BIN}/python -m pip install --upgrade pip
${PYTHON_BIN}/python -m pip install \
--requirement ${REPO_ROOT}/scripts/dev-requirements.txt
${PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/
done
${MAIN_PYTHON_BIN}/python -m pip install --upgrade pip
${MAIN_PYTHON_BIN}/python -m pip install \
--requirement ${REPO_ROOT}/scripts/dev-requirements.txt
${MAIN_PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/

# Bundle external shared libraries into the wheels
for whl in dist_wheels/google_crc32c*.whl; do
Expand Down
5 changes: 4 additions & 1 deletion scripts/manylinux/check.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ install_python_pyenv() {

SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12" "3.13")

PYTHON_VERSION_MIN=${SUPPORTED_PYTHON_VERSIONS[0]}
PY_TAG="cp${PYTHON_VERSION_MIN//.}-abi3"

for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
PYTHON=python${PYTHON_VERSION}
install_python_pyenv ${PYTHON_VERSION}
Expand All @@ -48,7 +51,7 @@ for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
${PYTHON} -m venv venv

# Install the wheel.
WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-${PY_TAG}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
venv/bin/pip install ${WHEEL_FILE}

# Verify that the module is installed and peek at contents.
Expand Down
17 changes: 9 additions & 8 deletions scripts/osx/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,15 @@ git submodule update --init --recursive

${OSX_DIR}/build_c_lib.sh

SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12" "3.13")

for PYTHON_VERSION in ${SUPPORTED_PYTHON_VERSIONS[@]}; do
echo "Build wheel for Python ${PYTHON_VERSION}"
export PY_BIN=$PYTHON_VERSION
export PY_TAG="cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}"
. /${OSX_DIR}/build_python_wheel.sh
done
export CRC32C_PURE_PYTHON=0
export CRC32C_LIMITED_API=1
SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12", "3.13")
PYTHON_VERSION=${SUPPORTED_PYTHON_VERSIONS[0]};
echo "Build wheel for Python ${PYTHON_VERSION}"
export PY_BIN=$PYTHON_VERSION
export PY_TAG="cp${PYTHON_VERSION//.}-abi3"
. /${OSX_DIR}/build_python_wheel.sh


# Clean up.
rm -fr ${CRC32C_INSTALL_PREFIX}
8 changes: 8 additions & 0 deletions scripts/windows/build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ setlocal ENABLEDELAYEDEXPANSION
set CMAKE_GENERATOR="Visual Studio 17 2022"
set CONFIGURATION=RelWithDebInfo
set CRC32C_INSTALL_PREFIX=%cd%\build\%CONFIGURATION%
set CRC32C_PURE_PYTHON=0
set CRC32C_LIMITED_API=1
set BUILT=0

@rem Iterate through supported Python versions.
@rem Unfortunately pyenv for Windows has an out-of-date versions list. Choco's
Expand All @@ -33,6 +36,8 @@ FOR %%P IN (3.9, 3.10, 3.11, 3.12, 3.13.1) DO (

py -%%P-64 -m pip install --upgrade pip

if "%BUILT%"=="1" (goto test)

echo "Installing cmake for Python %%P"
py -%%P-64 -m pip install cmake

Expand Down Expand Up @@ -73,6 +78,9 @@ FOR %%P IN (3.9, 3.10, 3.11, 3.12, 3.13.1) DO (
echo "Building Wheel"
py -%%P-64 -m pip wheel . --wheel-dir wheels/

set BUILT=1

:test
echo "Built wheel, now running tests."
call %~dp0/test.bat %%P || goto :error

Expand Down
11 changes: 10 additions & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
CRC32C_PURE_PYTHON_EXPLICIT = "CRC32C_PURE_PYTHON" in os.environ
_FALSE_OPTIONS = ("0", "false", "no", "False", "No", None)
CRC32C_PURE_PYTHON = os.getenv("CRC32C_PURE_PYTHON") not in _FALSE_OPTIONS
# Wether or not we want to build using limited API (ABI3)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: Whether is missing an h

CRC32C_LIMITED_API = os.getenv("CRC32C_LIMITED_API") not in _FALSE_OPTIONS


def copy_dll(build_lib):
Expand Down Expand Up @@ -90,6 +92,13 @@ def build_c_extension():
print("#### using global install of 'crc32c'")
kwargs = {}

if CRC32C_LIMITED_API:
kwargs["define_macros"] = [("Py_LIMITED_API", "0x03090000")]
kwargs["py_limited_api"] = True
options = {"bdist_wheel": {"py_limited_api": "cp39"}}
else:
options = {}

module_path = os.path.join("src", "google_crc32c", "_crc32c.c")
sources=[os.path.normcase(module_path)]
print(f"##### sources: {sources}")
Expand All @@ -106,7 +115,7 @@ def build_c_extension():
package_dir={"": "src"},
ext_modules=[module],
cmdclass={"build_ext": BuildExtWithDLL},
install_requires=["importlib_resources>=1.3 ; python_version < '3.9' and os_name == 'nt'"],
options=options,
)


Expand Down