Skip to content

Commit 592f227

Browse files
shanedsnydershanedsnyder
authored
MAINT, CI: cibuildwheel support (#741)
* pyproject.toml updates to support cibuildwheel * drop '--build-ext' option in setup.py for env var This option to build the C extension is only used in the wheel building process. cibuildwheel does not appear to support an ability to pass options to setuptools, and even if it did there appears to be no support for pip installing wheels with flags like '--build-option' (see pypa/pip#5771). This commit also updates our build-wheels script to use this env var, even though those scripts are more or less deprecated. * add first cut at build_wheels github action * tell build_wheels action about pydarshan directory * checkout autoperf submodule before building wheels * pin matplotlib<3.5 for cibuildwheel test phase see gh-479 for more details * avoid macos builds for now * move cibuildwheel linux specific bits * only build cpython, re-enable macos builds * install automake on macos * bug fix in specifying cibuildwheel before-all * debug pypy 3.9 builds * add more brew dependencies for macos * skip pypy builds, as well as ppc/s390x * explicitly specicy repair wheel commands * restrict when wheels are built for now, just runs manually via GitHub actions, or by appending a message to the commit message * add dummy file to test commit message wheel hooks [wheel build] * forgot part of the github workflow config [wheel build] * actually add the get commit message job [wheel build] * let's try one more time... [wheel build] * allow manual running of wheels workflow * update release checklist to reflect wheel changes
1 parent d35f49d commit 592f227

File tree

5 files changed

+117
-8
lines changed

5 files changed

+117
-8
lines changed

‎.github/workflows/build_wheels.yml

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Build Wheels
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- darshan-util/**
9+
- include/**
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- darshan-util/**
15+
- include/*
16+
workflow_dispatch:
17+
18+
jobs:
19+
get_commit_message:
20+
name: Get commit message
21+
runs-on: ubuntu-latest
22+
if: github.repository == 'darshan-hpc/darshan'
23+
outputs:
24+
message: ${{ steps.commit_message.outputs.message }}
25+
steps:
26+
- name: Checkout darshan
27+
uses: actions/checkout@v3
28+
# Gets the correct commit message for pull request
29+
with:
30+
ref: ${{ github.event.pull_request.head.sha }}
31+
- name: Get commit message
32+
id: commit_message
33+
run: |
34+
set -xe
35+
COMMIT_MSG=$(git log --no-merges -1 --oneline)
36+
echo "::set-output name=message::$COMMIT_MSG"
37+
echo github.ref ${{ github.ref }}
38+
39+
build_wheels:
40+
name: Build wheels on ${{ matrix.os }}
41+
needs: get_commit_message
42+
if: >-
43+
contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
44+
github.event_name == 'workflow_dispatch'
45+
runs-on: ${{ matrix.os }}
46+
strategy:
47+
matrix:
48+
os: [ubuntu-latest, macos-latest]
49+
50+
steps:
51+
- uses: actions/checkout@v2
52+
53+
- name: Build wheels
54+
uses: pypa/[email protected]
55+
with:
56+
package-dir: ./darshan-util/pydarshan
57+
58+
- uses: actions/upload-artifact@v2
59+
with:
60+
path: ./wheelhouse/*.whl

‎darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt

+9-5
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,15 @@ Notes on how to release a new version of PyDarshan
2525
(flake8 syntax warnings can be ignored)
2626
- TODO: CI?
2727

28-
- Submit to PyPi using twine:
29-
- make wheels # requires docker, creates ./wheelhouse and builds architecture-specific *.whl that include libdarshan-util
30-
- make dist # gathers relevant wheels build earlier, adds non-binary wheel and a source distribution (zip/tgz)
31-
- make release # pushes contents of ./dist/* to PyPi
32-
(be prompted for username/password)
28+
- Submit to PyPI using cibuildwheel/twine:
29+
- pipx run build --sdist
30+
(in the pydarshan directory, creates a source distribution)
31+
- download wheel artifacts from GitHub, copy to 'dist/' directory created above
32+
(click on Actions in GitHub, find most recent "Build Wheels" workflow run with artifacts)
33+
(download artifact zip, extract wheels to 'dist/' directory)
34+
- pipx run twine upload dist/*
35+
(requires PyPI credentials for darshan project)
36+
(use '--repository testpypi' to test wheels using https://test.pypi.org/project/darshan/ before uploading to PyPI)
3337

3438
- Add/update spack package: py-darshan
3539
- add version entry

‎darshan-util/pydarshan/devel/build-wheels.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,15 @@ cd /
4141

4242
ls /opt/python
4343

44+
# Force setup.py to build the C extension
45+
export PYDARSHAN_BUILD_EXT=1
4446

4547
# Compile wheels
4648
for PYBIN in /opt/python/*/bin; do
4749
# JL: we do not really need any dependencies to build the wheel,
4850
# but requirements install needs to be renabled when testing automatically
4951
#"${PYBIN}/pip" install -r /io/requirements_wheels.txt
50-
"${PYBIN}/pip" wheel /io/ --build-option "--with-extension" --no-deps -w /io/wheelhouse/${PLAT}
52+
"${PYBIN}/pip" wheel /io/ --no-deps -w /io/wheelhouse/${PLAT}
5153
done
5254

5355
# Bundle external shared libraries into the wheels

‎darshan-util/pydarshan/pyproject.toml

+44
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,47 @@ requires = [
33
"wheel",
44
"setuptools",
55
]
6+
7+
[tool.cibuildwheel]
8+
environment = "PYDARSHAN_BUILD_EXT=1"
9+
skip = [
10+
"pp*",
11+
"*musllinux*",
12+
"*i686*",
13+
"*_ppc64le",
14+
"*_s390x"
15+
]
16+
test-requires = [
17+
"pytest",
18+
"lxml",
19+
"matplotlib<3.5",
20+
"importlib_resources;python_version<'3.9'"
21+
]
22+
test-command = "pytest {package}"
23+
24+
[tool.cibuildwheel.linux]
25+
before-all = [
26+
"yum install -y blas-devel lapack-devel",
27+
"yum install -y libxslt-devel libxml2-devel",
28+
"yum install -y libjpeg libjpeg-devel",
29+
"git submodule update --init",
30+
"./prepare.sh",
31+
"./configure --disable-darshan-runtime --enable-apxc-mod --enable-apmpi-mod",
32+
"make install"
33+
]
34+
repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
35+
36+
[tool.cibuildwheel.macos]
37+
before-all = [
38+
"brew install automake",
39+
"brew install openblas",
40+
"brew install lapack",
41+
"git submodule update --init",
42+
"./prepare.sh",
43+
"./configure --disable-darshan-runtime --enable-apxc-mod --enable-apmpi-mod",
44+
"make install"
45+
]
46+
repair-wheel-command = [
47+
"delocate-listdeps {wheel}",
48+
"delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}",
49+
]

‎darshan-util/pydarshan/setup.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
# discoverable in the environment by means of LD_LIBRARY_PATH or
2929
# pkg-config there is no need to build the extension.
3030
ext_modules = []
31-
if "--with-extension" in sys.argv:
31+
if "PYDARSHAN_BUILD_EXT" in os.environ:
3232
ext_modules.append(
3333
Extension(
3434
"darshan.extension",
@@ -38,7 +38,6 @@
3838
libraries=["darshan-util"],
3939
)
4040
)
41-
sys.argv.remove("--with-extension")
4241

4342
#
4443
# Find backend python files in modules and copy them into lib

0 commit comments

Comments
 (0)