From 5562b04b702402eac3256ae453edf2fd54f16ce6 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 12:27:08 -0600 Subject: [PATCH 01/10] test plugin install --- .github/workflows/build-wheels-linux.yml | 42 +++++++++++----------- .github/workflows/build-wheels-macosx.yml | 8 +++-- .github/workflows/build-wheels-macosx2.yml | 6 +++- config.sh | 34 ++++++++++-------- 4 files changed, 52 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 35e3311..39800f7 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -24,28 +24,28 @@ jobs: fail-fast: false matrix: - #python-version: [ "3.10"] - #os: [ubuntu-latest] - #platform: [x64] - python-version: [ "3.6", "3.7", "3.8", "3.9", "3.10" ] + python-version: [ "3.10"] os: [ubuntu-latest] - platform: [x64, x32] - include: - - python-version: "3.6" - os: ubuntu-latest - platform: aarch64 - - python-version: "3.7" - os: ubuntu-latest - platform: aarch64 - - python-version: "3.8" - os: ubuntu-latest - platform: aarch64 - - python-version: "3.9" - os: ubuntu-latest - platform: aarch64 - - python-version: "3.10" - os: ubuntu-latest - platform: aarch64 + platform: [x64] + #python-version: [ "3.6", "3.7", "3.8", "3.9", "3.10" ] + #os: [ubuntu-latest] + #platform: [x64, x32] + #include: + # - python-version: "3.6" + # os: ubuntu-latest + # platform: aarch64 + # - python-version: "3.7" + # os: ubuntu-latest + # platform: aarch64 + # - python-version: "3.8" + # os: ubuntu-latest + # platform: aarch64 + # - python-version: "3.9" + # os: ubuntu-latest + # platform: aarch64 + # - python-version: "3.10" + # os: ubuntu-latest + # platform: aarch64 env: REPO_DIR: netcdf4-python diff --git a/.github/workflows/build-wheels-macosx.yml b/.github/workflows/build-wheels-macosx.yml index 9822e13..632eb8d 100644 --- a/.github/workflows/build-wheels-macosx.yml +++ b/.github/workflows/build-wheels-macosx.yml @@ -24,7 +24,8 @@ jobs: fail-fast: false matrix: - python-version: [ "3.8", "3.9","3.10" ] + #python-version: [ "3.8", "3.9","3.10" ] + python-version: [ "3.10" ] os: [macos-latest] platform: [x86_64] env: @@ -101,7 +102,10 @@ jobs: clean_code $REPO_DIR $BUILD_COMMIT echo "------- BUILD WHEEL --------" build_wheel $REPO_DIR $PLAT - echo "------- INSTALL_RUN --------" + echo "check wheel" + ls -l ${{ github.workspace }}/wheelhouse + unzip -l ${{ github.workspace }}/wheelhouse/netCDF4*whl + #echo "------- INSTALL_RUN --------" #install_run $PLAT - name: Upload wheels to release uses: svenstaro/upload-release-action@v2 diff --git a/.github/workflows/build-wheels-macosx2.yml b/.github/workflows/build-wheels-macosx2.yml index 1609a48..41e31ca 100644 --- a/.github/workflows/build-wheels-macosx2.yml +++ b/.github/workflows/build-wheels-macosx2.yml @@ -24,7 +24,8 @@ jobs: fail-fast: false matrix: - python-version: [ "3.8", "3.9","3.10" ] + #python-version: [ "3.8", "3.9","3.10" ] + python-version: [ "3.10" ] os: [macos-latest] platform: [x86_64] env: @@ -101,6 +102,9 @@ jobs: clean_code $REPO_DIR $BUILD_COMMIT echo "------- BUILD WHEEL --------" build_wheel $REPO_DIR $PLAT + echo "check wheel" + ls -l ${{ github.workspace }}/wheelhouse + unzip -l ${{ github.workspace }}/wheelhouse/netCDF4*whl echo "------- INSTALL_RUN --------" install_run $PLAT - name: Upload wheels to release diff --git a/config.sh b/config.sh index 1e336ed..a4f786f 100644 --- a/config.sh +++ b/config.sh @@ -8,7 +8,7 @@ export NO_CDL=1 # uncomment build_wheel and enable plugin install in build_netcdf. # if plugins not include, plugins will not work with these wheels unless user sets # HDF5_PLUGIN_PATH to point to locally installed plugins. -export NO_PLUGINS=1 +#export NO_PLUGINS=1 # Compile libs for macOS 10.9 or later export MACOSX_DEPLOYMENT_TARGET="10.9" @@ -25,11 +25,11 @@ export BZIP2_VERSION="1.0.8" export BLOSC_VERSION="1.21.1" # custom version that sets NETCDF_PLUGIN_DIR env var -#function build_wheel { -# # Set default building method to pip -# export NETCDF_PLUGIN_DIR=${BUILD_PREFIX}/lib/netcdf-plugins -# wrap_wheel_builder build_pip_wheel $@ -#} +function build_wheel { + # Set default building method to pip + export NETCDF_PLUGIN_DIR=${BUILD_PREFIX}/lib/netcdf-plugins + wrap_wheel_builder build_pip_wheel $@ +} # add --verbose to pip function pip_opts { @@ -124,17 +124,23 @@ function build_netcdf { && make install ) fi else + # plugins installed + (cd netcdf-c-${NETCDF_VERSION} \ + && export HDF5_PLUGIN_PATH=$BUILD_PREFIX/lib/netcdf-plugins \ + && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH \ + && make -j4 \ + && make install ) # use cmake for version 4.9.0 since autotools doesn't work # CMakeLists.txt patch needed for NETCDF_VERSION 4.9.0 # no plugins installed - (cd netcdf-c-${NETCDF_VERSION} \ - && curl https://raw.githubusercontent.com/MacPython/netcdf4-python-wheels/master/CMakeLists.txt.patch -o CMakeLists.txt.patch \ - && patch -p0 < CMakeLists.txt.patch \ - && mkdir build \ - && cd build \ - && cmake ../ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} -DENABLE_NETCDF_4=ON -DENABLE_DAP=ON -DBUILD_SHARED_LIBS=ON -DENABLE_PLUGIN_INSTALL=NO \ - && make -j4 \ - && make install ) + #(cd netcdf-c-${NETCDF_VERSION} \ + # && curl https://raw.githubusercontent.com/MacPython/netcdf4-python-wheels/master/CMakeLists.txt.patch -o CMakeLists.txt.patch \ + # && patch -p0 < CMakeLists.txt.patch \ + # && mkdir build \ + # && cd build \ + # && cmake ../ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} -DENABLE_NETCDF_4=ON -DENABLE_DAP=ON -DBUILD_SHARED_LIBS=ON -DENABLE_PLUGIN_INSTALL=NO \ + # && make -j4 \ + # && make install ) # plugins installed #(cd netcdf-c-${NETCDF_VERSION} \ # && curl https://raw.githubusercontent.com/MacPython/netcdf4-python-wheels/master/CMakeLists.txt.patch -o CMakeLists.txt.patch \ From c48612e2abeccc91031280620941937090501344 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 12:39:01 -0600 Subject: [PATCH 02/10] update --- netcdf4-python | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netcdf4-python b/netcdf4-python index f479c12..c6d29ec 160000 --- a/netcdf4-python +++ b/netcdf4-python @@ -1 +1 @@ -Subproject commit f479c122ee581a86c153a0a0ab79b331a4e79849 +Subproject commit c6d29ec96f8e089eb3b384546f8ea497e8d2f310 From b9321e247259ffaa791286b60db5999e16c8331e Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 12:51:09 -0600 Subject: [PATCH 03/10] update --- .github/workflows/build-wheels-linux.yml | 2 +- .github/workflows/build-wheels-macosx.yml | 2 +- .github/workflows/build-wheels-macosx2.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 39800f7..a9c7552 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -51,7 +51,7 @@ jobs: REPO_DIR: netcdf4-python PKG_NAME: netcdf4-python MB_ML_VER: 2014 - BUILD_COMMIT: v1.6.0rel + BUILD_COMMIT: test_plugins UNICODE_WIDTH: 32 MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/.github/workflows/build-wheels-macosx.yml b/.github/workflows/build-wheels-macosx.yml index 632eb8d..0bc6ce9 100644 --- a/.github/workflows/build-wheels-macosx.yml +++ b/.github/workflows/build-wheels-macosx.yml @@ -32,7 +32,7 @@ jobs: REPO_DIR: netcdf4-python PKG_NAME: netcdf4-python MB_ML_VER: 2014 - BUILD_COMMIT: v1.6.0rel + BUILD_COMMIT: test_plugins UNICODE_WIDTH: 32 PLAT: arm64 MB_PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/.github/workflows/build-wheels-macosx2.yml b/.github/workflows/build-wheels-macosx2.yml index 41e31ca..ce73747 100644 --- a/.github/workflows/build-wheels-macosx2.yml +++ b/.github/workflows/build-wheels-macosx2.yml @@ -32,7 +32,7 @@ jobs: REPO_DIR: netcdf4-python PKG_NAME: netcdf4-python MB_ML_VER: 2014 - BUILD_COMMIT: v1.6.0rel + BUILD_COMMIT: test_plugins UNICODE_WIDTH: 32 PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} From 1efde5e3ef8121d8b9fc19b6601849aaab8aef0f Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 13:04:43 -0600 Subject: [PATCH 04/10] try github main --- config.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index a4f786f..342e2da 100644 --- a/config.sh +++ b/config.sh @@ -107,7 +107,8 @@ function build_zstd { function build_netcdf { if [ -e netcdf-stamp ]; then return; fi - fetch_unpack https://downloads.unidata.ucar.edu/netcdf-c/${NETCDF_VERSION}/netcdf-c-${NETCDF_VERSION}.tar.gz + #fetch_unpack https://downloads.unidata.ucar.edu/netcdf-c/${NETCDF_VERSION}/netcdf-c-${NETCDF_VERSION}.tar.gz + git clone https://github.com/Unidata/netcdf-c netcdf-c-${NETCDF_VERSION} if [ -n "$IS_MACOS" ]; then if [[ "$PLAT" = "arm64" ]] && [[ "$CROSS_COMPILING" = "1" ]]; then # no plugins installed @@ -124,7 +125,7 @@ function build_netcdf { && make install ) fi else - # plugins installed + # use autotools, plugins installed (cd netcdf-c-${NETCDF_VERSION} \ && export HDF5_PLUGIN_PATH=$BUILD_PREFIX/lib/netcdf-plugins \ && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH \ From 801006534f0ce30e2e72e269c33b422f87bab233 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 14:01:30 -0600 Subject: [PATCH 05/10] update --- config.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.sh b/config.sh index 342e2da..1bd5506 100644 --- a/config.sh +++ b/config.sh @@ -109,6 +109,9 @@ function build_netcdf { if [ -e netcdf-stamp ]; then return; fi #fetch_unpack https://downloads.unidata.ucar.edu/netcdf-c/${NETCDF_VERSION}/netcdf-c-${NETCDF_VERSION}.tar.gz git clone https://github.com/Unidata/netcdf-c netcdf-c-${NETCDF_VERSION} + cd netcdf-c-${NETCDF_VERSION} + autoreconf -i + cd .. if [ -n "$IS_MACOS" ]; then if [[ "$PLAT" = "arm64" ]] && [[ "$CROSS_COMPILING" = "1" ]]; then # no plugins installed From d5e2b4a76dfa7ac2f0db32972bd4142c77152a45 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 16:26:42 -0600 Subject: [PATCH 06/10] disable libxml2 --- config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sh b/config.sh index 1bd5506..d6856ed 100644 --- a/config.sh +++ b/config.sh @@ -131,7 +131,7 @@ function build_netcdf { # use autotools, plugins installed (cd netcdf-c-${NETCDF_VERSION} \ && export HDF5_PLUGIN_PATH=$BUILD_PREFIX/lib/netcdf-plugins \ - && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH \ + && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH --disable-libxml2 \ && make -j4 \ && make install ) # use cmake for version 4.9.0 since autotools doesn't work From a20fe0dc123676723340674a4009e3abe2d93306 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 16:42:09 -0600 Subject: [PATCH 07/10] try not building zstd --- config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index d6856ed..845937a 100644 --- a/config.sh +++ b/config.sh @@ -261,8 +261,8 @@ function build_libs { build_lzo echo "build_lzf" build_lzf - echo "build_zstd" - build_zstd + #echo "build_zstd" + #build_zstd echo "build_bzip2" build_bzip2 echo "build_blosc" From 618c93345bdf84b15cff5e676f236a2bef3dc57a Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 17:03:57 -0600 Subject: [PATCH 08/10] try caching sizeof off_t --- config.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index 845937a..07e6056 100644 --- a/config.sh +++ b/config.sh @@ -131,6 +131,7 @@ function build_netcdf { # use autotools, plugins installed (cd netcdf-c-${NETCDF_VERSION} \ && export HDF5_PLUGIN_PATH=$BUILD_PREFIX/lib/netcdf-plugins \ + && export ac_cv_sizeof_off_t=8 \ && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH --disable-libxml2 \ && make -j4 \ && make install ) @@ -261,8 +262,8 @@ function build_libs { build_lzo echo "build_lzf" build_lzf - #echo "build_zstd" - #build_zstd + echo "build_zstd" + build_zstd echo "build_bzip2" build_bzip2 echo "build_blosc" From ec636cee3b220bbb5a3756718de1cbec8a643719 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 17:26:56 -0600 Subject: [PATCH 09/10] try zstd 1.5.0 --- config.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config.sh b/config.sh index 07e6056..e67e0bf 100644 --- a/config.sh +++ b/config.sh @@ -19,7 +19,7 @@ export OPENSSL_ROOT=openssl-1.0.2u export OPENSSL_HASH=ecd0c6ffb493dd06707d38b14bb4d8c2288bb7033735606569d8f90f89669d16 export CURL_VERSION="7.75.0" export LIBAEC_VERSION="1.0.6" -export ZSTD_VERSION="1.5.2" +export ZSTD_VERSION="1.5.0" export LZ4_VERSION="1.9.3" export BZIP2_VERSION="1.0.8" export BLOSC_VERSION="1.21.1" @@ -131,7 +131,6 @@ function build_netcdf { # use autotools, plugins installed (cd netcdf-c-${NETCDF_VERSION} \ && export HDF5_PLUGIN_PATH=$BUILD_PREFIX/lib/netcdf-plugins \ - && export ac_cv_sizeof_off_t=8 \ && ./configure --prefix=$BUILD_PREFIX --enable-netcdf-4 --enable-shared --enable-dap --with-plugin-dir=$HDF5_PLUGIN_PATH --disable-libxml2 \ && make -j4 \ && make install ) From 64fb3fe2c2302518c9a1409b36ad4a6c0da95a42 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 25 Jun 2022 19:12:26 -0600 Subject: [PATCH 10/10] update --- .github/workflows/build-wheels-linux.yml | 2 +- config.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index a9c7552..286cfd4 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -50,7 +50,7 @@ jobs: env: REPO_DIR: netcdf4-python PKG_NAME: netcdf4-python - MB_ML_VER: 2014 + MB_ML_VER: _2_24 BUILD_COMMIT: test_plugins UNICODE_WIDTH: 32 MB_PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/config.sh b/config.sh index e67e0bf..461f91d 100644 --- a/config.sh +++ b/config.sh @@ -19,7 +19,7 @@ export OPENSSL_ROOT=openssl-1.0.2u export OPENSSL_HASH=ecd0c6ffb493dd06707d38b14bb4d8c2288bb7033735606569d8f90f89669d16 export CURL_VERSION="7.75.0" export LIBAEC_VERSION="1.0.6" -export ZSTD_VERSION="1.5.0" +export ZSTD_VERSION="1.5.2" export LZ4_VERSION="1.9.3" export BZIP2_VERSION="1.0.8" export BLOSC_VERSION="1.21.1" @@ -92,16 +92,16 @@ function build_zstd { local root_name=v${ZSTD_VERSION} local tar_name=zstd-${root_name}.tar.gz fetch_unpack https://github.com/facebook/zstd/releases/download/${root_name}/zstd-${ZSTD_VERSION}.tar.gz - #(cd zstd-${ZSTD_VERSION} \ - # && cd build \ - # && mkdir build \ - # && cd build \ - # && cmake ../cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ - # && make \ - # && make install ) (cd zstd-${ZSTD_VERSION} \ + && cd build \ + && mkdir build \ + && cd build \ + && cmake ../cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ && make \ - && make install prefix=$BUILD_PREFIX ) + && make install ) + #(cd zstd-${ZSTD_VERSION} \ + # && make \ + # && make install prefix=$BUILD_PREFIX ) touch zstd-stamp }