From 7cc3f12803a04817f25a4f86750dbd3a2694b61c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 7 Oct 2022 16:08:10 -0700 Subject: [PATCH 1/2] Build wheel files --- .../workflows/ci-build-release-wheels.yaml | 145 ++++++++ .github/workflows/ci-pr-validation.yaml | 94 ++++- .gitignore | 4 +- CMakeLists.txt | 106 ++---- build-mac-wheels.sh | 334 ------------------ build-support/copy-deps-versionfile.sh | 29 ++ build-support/dep-version.py | 24 ++ build-support/install-cpp-client.sh | 47 ++- dependencies.yaml | 27 ++ pkg/build-wheel-inside-docker.sh | 42 +++ pkg/mac/build-dependencies.sh | 237 +++++++++++++ pkg/mac/build-mac-wheels.sh | 80 +++++ pkg/mac/build-pulsar-cpp.sh | 64 ++++ pkg/mac/common.sh | 25 ++ pkg/manylinux2014/Dockerfile | 56 +++ pkg/manylinux_musl/Dockerfile | 53 +++ pkg/test-wheel.sh | 28 ++ pulsar-client-cpp-version.txt | 2 +- pulsar/__init__.py | 2 +- 19 files changed, 984 insertions(+), 415 deletions(-) create mode 100644 .github/workflows/ci-build-release-wheels.yaml delete mode 100755 build-mac-wheels.sh create mode 100755 build-support/copy-deps-versionfile.sh create mode 100755 build-support/dep-version.py create mode 100644 dependencies.yaml create mode 100755 pkg/build-wheel-inside-docker.sh create mode 100755 pkg/mac/build-dependencies.sh create mode 100755 pkg/mac/build-mac-wheels.sh create mode 100755 pkg/mac/build-pulsar-cpp.sh create mode 100644 pkg/mac/common.sh create mode 100644 pkg/manylinux2014/Dockerfile create mode 100644 pkg/manylinux_musl/Dockerfile create mode 100755 pkg/test-wheel.sh diff --git a/.github/workflows/ci-build-release-wheels.yaml b/.github/workflows/ci-build-release-wheels.yaml new file mode 100644 index 0000000..faeb3c5 --- /dev/null +++ b/.github/workflows/ci-build-release-wheels.yaml @@ -0,0 +1,145 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Build release wheels +on: + push: + tags: + - '*' + branches: + - '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + linux-wheel: + name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} + runs-on: ubuntu-22.04 + timeout-minutes: 300 + + strategy: + fail-fast: false + matrix: + image: + - {name: 'manylinux2014', py_suffix: ''} + - {name: 'manylinux_musl', py_suffix: '-alpine'} + python: + - {version: '3.7', spec: 'cp37-cp37m'} + - {version: '3.8', spec: 'cp38-cp38'} + - {version: '3.9', spec: 'cp39-cp39'} + - {version: '3.10', spec: 'cp310-cp310'} + cpu: + - {arch: 'x86_64', platform: 'x86_64'} + - {arch: 'aarch64', platform: 'arm64'} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build Manylinux Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/${{matrix.image.name}} + load: true + tags: build:latest + platforms: linux/${{matrix.cpu.arch}} + build-args: | + PLATFORM=${{matrix.cpu.platform}} + ARCH=${{matrix.cpu.arch}} + PYTHON_VERSION=${{matrix.python.version}} + PYTHON_SPEC=${{matrix.python.spec}} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build wheel file + run: | + docker run -i -v $PWD:/pulsar-client-python build:latest \ + /pulsar-client-python/pkg/build-wheel-inside-docker.sh + + - name: Test wheel file + run: | + docker run -i -v $PWD:/pulsar-client-python \ + --platform linux/${{matrix.cpu.arch}} \ + python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ + /pulsar-client-python/pkg/test-wheel.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: wheel-${{matrix.image.name}}-py${{matrix.python.version}}-${{matrix.cpu.platform}} + path: wheelhouse/*.whl + + mac-wheels: + name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} + runs-on: macos-latest + timeout-minutes: 300 + + strategy: + fail-fast: false + matrix: + py: + - {version: '3.7', version_long: '3.7.14'} + - {version: '3.8', version_long: '3.8.13'} + - {version: '3.9', version_long: '3.9.14'} + - {version: '3.10', version_long: '3.10.7'} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Dependencies cache + uses: actions/cache@v3 + id: cache-deps + with: + path: .pulsar-mac-build/deps/install + key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} + + - name: Build dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} + + - name: Pulsar C++ cache + uses: actions/cache@v3 + id: cache-cpp + with: + path: .pulsar-mac-build/cpp/install + key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} + + - name: Build Pulsar C++ client + if: steps.cache-cpp.outputs.cache-hit != 'true' + run: pkg/mac/build-pulsar-cpp.sh + + - name: Build and test Mac wheels + run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: wheel-mac-py${{matrix.py.version}} + path: dist/*.whl + diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index fb2a888..0c73b99 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -21,6 +21,8 @@ name: PR validation on: pull_request: branches: ['main'] + push: + branches: ['*'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -30,7 +32,7 @@ jobs: unit-tests: name: Run unit tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 120 steps: @@ -64,3 +66,93 @@ jobs: - name: Stop Pulsar service run: ./build-support/pulsar-test-service-stop.sh + + linux-wheel: + name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} + runs-on: ubuntu-22.04 + timeout-minutes: 300 + + strategy: + fail-fast: false + matrix: + image: + - {name: 'manylinux2014', py_suffix: ''} + - {name: 'manylinux_musl', py_suffix: '-alpine'} + python: + - {version: '3.10', spec: 'cp310-cp310'} + cpu: + - {arch: 'x86_64', platform: 'x86_64'} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build Manylinux Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/${{matrix.image.name}} + load: true + tags: build:latest + platforms: linux/${{matrix.cpu.arch}} + build-args: | + PLATFORM=${{matrix.cpu.platform}} + ARCH=${{matrix.cpu.arch}} + PYTHON_VERSION=${{matrix.python.version}} + PYTHON_SPEC=${{matrix.python.spec}} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build wheel file + run: | + docker run -i -v $PWD:/pulsar-client-python build:latest \ + /pulsar-client-python/pkg/build-wheel-inside-docker.sh + + - name: Test wheel file + run: | + docker run -i -v $PWD:/pulsar-client-python python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ + /pulsar-client-python/pkg/test-wheel.sh + + mac-wheels: + name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} + runs-on: macos-latest + timeout-minutes: 300 + + strategy: + matrix: + py: + - {version: '3.10', version_long: '3.10.7'} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Dependencies cache + uses: actions/cache@v3 + id: cache-deps + with: + path: .pulsar-mac-build/deps/install + key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }} + + - name: Build dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}} + + - name: Pulsar C++ cache + uses: actions/cache@v3 + id: cache-cpp + with: + path: .pulsar-mac-build/cpp/install + key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }} + + - name: Build Pulsar C++ client + if: steps.cache-cpp.outputs.cache-hit != 'true' + run: pkg/mac/build-pulsar-cpp.sh + + - name: Build and test Mac wheels + run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} diff --git a/.gitignore b/.gitignore index fdff2ec..ee9119f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ cmake_install.cmake __pycache__ .build .pulsar-mac-wheels-cache -.DS_Store \ No newline at end of file +.DS_Store +wheelhouse +.pulsar-mac-build \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 087678c..10a70ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # project (pulsar-client-python) -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.18) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") option(LINK_STATIC "Link against static libraries" OFF) @@ -32,42 +32,22 @@ MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) if (LINK_STATIC) find_library(PULSAR_LIBRARY NAMES libpulsar.a) else() - find_library(PULSAR_LIBRARY NAMES libpulsar.so) + find_library(PULSAR_LIBRARY NAMES libpulsar.so libpulsar.dylib) endif() message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") find_path(PULSAR_INCLUDE pulsar/Client.h) message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}") -SET(Boost_NO_BOOST_CMAKE ON) -SET(Boost_USE_STATIC_LIBS ${LINK_STATIC}) - SET(CMAKE_CXX_STANDARD 11) -find_package(Boost) - -find_package (Python3 COMPONENTS Development) +find_package (Python3 REQUIRED COMPONENTS Development.Module) MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS}) -string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${Python3_VERSION}) - -set(BOOST_PYTHON_NAME_POSTFIX ${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}) -# For python3 the lib name is boost_python3 -set(BOOST_PYTHON_NAME_LIST python${BOOST_PYTHON_NAME_POSTFIX};python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt) - -foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST) - find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME}) - if (${Boost_FOUND}) - set(BOOST_PYTHON_NAME_FOUND ${BOOST_PYTHON_NAME}) - break() - endif() -endforeach() - -if (NOT ${Boost_FOUND}) - MESSAGE(FATAL_ERROR "Could not find Boost Python library") -endif () - -MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND}) +SET(Boost_USE_STATIC_LIBS ${LINK_STATIC}) +find_package(Boost REQUIRED COMPONENTS python3) +MESSAGE(STATUS "Boost Python3: " ${Boost_PYTHON3_LIBRARY}) +MESSAGE(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}") ######################################################################################################################## @@ -99,55 +79,43 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -Qunused-arguments -undefined dynamic_lookup") endif() -# Newer boost versions don't use the -mt suffix -if (NOT DEFINED ${Boost_PYTHON37-MT_LIBRARY}) - set(Boost_PYTHON37-MT_LIBRARY ${Boost_PYTHON37_LIBRARY}) -endif() +# Try all possible boost-python variable namings +set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY} + ${Boost_PYTHON3_LIBRARY}) -if (NOT DEFINED ${Boost_PYTHON38-MT_LIBRARY}) - set(Boost_PYTHON38-MT_LIBRARY ${Boost_PYTHON38_LIBRARY}) -endif() +message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") -if (NOT DEFINED ${Boost_PYTHON39-MT_LIBRARY}) - set(Boost_PYTHON39-MT_LIBRARY ${Boost_PYTHON39_LIBRARY}) -endif() +if (LINK_STATIC) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") -if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY}) - set(Boost_PYTHON310-MT_LIBRARY ${Boost_PYTHON310_LIBRARY}) -endif() + # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a + # with all the deps included. + find_package(OpenSSL REQUIRED) + message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) -# Try all possible boost-python variable namings -set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY} - ${Boost_PYTHON_LIBRARY} - ${Boost_PYTHON3_LIBRARY} - ${Boost_PYTHON37-MT_LIBRARY} - ${Boost_PYTHON38_LIBRARY} - ${Boost_PYTHON39_LIBRARY} - ${Boost_PYTHON310_LIBRARY} - ) - -if (APPLE) - if (Boost_PYTHON37-MT_LIBRARY_RELEASE) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON37-MT_LIBRARY_RELEASE}) - endif () - if (Boost_PYTHON38-MT_LIBRARY_RELEASE) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON38-MT_LIBRARY_RELEASE}) - endif () - if (Boost_PYTHON39-MT_LIBRARY_RELEASE) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE}) - endif () - if (Boost_PYTHON310-MT_LIBRARY_RELEASE) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON310-MT_LIBRARY_RELEASE}) - endif () -endif() + find_package(Protobuf REQUIRED) + message("Protobuf_LIBRARIES: " ${Protobuf_LIBRARIES}) -if (NOT PYTHON_WRAPPER_LIBS) - MESSAGE(FATAL_ERROR "Could not find Boost Python library") -endif () + find_package(curl REQUIRED) + message("CURL_LIBRARIES: " ${CURL_LIBRARIES}) -message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") + find_package(zlib) + message("ZLIB_LIBRARIES: " ${ZLIB_LIBRARIES}) + + find_library(LIB_ZSTD NAMES libzstd.a) + message(STATUS "ZStd: ${LIB_ZSTD}") + find_library(LIB_SNAPPY NAMES libsnappy.a) + message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}") + + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} + ${OPENSSL_LIBRARIES} + ${Protobuf_LIBRARIES} + ${ZLIB_LIBRARIES} + ${LIB_ZSTD} + ${LIB_SNAPPY} + ${CURL_LIBRARIES} + ) -if (LINK_STATIC) if (APPLE) set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS}) diff --git a/build-mac-wheels.sh b/build-mac-wheels.sh deleted file mode 100755 index 38072f6..0000000 --- a/build-mac-wheels.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -PYTHON_VERSIONS=( - '3.7 3.7.13' - '3.8 3.8.13' - '3.9 3.9.10' - '3.10 3.10.2' -) - -export MACOSX_DEPLOYMENT_TARGET=10.15 -MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} - -ZLIB_VERSION=1.2.12 -OPENSSL_VERSION=1_1_1n -BOOST_VERSION=1.78.0 -PROTOBUF_VERSION=3.20.0 -ZSTD_VERSION=1.5.2 -SNAPPY_VERSION=1.1.3 -CURL_VERSION=7.61.0 - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -PULSAR_VERSION=$(cat version.txt | grep pulsar-client-cpp | awk '{print $2}') - -# Compile and cache dependencies -CACHE_DIR=~/.pulsar-mac-wheels-cache -mkdir -p $CACHE_DIR - -cd $CACHE_DIR - -PREFIX=$CACHE_DIR/install - -############################################################################### -for line in "${PYTHON_VERSIONS[@]}"; do - read -r -a PY <<< "$line" - PYTHON_VERSION=${PY[0]} - PYTHON_VERSION_LONG=${PY[1]} - - if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then - echo "Building Python $PYTHON_VERSION_LONG" - curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz - tar xfz Python-${PYTHON_VERSION_LONG}.tgz - - PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION - pushd Python-${PYTHON_VERSION_LONG} - if [ $PYTHON_VERSION = '3.7' ]; then - UNIVERSAL_ARCHS='intel-64' - PY_CFLAGS=" -arch x86_64" - else - UNIVERSAL_ARCHS='universal2' - fi - - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ - LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ - ./configure --prefix=$PY_PREFIX --enable-shared --enable-universalsdk --with-universal-archs=${UNIVERSAL_ARCHS} - make -j16 - make install - - curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl - $PY_PREFIX/bin/pip3 install wheel-*.whl - - touch .done - popd - else - echo "Using cached Python $PYTHON_VERSION_LONG" - fi -done - - -############################################################################### -if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then - echo "Building ZLib" - curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz - tar xvfz zlib-$ZLIB_VERSION.tar.gz - pushd zlib-$ZLIB_VERSION - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached ZLib" -fi - -############################################################################### -if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION}.done ]; then - echo "Building OpenSSL" - curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION}.tar.gz - # -arch arm64 -arch x86_64 - tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz - mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-arm64 - pushd openssl-OpenSSL_${OPENSSL_VERSION}-arm64 - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared darwin64-arm64-cc - make -j8 - make install - popd - - tar xvfz OpenSSL_${OPENSSL_VERSION}.tar.gz - mv openssl-OpenSSL_${OPENSSL_VERSION} openssl-OpenSSL_${OPENSSL_VERSION}-x86_64 - pushd openssl-OpenSSL_${OPENSSL_VERSION}-x86_64 - CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./Configure --prefix=$PREFIX no-shared darwin64-x86_64-cc - make -j8 - make install - popd - - # Create universal binaries - lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libssl.a \ - -output $PREFIX/lib/libssl.a - lipo -create openssl-OpenSSL_${OPENSSL_VERSION}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION}-x86_64/libcrypto.a \ - -output $PREFIX/lib/libcrypto.a - - touch openssl-OpenSSL_${OPENSSL_VERSION}.done -else - echo "Using cached OpenSSL" -fi - -############################################################################### -BOOST_VERSION_=${BOOST_VERSION//./_} -for line in "${PYTHON_VERSIONS[@]}"; do - read -r -a PY <<< "$line" - PYTHON_VERSION=${PY[0]} - PYTHON_VERSION_LONG=${PY[1]} - - DIR=boost-src-${BOOST_VERSION}-python-${PYTHON_VERSION} - if [ ! -f $DIR/.done ]; then - echo "Building Boost for Py $PYTHON_VERSION" - curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz - tar xfz boost_${BOOST_VERSION_}.tar.gz - mv boost_${BOOST_VERSION_} $DIR - - PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION - PY_INCLUDE_DIR=${PY_PREFIX}/include/python${PYTHON_VERSION} - if [ $PYTHON_VERSION = '3.7' ]; then - PY_INCLUDE_DIR=${PY_INCLUDE_DIR}m - fi - - pushd $DIR - cat < user-config.jam - using python : $PYTHON_VERSION - : python3 - : ${PY_INCLUDE_DIR} - : ${PY_PREFIX}/lib - ; -EOF - ./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PY_PREFIX \ - --prefix=$CACHE_DIR/boost-py-$PYTHON_VERSION - ./b2 address-model=64 cxxflags="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - link=static threading=multi \ - --user-config=./user-config.jam \ - variant=release python=${PYTHON_VERSION} \ - -j16 \ - install - touch .done - popd - else - echo "Using cached Boost for Py $PYTHON_VERSION" - fi - -done - - - -############################################################################### -if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then - echo "Building Protobuf" - curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - tar xvfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz - pushd protobuf-${PROTOBUF_VERSION} - CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached Protobuf" -fi - -############################################################################### -if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then - echo "Building ZStd" - curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz - tar xvfz zstd-${ZSTD_VERSION}.tar.gz - pushd zstd-${ZSTD_VERSION} - CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ - make -j16 install - touch .done - popd -else - echo "Using cached ZStd" -fi - -############################################################################### -if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then - echo "Building Snappy" - curl -O -L https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz - tar xvfz snappy-${SNAPPY_VERSION}.tar.gz - pushd snappy-${SNAPPY_VERSION} - CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --prefix=$PREFIX - make -j16 - make install - touch .done - popd -else - echo "Using cached Snappy" -fi - -############################################################################### -if [ ! -f curl-${CURL_VERSION}/.done ]; then - echo "Building LibCurl" - CURL_VERSION_=${CURL_VERSION//./_} - curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz - tar xfz curl-${CURL_VERSION}.tar.gz - pushd curl-${CURL_VERSION} - CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --with-ssl=$PREFIX \ - --without-nghttp2 --without-libidn2 --disable-ldap \ - --prefix=$PREFIX - make -j16 install - touch .done - popd -else - echo "Using cached LibCurl" -fi - -############################################################################### -if [ ! -f apache-pulsar-${PULSAR_VERSION}-src/.done ]; then - echo "Building Pulsar C++ client - ${PULSAR_VERSION}" - curl -O -L https://archive.apache.org/dist/pulsar/pulsar-${PULSAR_VERSION}/apache-pulsar-${PULSAR_VERSION}-src.tar.gz - rm -rf apache-pulsar-${PULSAR_VERSION}-src/pulsar-client-cpp - tar xfz apache-pulsar-${PULSAR_VERSION}-src.tar.gz - pushd apache-pulsar-${PULSAR_VERSION}-src - pushd pulsar-client-cpp - ARCHS='arm64;x86_64' - - chmod +x build-support/merge_archives.sh - set -x - cmake . \ - -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ - -DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \ - -DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \ - -DLINK_STATIC=OFF \ - -DBUILD_TESTS=OFF \ - -DBUILD_PYTHON_WRAPPER=OFF \ - -DBUILD_WIRESHARK=OFF \ - -DBUILD_DYNAMIC_LIB=OFF \ - -DBUILD_STATIC_LIB=ON \ - -DPROTOC_PATH=$PREFIX/bin/protoc - - make -j16 install - popd - touch .done - popd -else - echo "Using cached Pulsar C++ client" -fi - -############################################################################### -############################################################################### -############################################################################### -############################################################################### - -for line in "${PYTHON_VERSIONS[@]}"; do - read -r -a PY <<< "$line" - PYTHON_VERSION=${PY[0]} - PYTHON_VERSION_LONG=${PY[1]} - echo '----------------------------------------------------------------------------' - echo '----------------------------------------------------------------------------' - echo '----------------------------------------------------------------------------' - echo "Build wheel for Python $PYTHON_VERSION" - - cd "${ROOT_DIR}" - - find . -name CMakeCache.txt | xargs -r rm - find . -name CMakeFiles | xargs -r rm -rf - - PY_PREFIX=$CACHE_DIR/py-$PYTHON_VERSION - PY_EXE=$PY_PREFIX/bin/python3 - - PY_INCLUDE_DIR=${PY_PREFIX}/include/python${PYTHON_VERSION} - ARCHS='arm64;x86_64' - if [ $PYTHON_VERSION = '3.7' ]; then - PY_INCLUDE_DIR=${PY_INCLUDE_DIR}m - ARCHS='x86_64' - fi - - set -x - cmake . \ - -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ - -DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \ - -DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \ - -DPython3_INCLUDE_DIR=$PY_INCLUDE_DIR \ - -DPython3_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \ - -DPULSAR_INCLUDE=${PREFIX}/include - - make clean - make -j16 - - $PY_EXE setup.py bdist_wheel -done diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh new file mode 100755 index 0000000..0a36d10 --- /dev/null +++ b/build-support/copy-deps-versionfile.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +ROOT_DIR=$(git rev-parse --show-toplevel) + +for dir in manylinux2014 manylinux_musl; do + mkdir -p pkg/$dir/.build + cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build + cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build +done diff --git a/build-support/dep-version.py b/build-support/dep-version.py new file mode 100755 index 0000000..31200cd --- /dev/null +++ b/build-support/dep-version.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import yaml, sys + +deps = yaml.safe_load(open("dependencies.yaml")) +print(deps[sys.argv[1]]) diff --git a/build-support/install-cpp-client.sh b/build-support/install-cpp-client.sh index 2efe8f2..ac493c1 100755 --- a/build-support/install-cpp-client.sh +++ b/build-support/install-cpp-client.sh @@ -20,17 +20,48 @@ set -e -x -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR=$(dirname $(dirname $0)) +CPP_CLIENT_VERSION=$(cat $ROOT_DIR/pulsar-client-cpp-version.txt | xargs) -cd $ROOT_DIR +if [ $USER != "root" ]; then + SUDO="sudo" +fi -CPP_CLIENT_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) +# Get the flavor of Linux +export $(cat /etc/*-release | grep "^ID=") + +cd /tmp # Fetch the client binaries ## TODO: Fetch from official release once it's available -pushd /tmp - curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client.deb - curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client-dev.deb -popd +BASE_URL=https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${CPP_CLIENT_VERSION}-candidate-1 + +UNAME_ARCH=$(uname -m) +if [ $UNAME_ARCH == 'aarch64' ]; then + PLATFORM=arm64 +else + PLATFORM=x86_64 +fi + +if [ $ID == 'ubuntu' ]; then + curl -L -O ${BASE_URL}/deb-${PLATFORM}/apache-pulsar-client.deb + curl -L -O ${BASE_URL}/deb-${PLATFORM}/apache-pulsar-client-dev.deb + $SUDO apt install -y /tmp/*.deb + +elif [ $ID == 'alpine' ]; then + curl -L -O ${BASE_URL}/apk-${PLATFORM}/apache-pulsar-client-${CPP_CLIENT_VERSION}-r0.apk + curl -L -O ${BASE_URL}/apk-${PLATFORM}/apache-pulsar-client-dev-${CPP_CLIENT_VERSION}-r0.apk + $SUDO apk add --allow-untrusted /tmp/*.apk + +elif [ $ID == '"centos"' ]; then + curl -L -O ${BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm + curl -L -O ${BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-devel-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm + $SUDO rpm -i /tmp/*.rpm + +else + echo "Unknown Linux distribution: '$ID'" + exit 1 +fi + + -sudo apt install /tmp/apache-pulsar-client.deb /tmp/apache-pulsar-client-dev.deb diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 0000000..91aaf57 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +boost: 1.80.0 +cmake: 3.24.2 +protobuf: 3.20.0 +zlib: 1.2.12 +zstd: 1.5.2 +snappy: 1.1.9 +openssl: 1.1.1q +curl: 7.85.0 diff --git a/pkg/build-wheel-inside-docker.sh b/pkg/build-wheel-inside-docker.sh new file mode 100755 index 0000000..a477b35 --- /dev/null +++ b/pkg/build-wheel-inside-docker.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +cd /pulsar-client-python + +build-support/install-cpp-client.sh + +rm -f CMakeCache.txt CMakeFiles + +cmake . \ + -DCMAKE_BUILD_TYPE=Release + +make -j4 + +./setup.py bdist_wheel + +# Audit wheel is required to convert a wheel that is tagged as generic +# 'linux' into a 'multilinux' wheel. +# Only wheel files tagged as multilinux can be uploaded to PyPI +# Audit wheel will make sure no external dependencies are needed for +# the shared library and that only symbols supported by most linux +# distributions are used. +auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_*.whl diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh new file mode 100755 index 0000000..35f8685 --- /dev/null +++ b/pkg/mac/build-dependencies.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd "${ROOT_DIR}" + +PYTHON_VERSION=$1 +PYTHON_VERSION_LONG=$2 + +source pkg/mac/common.sh + +pip3 install pyyaml + +dep=$ROOT_DIR/build-support/dep-version.py +ZLIB_VERSION=$($dep zlib) +OPENSSL_VERSION=$($dep openssl) +BOOST_VERSION=$($dep boost) +PROTOBUF_VERSION=$($dep protobuf) +ZSTD_VERSION=$($dep zstd) +SNAPPY_VERSION=$($dep snappy) +CURL_VERSION=$($dep curl) + +# Compile and cache dependencies +CACHE_DIR=${CACHE_DIR_DEPS} +mkdir -p $CACHE_DIR +cd $CACHE_DIR + +PREFIX=$CACHE_DIR/install + + +############################################################################### +if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then + echo "Building ZLib" + curl -O -L https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz + tar xfz zlib-$ZLIB_VERSION.tar.gz + pushd zlib-$ZLIB_VERSION + CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached ZLib" +fi + +############################################################################### +if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then + echo "Building Python $PYTHON_VERSION_LONG" + curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz + tar xfz Python-${PYTHON_VERSION_LONG}.tgz + + pushd Python-${PYTHON_VERSION_LONG} + if [ $PYTHON_VERSION = '3.7' ]; then + UNIVERSAL_ARCHS='intel-64' + PY_CFLAGS=" -arch x86_64" + else + UNIVERSAL_ARCHS='universal2' + fi + + CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ + LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ + ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=${UNIVERSAL_ARCHS} + make -j16 + make install + + curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl + $PREFIX/bin/pip3 install wheel-*.whl + + touch .done + popd +else + echo "Using cached Python $PYTHON_VERSION_LONG" +fi + +############################################################################### +OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then + echo "Building OpenSSL" + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + # -arch arm64 -arch x86_64 + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc + make -j8 + make install_sw + popd + + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz + mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc + make -j8 + make install_sw + popd + + # Create universal binaries + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ + -output $PREFIX/lib/libssl.a + lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ + -output $PREFIX/lib/libcrypto.a + + touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done +else + echo "Using cached OpenSSL" +fi + +############################################################################### +BOOST_VERSION_=${BOOST_VERSION//./_} +DIR=boost-src-${BOOST_VERSION} +if [ ! -f $DIR/.done ]; then + echo "Building Boost for Py $PYTHON_VERSION" + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz + tar xfz boost_${BOOST_VERSION_}.tar.gz + mv boost_${BOOST_VERSION_} $DIR + + PY_INCLUDE_DIR=${PREFIX}/include/python${PYTHON_VERSION} + if [ $PYTHON_VERSION = '3.7' ]; then + PY_INCLUDE_DIR=${PY_INCLUDE_DIR}m + fi + + pushd $DIR + cat < user-config.jam + using python : $PYTHON_VERSION + : python3 + : ${PY_INCLUDE_DIR} + : ${PREFIX}/lib + ; +EOF + ./bootstrap.sh --with-libraries=python --with-python=python3 --with-python-root=$PREFIX \ + --prefix=${PREFIX} + ./b2 -d0 address-model=64 cxxflags="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + link=static threading=multi \ + --user-config=./user-config.jam \ + variant=release python=${PYTHON_VERSION} \ + -j16 \ + install + touch .done + popd +else + echo "Using cached Boost for Py $PYTHON_VERSION" +fi + + + +############################################################################### +if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then + echo "Building Protobuf" + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + pushd protobuf-${PROTOBUF_VERSION} + CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached Protobuf" +fi + +############################################################################### +if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then + echo "Building ZStd" + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz + tar xfz zstd-${ZSTD_VERSION}.tar.gz + pushd zstd-${ZSTD_VERSION} + CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ + make -j16 -C lib install-static install-includes + touch .done + popd +else + echo "Using cached ZStd" +fi + +############################################################################### +if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then + echo "Building Snappy" + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz + tar xfz ${SNAPPY_VERSION}.tar.gz + pushd snappy-${SNAPPY_VERSION} + CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF + make -j16 + make install + touch .done + popd +else + echo "Using cached Snappy" +fi + +############################################################################### +if [ ! -f curl-${CURL_VERSION}/.done ]; then + echo "Building LibCurl" + CURL_VERSION_=${CURL_VERSION//./_} + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz + tar xfz curl-${CURL_VERSION}.tar.gz + pushd curl-${CURL_VERSION} + CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --with-ssl=$PREFIX \ + --without-nghttp2 \ + --without-libidn2 \ + --disable-ldap \ + --without-brotli \ + --without-secure-transport \ + --disable-ipv6 \ + --prefix=$PREFIX + make -j16 install + touch .done + popd +else + echo "Using cached LibCurl" +fi diff --git a/pkg/mac/build-mac-wheels.sh b/pkg/mac/build-mac-wheels.sh new file mode 100755 index 0000000..edb23f5 --- /dev/null +++ b/pkg/mac/build-mac-wheels.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd "${ROOT_DIR}" + +source pkg/mac/common.sh + +PYTHON_CLIENT_VERSION=$(cat version.txt | xargs) + +PYTHON_VERSION=$1 + +############################################################################### +############################################################################### +############################################################################### +############################################################################### +PREFIX=${CACHE_DIR_DEPS}/install + +echo '----------------------------------------------------------------------------' +echo '----------------------------------------------------------------------------' +echo '----------------------------------------------------------------------------' +echo "Build wheel for Python $PYTHON_VERSION" + +cd "${ROOT_DIR}" + +rm -f CMakeCache.txt + +PY_EXE=$PREFIX/bin/python3 +PIP_EXE=$PREFIX/bin/pip3 + +ARCHS='arm64;x86_64' +PIP_TAG='universal2' +if [ $PYTHON_VERSION = '3.7' ]; then + ARCHS='x86_64' + PIP_TAG=$ARCHS +fi + +cmake . \ + -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DCMAKE_CXX_FLAGS=-I$PREFIX/include \ + -DLINK_STATIC=ON \ + -DPULSAR_LIBRARY=${CACHE_DIR_CPP_CLIENT}/install/lib/libpulsar.a \ + -DPULSAR_INCLUDE=${CACHE_DIR_CPP_CLIENT}/install/include \ + -DPython3_ROOT_DIR=$PREFIX \ + -DBOOST_ROOT=${PREFIX} + +make clean +make -j16 VERBOSE=1 + +$PY_EXE setup.py bdist_wheel + +PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') + +cd /tmp +$PIP_EXE install --no-dependencies --force-reinstall \ + $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx_10_15_${PIP_TAG}.whl +$PY_EXE -c 'import pulsar' + diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh new file mode 100755 index 0000000..c1eb514 --- /dev/null +++ b/pkg/mac/build-pulsar-cpp.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd "${ROOT_DIR}" + +source pkg/mac/common.sh + +PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) + +# Compile and cache dependencies +mkdir -p $CACHE_DIR_CPP_CLIENT +cd $CACHE_DIR_CPP_CLIENT + +PREFIX=$CACHE_DIR_CPP_CLIENT/install + +DEPS_PREFIX=${CACHE_DIR_DEPS}/install + +############################################################################### + +## TODO: Fetch from official release +curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}-candidate-1/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz + +if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then + pushd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} + ARCHS='arm64;x86_64' + + cmake . \ + -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=${DEPS_PREFIX} \ + -DCMAKE_CXX_FLAGS=-I${DEPS_PREFIX}/include \ + -DLINK_STATIC=OFF \ + -DBUILD_TESTS=OFF \ + -DBUILD_WIRESHARK=OFF \ + -DBUILD_DYNAMIC_LIB=OFF \ + -DBUILD_STATIC_LIB=ON \ + -DPROTOC_PATH=${DEPS_PREFIX}/bin/protoc + + make -j16 install + popd +fi diff --git a/pkg/mac/common.sh b/pkg/mac/common.sh new file mode 100644 index 0000000..fa2ed35 --- /dev/null +++ b/pkg/mac/common.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +export MACOSX_DEPLOYMENT_TARGET=10.15 +export MACOSX_DEPLOYMENT_TARGET_MAJOR=${MACOSX_DEPLOYMENT_TARGET%%.*} + +CACHE_DIR_DEPS=$PWD/.pulsar-mac-build/deps +CACHE_DIR_CPP_CLIENT=$PWD/.pulsar-mac-build/cpp diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile new file mode 100644 index 0000000..7355585 --- /dev/null +++ b/pkg/manylinux2014/Dockerfile @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ARG ARCH +FROM quay.io/pypa/manylinux2014_${ARCH} + +ARG PYTHON_VERSION +ARG PYTHON_SPEC +ARG PLATFORM +ARG ARCH + +ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV PYTHON_SPEC=${PYTHON_SPEC} + +ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" + +ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include +ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} + +RUN pip3 install pyyaml + +ADD .build/dependencies.yaml / +ADD .build/dep-version.py /usr/local/bin + +# Download and compile boost +RUN BOOST_VERSION=$(dep-version.py boost) && \ + BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + ./bootstrap.sh --with-libraries=python && \ + ./b2 -d0 address-model=64 cxxflags=-fPIC link=shared threading=multi variant=release install && \ + rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + +RUN CMAKE_VERSION=$(dep-version.py cmake) && \ + curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz && \ + tar xfz cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz && \ + cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \ + cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \ + rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile new file mode 100644 index 0000000..f402bec --- /dev/null +++ b/pkg/manylinux_musl/Dockerfile @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ARG ARCH +FROM quay.io/pypa/musllinux_1_1_${ARCH} + +ARG PYTHON_VERSION +ARG PYTHON_SPEC + +ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV PYTHON_SPEC=${PYTHON_SPEC} + +ARG ARCH +ENV ARCH=${ARCH} + +ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" +ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include +ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION} + +RUN pip install pyyaml + +RUN apk add cmake + +ADD .build/dependencies.yaml / +ADD .build/dep-version.py /usr/local/bin + +# Download and compile boost +RUN BOOST_VERSION=$(dep-version.py boost) && \ + BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + ./bootstrap.sh --with-libraries=python && \ + ./b2 -d0 address-model=64 cxxflags=-fPIC link=shared threading=multi variant=release install && \ + rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + + diff --git a/pkg/test-wheel.sh b/pkg/test-wheel.sh new file mode 100755 index 0000000..d19b280 --- /dev/null +++ b/pkg/test-wheel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +cd / + +pip3 install /pulsar-client-python/wheelhouse/pulsar_client-*.whl + +# Load the wheel to ensure there are no linking problems +python3 -c 'import pulsar' diff --git a/pulsar-client-cpp-version.txt b/pulsar-client-cpp-version.txt index 6390d75..4a36342 100644 --- a/pulsar-client-cpp-version.txt +++ b/pulsar-client-cpp-version.txt @@ -1 +1 @@ -3.0.0-pre-1 +3.0.0 diff --git a/pulsar/__init__.py b/pulsar/__init__.py index 942ec8f..ef17844 100644 --- a/pulsar/__init__.py +++ b/pulsar/__init__.py @@ -116,7 +116,6 @@ def send_callback(res, msg_id): import re _retype = type(re.compile('x')) -import certifi from datetime import timedelta @@ -469,6 +468,7 @@ def __init__(self, service_url, if tls_trust_certs_file_path: conf.tls_trust_certs_file_path(tls_trust_certs_file_path) else: + import certifi conf.tls_trust_certs_file_path(certifi.where()) conf.tls_allow_insecure_connection(tls_allow_insecure_connection) conf.tls_validate_hostname(tls_validate_hostname) From 3ee7b1c5f0ac98b8105c5ccd82457b097f61ce7c Mon Sep 17 00:00:00 2001 From: Demogorgon314 Date: Wed, 12 Oct 2022 16:14:49 +0800 Subject: [PATCH 2/2] Upload artifacts --- .github/workflows/ci-build-release-wheels.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-build-release-wheels.yaml b/.github/workflows/ci-build-release-wheels.yaml index faeb3c5..ddc50be 100644 --- a/.github/workflows/ci-build-release-wheels.yaml +++ b/.github/workflows/ci-build-release-wheels.yaml @@ -18,12 +18,17 @@ # name: Build release wheels +#on: +# push: +# tags: +# - '*' +# branches: +# - '*' on: + pull_request: + branches: ['main'] push: - tags: - - '*' - branches: - - '*' + branches: ['*'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -142,4 +147,3 @@ jobs: with: name: wheel-mac-py${{matrix.py.version}} path: dist/*.whl -