diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index e1b7434..6f3f894 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -67,6 +67,7 @@ jobs: linux-wheel: name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} + needs: unit-tests runs-on: ubuntu-22.04 timeout-minutes: 300 @@ -118,6 +119,7 @@ jobs: mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} + needs: unit-tests runs-on: macos-12 timeout-minutes: 300 @@ -155,11 +157,92 @@ jobs: - name: Build and test Mac wheels run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} + windows-wheels: + name: "Python ${{ matrix.python.version }} Wheel on ${{ matrix.windows.name }}" + needs: unit-tests + runs-on: ${{ matrix.windows.os }} + timeout-minutes: 120 + + env: + PULSAR_CPP_DIR: 'C:\\pulsar-cpp' + strategy: + fail-fast: false + matrix: + windows: + - name: 'Windows x64' + os: windows-2022 + arch: '-A x64' + triplet: 'x64-windows' + python: + - version: '3.7' + - version: '3.8' + - version: '3.9' + - version: '3.10' + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python.version }} + + - name: Prepare vcpkg.json + shell: bash + run: | + python --version + cp -f vcpkg-${{ matrix.python.version }}.json vcpkg.json + cat vcpkg.json + + - name: Download Pulsar C++ client on Windows + shell: bash + run: | + mkdir -p ${{ env.PULSAR_CPP_DIR }} + cd ${{ env.PULSAR_CPP_DIR }} + # TODO: switch to official releases + curl -O -L https://github.com/BewareMyPower/pulsar-client-cpp/releases/download/v3.1.0-rc-20221028/${{ matrix.windows.triplet }}-static.zip + unzip -q ${{ matrix.windows.triplet }}-static.zip + ls -l ${{ env.PULSAR_CPP_DIR }} + + - name: Cache Vcpkg + uses: actions/cache@v3 + id: cache-vcpkg + with: + path: build/vcpkg_installed + key: ${{ matrix.python.version }}-${{ hashFiles(format('vcpkg-{0}.json', matrix.python.version)) }} + + - name: Install dependencies and configure CMake + shell: bash + run: | + COMMIT_ID=$(grep baseline vcpkg.json | sed 's/[",]//g' | awk '{print $2}') + cd vcpkg + echo "git fetch origin $COMMIT_ID" + git fetch origin $COMMIT_ID + cd - + cmake -B build ${{ matrix.windows.arch }} \ + -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \ + -DUSE_VCPKG=ON \ + -DLINK_STATIC=ON + + - name: Build Python wheel + shell: bash + run: | + cmake --build build --config Release --target install + python -m pip install wheel + python setup.py bdist_wheel + python -m pip install ./dist/*.whl + cp ./build/Release/boost_python*.dll . + echo "The extra DLLs:" + ls -l *.dll + python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' + + # Job that will be required to complete and depends on all the other jobs check-completion: name: Check Completion runs-on: ubuntu-latest - needs: [unit-tests, linux-wheel, mac-wheels] + needs: [unit-tests, linux-wheel, mac-wheels, windows-wheels] steps: - run: true diff --git a/.gitignore b/.gitignore index ee9119f..72f931b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ __pycache__ .pulsar-mac-wheels-cache .DS_Store wheelhouse -.pulsar-mac-build \ No newline at end of file +.pulsar-mac-build +vcpkg_installed/ +*.pyd +*.lib diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a0a57f3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vcpkg"] + path = vcpkg + url = https://github.com/microsoft/vcpkg.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 10a70ec..a60e1c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,10 +17,15 @@ # under the License. # -project (pulsar-client-python) cmake_minimum_required(VERSION 3.18) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") +option(USE_VCPKG "Use Vcpkg to install dependencies" OFF) +if (USE_VCPKG) + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake" + CACHE STRING "Vcpkg toolchain file") +endif () +project (pulsar-client-python) option(LINK_STATIC "Link against static libraries" OFF) MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) @@ -29,10 +34,19 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) +if (MSVC) + add_compile_options(/wd4819) +endif () + if (LINK_STATIC) - find_library(PULSAR_LIBRARY NAMES libpulsar.a) + if (MSVC) + find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib) + else () + find_library(PULSAR_LIBRARY NAMES libpulsar.a) + endif () + add_definitions("-DPULSAR_STATIC") else() - find_library(PULSAR_LIBRARY NAMES libpulsar.so libpulsar.dylib) + find_library(PULSAR_LIBRARY NAMES pulsar libpulsar) endif() message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") @@ -44,10 +58,23 @@ SET(CMAKE_CXX_STANDARD 11) find_package (Python3 REQUIRED COMPONENTS Development.Module) MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS}) +find_package(Boost REQUIRED ${Boost_INCLUDE_DIRS}) +message(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}") + SET(Boost_USE_STATIC_LIBS ${LINK_STATIC}) -find_package(Boost REQUIRED COMPONENTS python3) -MESSAGE(STATUS "Boost Python3: " ${Boost_PYTHON3_LIBRARY}) -MESSAGE(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}") + +set(BOOST_PYTHON_NAME_LIST python3 python310 python39 python38 python37) +foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST) + find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME}) + if (${Boost_FOUND}) + set(BOOST_PYTHON_COMPONENT_FOUND ${BOOST_PYTHON_NAME}) + message(STATUS "Found Boost COMPONENTS " ${BOOST_PYTHON_COMPONENT_FOUND}) + break () + endif () +endforeach () +if (NOT BOOST_PYTHON_COMPONENT_FOUND) + message(FATAL_ERROR "Could not find Boost Python library") +endif () ######################################################################################################################## @@ -68,8 +95,11 @@ ADD_LIBRARY(_pulsar SHARED src/pulsar.cc src/utils.cc ) -SET(CMAKE_SHARED_LIBRARY_PREFIX ) -SET(CMAKE_SHARED_LIBRARY_SUFFIX .so) +if (MSVC) + set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd) +else () + set(CMAKE_SHARED_LIBRARY_SUFFIX .so) +endif () if (NOT APPLE AND NOT MSVC) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_PYTHON}") @@ -80,12 +110,17 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") endif() # Try all possible boost-python variable namings -set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY} - ${Boost_PYTHON3_LIBRARY}) +set(PYTHON_WRAPPER_LIBS + ${PULSAR_LIBRARY} + Boost::${BOOST_PYTHON_COMPONENT_FOUND} +) +if (MSVC) + set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) +endif () message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") -if (LINK_STATIC) +if (LINK_STATIC AND NOT MSVC) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a @@ -125,9 +160,14 @@ if (LINK_STATIC) endif() target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () +elseif (LINK_STATIC) # MSVC + set_property(TARGET _pulsar PROPERTY + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) else() target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () +install(TARGETS _pulsar DESTINATION ${CMAKE_SOURCE_DIR}) find_package(ClangTools) set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") diff --git a/README.md b/README.md index 0d403cd..b0857a0 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,39 @@ ## Install the Python wheel +### Windows (with Vcpkg) + +First, install the dependencies via [Vcpkg](https://github.com/microsoft/vcpkg). + +```PowerShell +vcpkg install --feature-flags=manifests --triplet x64-windows +``` + +> NOTE: For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/tree/master/triplets) for all available triplets. + +Then, build and install the Python wheel. + +```PowerShell +# Assuming the Pulsar C++ client has been installed under the `PULSAR_CPP` directory. +cmake -B build -DUSE_VCPKG=ON -DCMAKE_PREFIX_PATH="$env:PULSAR_CPP" -DLINK_STATIC=ON +cmake --build build --config Release +cmake --install build +py setup.py bdist_wheel +py -m pip install ./dist/pulsar_client-*.whl +``` + +Since the Python client links to Boost.Python dynamically, you have to copy the dll (e.g. `boost_python310-vc142-mt-x64-1_80.dll`) into the system path (the `PATH` environment variable). If the `-DLINK_STATIC=ON` option is not specified, you have to copy the `pulsar.dll` into the system path as well. + +### Linux or macOS + +Assuming the Pulsar C++ client and Boost.Python have been installed under the system path. + ```bash cmake -B build cmake --build build -j8 -cp build/_pulsar.so . +cmake --install build ./setup.py bdist_wheel pip3 install dist/pulsar_client-*.whl --force-reinstall -rm _pulsar.so ``` > **NOTE** @@ -45,6 +71,8 @@ rm _pulsar.so > 1. Here a separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` is under the project directory. > 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before. +## Running examples + You can run `python3 -c 'import pulsar'` to see whether the wheel has been installed successfully. If it failed, check whether dependencies (e.g. `libpulsar.so`) are in the system path. If not, make sure the dependencies are in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS). Then you can run examples as a simple end-to-end test. diff --git a/setup.py b/setup.py index f20446f..046b2bb 100755 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ from setuptools import setup from distutils.core import Extension from os import environ, path +import platform from distutils.command import build_ext @@ -58,7 +59,13 @@ def build_extension(self, ext): except OSError as e: if e.errno != 17: # already exists raise - shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name)) + if 'Windows' in platform.platform(): + shutil.copyfile('_pulsar.pyd', self.get_ext_fullpath(ext.name)) + else: + try: + shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name)) + except FileNotFoundError: + shutil.copyfile('lib_pulsar.so', self.get_ext_fullpath(ext.name)) # Core Client dependencies diff --git a/vcpkg b/vcpkg new file mode 160000 index 0000000..2537044 --- /dev/null +++ b/vcpkg @@ -0,0 +1 @@ +Subproject commit 253704407ae68efa37bf8f5b59b3e06dd40d3d3f diff --git a/vcpkg-3.10.json b/vcpkg-3.10.json new file mode 100644 index 0000000..fdb1128 --- /dev/null +++ b/vcpkg-3.10.json @@ -0,0 +1,12 @@ +{ + "name": "pulsar-python", + "version": "3.0.0", + "description": "Pulsar Python SDK (Python 3.10)", + "dependencies": [ + { + "name": "boost-python", + "version>=": "1.79.0" + } + ], + "builtin-baseline": "c266859544a3cdcfd952d218039c55a268863740" +} diff --git a/vcpkg-3.7.json b/vcpkg-3.7.json new file mode 100644 index 0000000..1b7b846 --- /dev/null +++ b/vcpkg-3.7.json @@ -0,0 +1,18 @@ +{ + "name": "pulsar-python", + "version": "3.0.0", + "description": "Pulsar Python SDK (Python 3.7)", + "dependencies": [ + { + "name": "boost-python", + "version>=": "1.76.0" + } + ], + "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379", + "overrides": [ + { + "name": "python3", + "version": "3.7.3" + } + ] +} diff --git a/vcpkg-3.8.json b/vcpkg-3.8.json new file mode 100644 index 0000000..b210e87 --- /dev/null +++ b/vcpkg-3.8.json @@ -0,0 +1,18 @@ +{ + "name": "pulsar-python", + "version": "3.0.0", + "description": "Pulsar Python SDK (Python 3.8)", + "dependencies": [ + { + "name": "boost-python", + "version>=": "1.76.0" + } + ], + "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379", + "overrides": [ + { + "name": "python3", + "version": "3.8.3" + } + ] +} diff --git a/vcpkg-3.9.json b/vcpkg-3.9.json new file mode 100644 index 0000000..250a1ee --- /dev/null +++ b/vcpkg-3.9.json @@ -0,0 +1,12 @@ +{ + "name": "pulsar-python", + "version": "3.0.0", + "description": "Pulsar Python SDK (Python 3.9)", + "dependencies": [ + { + "name": "boost-python", + "version>=": "1.76.0" + } + ], + "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379" +} diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..ef5c7c2 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "pulsar-python", + "version": "3.0.0", + "description": "Pulsar Python SDK", + "dependencies": ["boost-python"] +}