Skip to content

Commit af96031

Browse files
committed
Migrate from Boost.Python to PyBind11
Fixes apache#24 ### Motivation Remove the Boost.Python dependency so that on Windows there will be no DLL dependencies because PyBind11 is header only. ### Modifications Since PyBind11 can perform type conversions between C++ types (STL, function, etc.) and Python types (list, dict, lambda, etc.), some wrapper classes are replaced with the classes in the Pulsar C++ library. The only API changes are related to the `_pulsar` module, which should not be used directly. The authentication related classes were wrapper classes with constructors before, now they are created by the static `create` methods from Pulsar C++ API. Fix the CMakeLists.txt and the workflows to build Python wheels on Linux, macOS and Windows. Finally add a workflow to build Windows wheels during a release.
1 parent c5c177a commit af96031

39 files changed

+697
-872
lines changed

.github/workflows/ci-build-release-wheels.yaml

+59
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,62 @@ jobs:
141141
name: wheel-mac-py${{matrix.py.version}}
142142
path: dist/*.whl
143143

144+
windows-wheels:
145+
name: Wheel Windows - Py ${{matrix.py.version}}
146+
runs-on: windows-2019
147+
env:
148+
PULSAR_CPP_DIR: 'C:\\pulsar-cpp'
149+
timeout-minutes: 300
150+
151+
strategy:
152+
fail-fast: false
153+
matrix:
154+
windows:
155+
- name: 'Windows x64'
156+
os: windows-2019
157+
arch: '-A x64'
158+
triplet: 'x64-windows'
159+
python:
160+
- {version: '3.7'}
161+
- {version: '3.8'}
162+
- {version: '3.9'}
163+
- {version: '3.10'}
164+
165+
steps:
166+
- uses: actions/checkout@v3
167+
168+
- uses: actions/setup-python@v4
169+
with:
170+
python-version: ${{ matrix.python.version }}
171+
172+
- name: Download Pulsar C++ client on Windows
173+
shell: bash
174+
run: |
175+
mkdir -p ${{ env.PULSAR_CPP_DIR }}
176+
cd ${{ env.PULSAR_CPP_DIR }}
177+
# TODO: switch to official releases
178+
curl -O -L https://github.com/BewareMyPower/pulsar-client-cpp/releases/download/v3.1.0-rc-20221028/x64-windows-static.zip
179+
unzip -q x64-windows-static.zip
180+
ls -l ${{ env.PULSAR_CPP_DIR }}
181+
182+
- name: Configure CMake
183+
shell: bash
184+
run: |
185+
pip3 install pyyaml
186+
export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
187+
curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
188+
tar zxf v${PYBIND11_VERSION}.tar.gz
189+
rm -rf pybind11
190+
mv pybind11-${PYBIND11_VERSION} pybind11
191+
cmake -B build ${{ matrix.windows.arch }} \
192+
-DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
193+
-DLINK_STATIC=ON
194+
195+
- name: Build Python wheel
196+
shell: bash
197+
run: |
198+
cmake --build build --config Release --target install
199+
python -m pip install wheel
200+
python setup.py bdist_wheel
201+
python -m pip install ./dist/*.whl
202+
python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()'

.github/workflows/ci-pr-validation.yaml

+8-30
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,8 @@ jobs:
5555
- name: checkout
5656
uses: actions/checkout@v3
5757

58-
- name: Install deps
59-
run: sudo apt-get install -y libboost-python-dev
60-
6158
- name: Install Pulsar C++ client
62-
run: build-support/install-cpp-client.sh
59+
run: build-support/install-dependencies.sh
6360

6461
- name: CMake
6562
run: cmake .
@@ -199,20 +196,11 @@ jobs:
199196

200197
steps:
201198
- uses: actions/checkout@v3
202-
with:
203-
submodules: true
204199

205200
- uses: actions/setup-python@v4
206201
with:
207202
python-version: ${{ matrix.python.version }}
208203

209-
- name: Prepare vcpkg.json
210-
shell: bash
211-
run: |
212-
python --version
213-
cp -f vcpkg-${{ matrix.python.version }}.json vcpkg.json
214-
cat vcpkg.json
215-
216204
- name: Download Pulsar C++ client on Windows
217205
shell: bash
218206
run: |
@@ -223,24 +211,17 @@ jobs:
223211
unzip -q ${{ matrix.windows.triplet }}-static.zip
224212
ls -l ${{ env.PULSAR_CPP_DIR }}
225213
226-
- name: Cache Vcpkg
227-
uses: actions/cache@v3
228-
id: cache-vcpkg
229-
with:
230-
path: build/vcpkg_installed
231-
key: ${{ matrix.python.version }}-${{ hashFiles(format('vcpkg-{0}.json', matrix.python.version)) }}
232-
233-
- name: Install dependencies and configure CMake
214+
- name: Configure CMake
234215
shell: bash
235216
run: |
236-
COMMIT_ID=$(grep baseline vcpkg.json | sed 's/[",]//g' | awk '{print $2}')
237-
cd vcpkg
238-
echo "git fetch origin $COMMIT_ID"
239-
git fetch origin $COMMIT_ID
240-
cd -
217+
pip3 install pyyaml
218+
export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
219+
curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
220+
tar zxf v${PYBIND11_VERSION}.tar.gz
221+
rm -rf pybind11
222+
mv pybind11-${PYBIND11_VERSION} pybind11
241223
cmake -B build ${{ matrix.windows.arch }} \
242224
-DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
243-
-DUSE_VCPKG=ON \
244225
-DLINK_STATIC=ON
245226
246227
- name: Build Python wheel
@@ -250,9 +231,6 @@ jobs:
250231
python -m pip install wheel
251232
python setup.py bdist_wheel
252233
python -m pip install ./dist/*.whl
253-
cp ./build/Release/boost_python*.dll .
254-
echo "The extra DLLs:"
255-
ls -l *.dll
256234
python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()'
257235
258236
check-completion:

.gitmodules

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
[submodule "vcpkg"]
2-
path = vcpkg
3-
url = https://github.com/microsoft/vcpkg.git
1+
[submodule "pybind11"]
2+
path = pybind11
3+
url = https://github.com/pybind/pybind11.git

CMakeLists.txt

+6-40
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,8 @@
2020
cmake_minimum_required(VERSION 3.18)
2121
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
2222

23-
option(USE_VCPKG "Use Vcpkg to install dependencies" OFF)
24-
if (USE_VCPKG)
25-
set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake"
26-
CACHE STRING "Vcpkg toolchain file")
27-
endif ()
2823
project (pulsar-client-python)
24+
set(CMAKE_PREFIX_PATH ${PROJECT_SOURCE_DIR}/pybind11/include ${CMAKE_PREFIX_PATH})
2925
option(LINK_STATIC "Link against static libraries" OFF)
3026
MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC})
3127

@@ -58,43 +54,15 @@ SET(CMAKE_CXX_STANDARD 11)
5854
find_package (Python3 REQUIRED COMPONENTS Development.Module)
5955
MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})
6056

61-
find_package(Boost REQUIRED ${Boost_INCLUDE_DIRS})
62-
message(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
63-
64-
SET(Boost_USE_STATIC_LIBS ${LINK_STATIC})
65-
66-
set(BOOST_PYTHON_NAME_LIST python3 python310 python39 python38 python37)
67-
foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
68-
find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
69-
if (${Boost_FOUND})
70-
set(BOOST_PYTHON_COMPONENT_FOUND ${BOOST_PYTHON_NAME})
71-
message(STATUS "Found Boost COMPONENTS " ${BOOST_PYTHON_COMPONENT_FOUND})
72-
break ()
73-
endif ()
74-
endforeach ()
75-
if (NOT BOOST_PYTHON_COMPONENT_FOUND)
76-
message(FATAL_ERROR "Could not find Boost Python library")
77-
endif ()
57+
find_path(PYBIND11_INCLUDE_DIRS NAMES "pybind11/pybind11.h")
58+
message(STATUS "PYBIND11_INCLUDE_DIRS: " ${PYBIND11_INCLUDE_DIRS})
7859

7960
########################################################################################################################
8061

81-
INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")
82-
83-
ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
84-
src/producer.cc
85-
src/consumer.cc
86-
src/config.cc
87-
src/enums.cc
88-
src/client.cc
89-
src/message.cc
90-
src/authentication.cc
91-
src/reader.cc
92-
src/schema.cc
93-
src/cryptoKeyReader.cc
94-
src/exceptions.cc
95-
src/utils.cc
96-
)
62+
INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} ${PYBIND11_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS})
9763

64+
file(GLOB SOURCES src/*.cc)
65+
ADD_LIBRARY(_pulsar SHARED ${SOURCES})
9866
if (MSVC)
9967
set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd)
10068
else ()
@@ -109,10 +77,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
10977
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -Qunused-arguments -undefined dynamic_lookup")
11078
endif()
11179

112-
# Try all possible boost-python variable namings
11380
set(PYTHON_WRAPPER_LIBS
11481
${PULSAR_LIBRARY}
115-
Boost::${BOOST_PYTHON_COMPONENT_FOUND}
11682
)
11783
if (MSVC)
11884
set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module)

README.md

+18-24
Original file line numberDiff line numberDiff line change
@@ -27,49 +27,43 @@
2727
- A C++ compiler that supports C++11
2828
- CMake >= 3.18
2929
- [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp)
30-
- [Boost.Python](https://github.com/boostorg/python)
30+
- [PyBind11](https://github.com/pybind/pybind11)
3131

32-
## Install the Python wheel
33-
34-
### Windows (with Vcpkg)
32+
PyBind11 is a header-only library and a submodule, so you can simply download the submodule so that CMake can find this dependency.
3533

36-
First, install the dependencies via [Vcpkg](https://github.com/microsoft/vcpkg).
37-
38-
```PowerShell
39-
vcpkg install --feature-flags=manifests --triplet x64-windows
34+
```bash
35+
git submodule update --init
4036
```
4137

42-
> NOTE: For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/tree/master/triplets) for all available triplets.
38+
You can also download the pybind11 directly like:
4339

44-
Then, build and install the Python wheel.
45-
46-
```PowerShell
47-
# Assuming the Pulsar C++ client has been installed under the `PULSAR_CPP` directory.
48-
cmake -B build -DUSE_VCPKG=ON -DCMAKE_PREFIX_PATH="$env:PULSAR_CPP" -DLINK_STATIC=ON
49-
cmake --build build --config Release
50-
cmake --install build
51-
py setup.py bdist_wheel
52-
py -m pip install ./dist/pulsar_client-*.whl
40+
```bash
41+
pip3 install pyyaml
42+
export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
43+
curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
44+
tar zxf v${PYBIND11_VERSION}.tar.gz
45+
mv pybind11-${PYBIND11_VERSION} pybind11
5346
```
5447

55-
Since the Python client links to Boost.Python dynamically, you have to copy the dll (e.g. `boost_python310-vc142-mt-x64-1_80.dll`) into the system path (the `PATH` environment variable). If the `-DLINK_STATIC=ON` option is not specified, you have to copy the `pulsar.dll` into the system path as well.
48+
After that, you only need to install the Pulsar C++ client dependency into the system path. You can [install the pre-built binaries](https://pulsar.apache.org/docs/next/client-libraries-cpp/#installation) or [build from source](https://github.com/apache/pulsar-client-cpp#compilation).
5649

57-
### Linux or macOS
50+
## Install the Python wheel
5851

59-
Assuming the Pulsar C++ client and Boost.Python have been installed under the system path.
52+
Make sure the PyBind11 submodule has been downloaded and the Pulsar C++ client has been installed. Then run the following commands:
6053

6154
```bash
6255
cmake -B build
63-
cmake --build build -j8
56+
cmake --build build
6457
cmake --install build
65-
./setup.py bdist_wheel
66-
pip3 install dist/pulsar_client-*.whl --force-reinstall
58+
python3 ./setup.py bdist_wheel
59+
python3 -m pip install dist/pulsar_client-*.whl --force-reinstall
6760
```
6861

6962
> **NOTE**
7063
>
7164
> 1. Here a separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` is under the project directory.
7265
> 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before.
66+
> 3. On Windows, the Python command is `py` instead of `python3`.
7367
7468
## Running examples
7569

build-support/copy-deps-versionfile.sh

-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,5 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
2525
for dir in manylinux2014 manylinux_musl; do
2626
mkdir -p pkg/$dir/.build
2727
cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
28-
cp $ROOT_DIR/pulsar-client-cpp-version.txt pkg/$dir/.build
2928
cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
3029
done

build-support/dep-version.py

+11-1
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,15 @@
2020

2121
import yaml, sys
2222

23-
deps = yaml.safe_load(open("dependencies.yaml"))
23+
if len(sys.argv) < 2:
24+
print(f'''Usage: {sys.argv[0]} dependency-name [dependency-file]
25+
26+
The dependency file is "dependencies.yaml" by default.''')
27+
sys.exit(1)
28+
29+
if len(sys.argv) > 2:
30+
dependency_file = sys.argv[2]
31+
else:
32+
dependency_file = 'dependencies.yaml'
33+
deps = yaml.safe_load(open(dependency_file))
2434
print(deps[sys.argv[1]])

build-support/install-cpp-client.sh renamed to build-support/install-dependencies.sh

+8-4
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020

2121
set -e -x
2222

23-
ROOT_DIR=$(dirname $(dirname $0))
24-
CPP_CLIENT_VERSION=$(cat $ROOT_DIR/pulsar-client-cpp-version.txt | xargs)
23+
cd `dirname $0`
24+
25+
CPP_CLIENT_VERSION=$(./dep-version.py pulsar-cpp ../dependencies.yaml)
26+
PYBIND11_VERSION=$(./dep-version.py pybind11 ../dependencies.yaml)
2527

2628
if [ $USER != "root" ]; then
2729
SUDO="sudo"
@@ -62,5 +64,7 @@ else
6264
exit 1
6365
fi
6466

65-
66-
67+
curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
68+
tar zxf v${PYBIND11_VERSION}.tar.gz
69+
$SUDO cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 /usr/include/
70+
rm -rf pybind11-${PYBIND11_VERSION} v${PYBIND11_VERSION}.tar.gz

dependencies.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@
1717
# under the License.
1818
#
1919

20-
boost: 1.80.0
2120
cmake: 3.24.2
21+
pulsar-cpp: 3.0.0
22+
pybind11: 2.10.1
23+
boost: 1.80.0
2224
protobuf: 3.20.0
2325
zlib: 1.2.13
2426
zstd: 1.5.2

pkg/build-wheel-inside-docker.sh

+6
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ set -e -x
2222

2323
cd /pulsar-client-python
2424

25+
PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
26+
curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
27+
tar zxf v${PYBIND11_VERSION}.tar.gz
28+
rm -rf pybind11
29+
mv pybind11-${PYBIND11_VERSION} pybind11
30+
2531
rm -f CMakeCache.txt CMakeFiles
2632

2733
cmake . \

0 commit comments

Comments
 (0)