Skip to content

Commit 1554e39

Browse files
merlimatmerlimat
authored
Fixed building of Linux wheel files (#45)
The current build process for Python wheels is to fetch the RPM packages of the C++ release and use the `libpulsar.so` from there. There are some issues due to incompatibilities between different versions of `libstdc++`, causing segmentation faults. In particular problems with callbacks and `std::function` objects compiled in different environments. To avoid these issues, we need to compile the C++ client lib when we're building the Python wrapper.
1 parent e0ff5ee commit 1554e39

File tree

5 files changed

+138
-5
lines changed

5 files changed

+138
-5
lines changed

‎build-support/copy-deps-versionfile.sh

+1
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
2525
for dir in manylinux2014 manylinux_musl; do
2626
mkdir -p pkg/$dir/.build
2727
cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
28+
cp $ROOT_DIR/pulsar-client-cpp-version.txt pkg/$dir/.build
2829
cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
2930
done

‎dependencies.yaml

-2
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
#
1919

2020
boost: 1.80.0
21-
22-
# Only used for MacOS builds
2321
cmake: 3.24.2
2422
protobuf: 3.20.0
2523
zlib: 1.2.13

‎pkg/build-wheel-inside-docker.sh

-2
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ set -e -x
2222

2323
cd /pulsar-client-python
2424

25-
build-support/install-cpp-client.sh
26-
2725
rm -f CMakeCache.txt CMakeFiles
2826

2927
cmake . \

‎pkg/manylinux2014/Dockerfile

+69-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ FROM quay.io/pypa/manylinux2014_${ARCH}
2222

2323
ARG PYTHON_VERSION
2424
ARG PYTHON_SPEC
25-
ARG PLATFORM
2625
ARG ARCH
2726

2827
ENV PYTHON_VERSION=${PYTHON_VERSION}
@@ -36,6 +35,7 @@ ENV PYTHON_LIBRARIES /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
3635
RUN pip3 install pyyaml
3736

3837
ADD .build/dependencies.yaml /
38+
ADD .build/pulsar-client-cpp-version.txt /
3939
ADD .build/dep-version.py /usr/local/bin
4040

4141
# Download and compile boost
@@ -54,3 +54,71 @@ RUN CMAKE_VERSION=$(dep-version.py cmake) && \
5454
cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \
5555
cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \
5656
rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz
57+
58+
# Download and compile protobuf
59+
RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
60+
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
61+
tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
62+
cd protobuf-${PROTOBUF_VERSION}/ && \
63+
CXXFLAGS=-fPIC ./configure && \
64+
make -j8 && make install && ldconfig && \
65+
rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
66+
67+
# ZLib
68+
RUN ZLIB_VERSION=$(dep-version.py zlib) && \
69+
curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
70+
tar xfz v${ZLIB_VERSION}.tar.gz && \
71+
cd zlib-${ZLIB_VERSION} && \
72+
CFLAGS="-fPIC -O3" ./configure && \
73+
make -j8 && make install && \
74+
rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
75+
76+
# Zstandard
77+
RUN ZSTD_VERSION=$(dep-version.py zstd) && \
78+
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
79+
tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
80+
cd zstd-${ZSTD_VERSION} && \
81+
CFLAGS="-fPIC -O3" make -j8 && \
82+
make install && \
83+
rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
84+
85+
# Snappy
86+
RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
87+
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
88+
tar xfz ${SNAPPY_VERSION}.tar.gz && \
89+
cd snappy-${SNAPPY_VERSION} && \
90+
CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
91+
make -j8 && make install && \
92+
rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
93+
94+
RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
95+
OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
96+
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
97+
tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
98+
cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
99+
./config -fPIC --prefix=/usr/local/ssl/ && \
100+
make -j8 && make install && \
101+
rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
102+
103+
ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib
104+
ENV OPENSSL_ROOT_DIR /usr/local/ssl/
105+
106+
# LibCurl
107+
RUN CURL_VERSION=$(dep-version.py curl) && \
108+
CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
109+
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
110+
tar xfz curl-${CURL_VERSION}.tar.gz && \
111+
cd curl-${CURL_VERSION} && \
112+
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
113+
make -j8 && make install && \
114+
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
115+
116+
# Pulsar client C++
117+
RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \
118+
curl -O -L https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
119+
tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
120+
cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \
121+
cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \
122+
make -j8 && \
123+
make install && \
124+
rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz

‎pkg/manylinux_musl/Dockerfile

+68
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ RUN pip install pyyaml
3838
RUN apk add cmake
3939

4040
ADD .build/dependencies.yaml /
41+
ADD .build/pulsar-client-cpp-version.txt /
4142
ADD .build/dep-version.py /usr/local/bin
4243

4344
# Download and compile boost
@@ -51,3 +52,70 @@ RUN BOOST_VERSION=$(dep-version.py boost) && \
5152
rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE}
5253

5354

55+
# Download and compile protobuf
56+
RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
57+
curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
58+
tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
59+
cd protobuf-${PROTOBUF_VERSION}/ && \
60+
CXXFLAGS=-fPIC ./configure && \
61+
make -j8 && make install && \
62+
rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
63+
64+
# ZLib
65+
RUN ZLIB_VERSION=$(dep-version.py zlib) && \
66+
curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
67+
tar xfz v${ZLIB_VERSION}.tar.gz && \
68+
cd zlib-${ZLIB_VERSION} && \
69+
CFLAGS="-fPIC -O3" ./configure && \
70+
make -j8 && make install && \
71+
rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
72+
73+
# Zstandard
74+
RUN ZSTD_VERSION=$(dep-version.py zstd) && \
75+
curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
76+
tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
77+
cd zstd-${ZSTD_VERSION} && \
78+
CFLAGS="-fPIC -O3" make -j8 && \
79+
make install && \
80+
rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
81+
82+
# Snappy
83+
RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
84+
curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
85+
tar xfz ${SNAPPY_VERSION}.tar.gz && \
86+
cd snappy-${SNAPPY_VERSION} && \
87+
CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
88+
make -j8 && make install && \
89+
rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
90+
91+
RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
92+
OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
93+
curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
94+
tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
95+
cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
96+
./config -fPIC --prefix=/usr/local/ssl/ && \
97+
make -j8 && make install && \
98+
rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
99+
100+
ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:/usr/local/lib
101+
ENV OPENSSL_ROOT_DIR /usr/local/ssl/
102+
103+
# LibCurl
104+
RUN CURL_VERSION=$(dep-version.py curl) && \
105+
CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
106+
curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
107+
tar xfz curl-${CURL_VERSION}.tar.gz && \
108+
cd curl-${CURL_VERSION} && \
109+
CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
110+
make -j8 && make install && \
111+
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
112+
113+
# Pulsar client C++
114+
RUN PULSAR_CPP_VERSION=$(cat pulsar-client-cpp-version.txt | xargs) && \
115+
curl -O -L https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
116+
tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
117+
cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \
118+
cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \
119+
make -j8 && \
120+
make install && \
121+
rm -rf apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz

0 commit comments

Comments
 (0)