Skip to content

PIP-209: Compile Python client wrapper #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 30, 2022
Merged

PIP-209: Compile Python client wrapper #1

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c46d801
PIP-209: Compile Python client wrapper
merlimat Sep 29, 2022
718602e
Removed Optional
merlimat Sep 30, 2022
7f289ec
Fixed CMakefile
merlimat Sep 30, 2022
112dd6f
Added __pycache__ to gitignore
merlimat Sep 30, 2022
0b3b5c0
Added clang-format support
merlimat Sep 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,10 @@ MANIFEST
build
dist
*.egg-info
.idea
CMakeCache.txt
CMakeFiles
Makefile
_pulsar.so
cmake_install.cmake

105 changes: 99 additions & 6 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,93 @@
# under the License.
#

INCLUDE_DIRECTORIES("${Boost_INCLUDE_DIRS}" "${PYTHON_INCLUDE_DIRS}")
project (pulsar-client-python)
cmake_minimum_required(VERSION 3.12)

MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
set(THREADS_PREFER_PTHREAD_FLAG TRUE)
find_package(Threads REQUIRED)
MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})


find_library(PULSAR_LIBRARY REQUIRED NAMES libpulsar.a)
message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")

find_path(PULSAR_INCLUDE REQUIRED pulsar/Client.h)
message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}")

SET(Boost_NO_BOOST_CMAKE ON)
SET(Boost_USE_STATIC_LIBS ON)

SET(CMAKE_CXX_STANDARD 11)

find_package(Boost)

find_package (Python3 COMPONENTS Development)
MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})

string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${Python3_VERSION})

set(BOOST_PYTHON_NAME_POSTFIX ${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR})
# For python3 the lib name is boost_python3
set(BOOST_PYTHON_NAME_LIST python${BOOST_PYTHON_NAME_POSTFIX};python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt)

foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
if (${Boost_FOUND})
set(BOOST_PYTHON_NAME_FOUND ${BOOST_PYTHON_NAME})
break()
endif()
endforeach()

if (NOT ${Boost_FOUND})
MESSAGE(FATAL_ERROR "Could not find Boost Python library")
endif ()

MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND})

set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/)

### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY
### After find the libs, will unset related cache, and will not affect another same call to find_package.
if (APPLE)
set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include)
set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
endif ()

set(OPENSSL_USE_STATIC_LIBS TRUE)
find_package(OpenSSL REQUIRED)
set(RECORD_OPENSSL_SSL_LIBRARY ${OPENSSL_SSL_LIBRARY})
set(RECORD_OPENSSL_CRYPTO_LIBRARY ${OPENSSL_CRYPTO_LIBRARY})

unset(OPENSSL_FOUND CACHE)
unset(OPENSSL_INCLUDE_DIR CACHE)
unset(OPENSSL_CRYPTO_LIBRARY CACHE)
unset(OPENSSL_CRYPTO_LIBRARIES CACHE)
unset(OPENSSL_SSL_LIBRARY CACHE)
unset(OPENSSL_SSL_LIBRARIES CACHE)
unset(OPENSSL_LIBRARIES CACHE)
unset(OPENSSL_VERSION CACHE)

find_package(OpenSSL REQUIRED)

find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib)
message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")

find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf)
message(STATUS "Protobuf: ${Protobuf_LIBRARIES}")
find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
find_library(LIB_ZSTD NAMES libzstd.a)
message(STATUS "ZStd: ${LIB_ZSTD}")
find_library(LIB_SNAPPY NAMES libsnappy.a)
message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")

########################################################################################################################

INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")

ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
src/producer.cc
Expand Down Expand Up @@ -63,7 +149,14 @@ if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY})
endif()

# Try all possible boost-python variable namings
set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
${OPENSSL_LIBRARIES}
${ZLIB_LIBRARIES}
${CURL_LIBRARIES}
${Protobuf_LIBRARIES}
${LIB_ZSTD}
${LIB_SNAPPY}
${Boost_PYTHON_LIBRARY}
${Boost_PYTHON3_LIBRARY}
${Boost_PYTHON37-MT_LIBRARY}
${Boost_PYTHON38_LIBRARY}
Expand All @@ -86,18 +179,18 @@ if (APPLE)
endif ()
endif()

message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}")

if (NOT PYTHON_WRAPPER_LIBS)
MESSAGE(FATAL_ERROR "Could not find Boost Python library")
endif ()

message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")

if (APPLE)
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
target_link_libraries(_pulsar -Wl,-all_load pulsarStatic ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS} ${ICU_LIBS})
target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS} ${ICU_LIBS})
else ()
if (NOT MSVC)
set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++")
endif()
target_link_libraries(_pulsar pulsarStatic ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS})
target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS} ${COMMON_LIBS})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

COMMON_LIBS can be removed because it's originally defined in CMakeLists.txt of pulsar-client-cpp. ICU_LIBS can be removed as well. (I never found a reference)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

endif ()
54 changes: 44 additions & 10 deletions build-mac-wheels.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ SNAPPY_VERSION=1.1.3
CURL_VERSION=7.61.0

ROOT_DIR=$(git rev-parse --show-toplevel)
cd "${ROOT_DIR}/pulsar-client-cpp"
cd "${ROOT_DIR}"

PULSAR_VERSION=$(cat version.txt | grep pulsar-client-cpp | awk '{print $2}')

# Compile and cache dependencies
CACHE_DIR=~/.pulsar-mac-wheels-cache
Expand Down Expand Up @@ -246,6 +247,43 @@ else
echo "Using cached LibCurl"
fi

###############################################################################
if [ ! -f apache-pulsar-${PULSAR_VERSION}-src/.done ]; then
echo "Building Pulsar C++ client - ${PULSAR_VERSION}"
curl -O -L https://archive.apache.org/dist/pulsar/pulsar-${PULSAR_VERSION}/apache-pulsar-${PULSAR_VERSION}-src.tar.gz
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should probably add some validation of the integrity of the archive

and also provide a way to pass a custom .tar.gz, in order to ease development/testing with a different version
maybe making the full URL configurable will help

we can do these improvements as a follow up patch

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we could be checking the expected hash of the archive

rm -rf apache-pulsar-${PULSAR_VERSION}-src/pulsar-client-cpp
tar xfz apache-pulsar-${PULSAR_VERSION}-src.tar.gz
pushd apache-pulsar-${PULSAR_VERSION}-src
pushd pulsar-client-cpp
ARCHS='arm64;x86_64'

chmod +x build-support/merge_archives.sh
set -x
cmake . \
-DCMAKE_OSX_ARCHITECTURES=${ARCHS} \
-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH=$PREFIX \
-DCMAKE_CXX_FLAGS=-I$PREFIX/include \
-DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \
-DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \
-DLINK_STATIC=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_PYTHON_WRAPPER=OFF \
-DBUILD_WIRESHARK=OFF \
-DBUILD_DYNAMIC_LIB=OFF \
-DBUILD_STATIC_LIB=ON \
-DPROTOC_PATH=$PREFIX/bin/protoc

make -j16 install
popd
touch .done
popd
else
echo "Using cached Pulsar C++ client"
fi

###############################################################################
###############################################################################
###############################################################################
Expand All @@ -260,7 +298,7 @@ for line in "${PYTHON_VERSIONS[@]}"; do
echo '----------------------------------------------------------------------------'
echo "Build wheel for Python $PYTHON_VERSION"

cd "${ROOT_DIR}/pulsar-client-cpp"
cd "${ROOT_DIR}"

find . -name CMakeCache.txt | xargs -r rm
find . -name CMakeFiles | xargs -r rm -rf
Expand All @@ -285,16 +323,12 @@ for line in "${PYTHON_VERSIONS[@]}"; do
-DCMAKE_CXX_FLAGS=-I$PREFIX/include \
-DBoost_INCLUDE_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/include \
-DBoost_LIBRARY_DIR=$CACHE_DIR/boost-py-$PYTHON_VERSION/lib \
-DPYTHON_INCLUDE_DIR=$PY_INCLUDE_DIR \
-DPYTHON_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \
-DLINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DBUILD_WIRESHARK=OFF \
-DPROTOC_PATH=$PREFIX/bin/protoc
-DPython3_INCLUDE_DIR=$PY_INCLUDE_DIR \
-DPython3_LIBRARY=$PY_PREFIX/lib/libpython${PYTHON_VERSION}.dylib \
-DPULSAR_INCLUDE=${PREFIX}/include

make clean
make _pulsar -j16
make -j16

cd python
$PY_EXE setup.py bdist_wheel
done
30 changes: 10 additions & 20 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,29 +19,19 @@

from setuptools import setup
from distutils.core import Extension
from distutils.util import strtobool
from os import environ
from os import environ, path

from distutils.command import build_ext

import xml.etree.ElementTree as ET
from os.path import dirname, realpath, join

def get_version():
use_full_pom_name = strtobool(environ.get('USE_FULL_POM_NAME', 'False'))

# Get the pulsar version from pom.xml
TOP_LEVEL_PATH = dirname(dirname(dirname(realpath(__file__))))
POM_PATH = join(TOP_LEVEL_PATH, 'pom.xml')
root = ET.XML(open(POM_PATH).read())
version = root.find('{http://maven.apache.org/POM/4.0.0}version').text.strip()

if use_full_pom_name:
return version
else:
# Strip the '-incubating' suffix, since it prevents the packages
# from being uploaded into PyPI
return version.split('-')[0]
# Get the pulsar version from version.txt
root = path.dirname(path.realpath(__file__))
version_file = path.join(root, 'version.txt')
with open(version_file) as f:
for line in f.readlines():
if 'pulsar-client-python: ' in line:
return line.split()[-1].strip()


def get_name():
Expand All @@ -53,8 +43,8 @@ def get_name():
VERSION = get_version()
NAME = get_name()

print(VERSION)
print(NAME)
print('NAME: %s' % NAME)
print('VERSION: %s' % VERSION)


# This is a workaround to have setuptools to include
Expand Down
1 change: 0 additions & 1 deletion src/config.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
#include "utils.h"
#include <pulsar/ConsoleLoggerFactory.h>
#include "lib/Utils.h"
#include <memory>

template <typename T>
Expand Down
Loading