Support building with MSVC

BewareMyPower · BewareMyPower · commit a2c04af9edde · 2022-11-01T17:26:26.000+08:00
Fixes apache#7 The Python client cannot be built with MSVC. CMakeLists.txt: 1. Boost.Python cannot be found on Windows. The component of Boost cannot be `python3`. It should be a specific version like `python310`. Therefore, find all possible components from 3.10 to 3.7 until the first component is available. 2. For MSVC, link to `pulsarWithDeps.lib` when `LINK_STATIC` is `ON` and set the `MSVC_RUNTIME_LIBRARY` target property to `MultiThreaded`. 3. Change the suffix from `.so` to `.pyd` because Python on Windows recognizes `*.pyd` as the dynamic library. 4. Link to Python library with MSVC, otherwise the symbos cannot be found when linking `boost-python`. README: tell users how to build Python client on Windows. Add a workflow to build Python wheels of versions 3.7 to 3.10 on Windows and verify the build will succeed. With this PR, Windows users still need to put the related `boost_python*.dll` under the system path. In future, the `boost-python` dependency will be removed. See apache#24.
diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml
@@ -67,6 +67,7 @@ jobs:
 
   linux-wheel:
     name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}}
+    needs: unit-tests
     runs-on: ubuntu-22.04
     timeout-minutes: 300
 
@@ -118,6 +119,7 @@ jobs:
 
   mac-wheels:
     name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
+    needs: unit-tests
     runs-on: macos-12
     timeout-minutes: 300
 
@@ -155,11 +157,92 @@ jobs:
       - name: Build and test Mac wheels
         run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
 
+  windows-wheels:
+    name: "Python ${{ matrix.python.version }} Wheel on ${{ matrix.windows.name }}"
+    needs: unit-tests
+    runs-on: ${{ matrix.windows.os }}
+    timeout-minutes: 120
+
+    env:
+      PULSAR_CPP_DIR: 'C:\\pulsar-cpp'
+    strategy:
+      fail-fast: false
+      matrix:
+        windows:
+          - name: 'Windows x64'
+            os: windows-2022
+            arch: '-A x64'
+            triplet: 'x64-windows'
+        python:
+          - version: '3.7'
+          - version: '3.8'
+          - version: '3.9'
+          - version: '3.10'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python.version }}
+
+      - name: Prepare vcpkg.json
+        shell: bash
+        run: |
+          python --version
+          cp -f vcpkg-${{ matrix.python.version }}.json vcpkg.json
+          cat vcpkg.json
+
+      - name: Download Pulsar C++ client on Windows
+        shell: bash
+        run: |
+          mkdir -p ${{ env.PULSAR_CPP_DIR }}
+          cd ${{ env.PULSAR_CPP_DIR }}
+          # TODO: switch to official releases
+          curl -O -L https://github.com/BewareMyPower/pulsar-client-cpp/releases/download/v3.1.0-rc-20221028/${{ matrix.windows.triplet }}-static.zip
+          unzip -q ${{ matrix.windows.triplet }}-static.zip
+          ls -l ${{ env.PULSAR_CPP_DIR }}
+
+      - name: Dependencies cache
+        uses: actions/cache@v3
+        id: cache-deps
+        with:
+          path: build/vcpkg_installed
+          key: ${{ matrix.python.version }}-${{ hashFiles("vcpkg-${{ matrix.python.version }}.json") }}
+
+      - name: Install dependencies and configure CMake
+        shell: bash
+        run: |
+          COMMIT_ID=$(grep baseline vcpkg.json | sed 's/[",]//g' | awk '{print $2}')
+          cd vcpkg
+          echo "git fetch origin $COMMIT_ID"
+          git fetch origin $COMMIT_ID
+          cd -
+          cmake -B build ${{ matrix.windows.arch }} \
+            -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
+            -DUSE_VCPKG=ON \
+            -DLINK_STATIC=ON
+
+      - name: Build Python wheel
+        shell: bash
+        run: |
+          cmake --build build --config Release --target install
+          python -m pip install wheel
+          python setup.py bdist_wheel
+          python -m pip install ./dist/*.whl
+          cp ./build/Release/boost_python*.dll .
+          echo "The extra DLLs:"
+          ls -l *.dll
+          python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()'
+
+
   # Job that will be required to complete and depends on all the other jobs
   check-completion:
     name: Check Completion
     runs-on: ubuntu-latest
-    needs: [unit-tests, linux-wheel, mac-wheels]
+    needs: [unit-tests, linux-wheel, mac-wheels, windows-wheels]
 
     steps:
       - run: true
diff --git a/.gitignore b/.gitignore
@@ -13,4 +13,7 @@ __pycache__
 .pulsar-mac-wheels-cache
 .DS_Store
 wheelhouse
-.pulsar-mac-build
+.pulsar-mac-build
+vcpkg_installed/
+*.pyd
+*.lib
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -17,10 +17,15 @@
 # under the License.
 #
 
-project (pulsar-client-python)
 cmake_minimum_required(VERSION 3.18)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
+option(USE_VCPKG "Use Vcpkg to install dependencies" OFF)
+if (USE_VCPKG)
+    set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake"
+        CACHE STRING "Vcpkg toolchain file")
+endif ()
+project (pulsar-client-python)
 option(LINK_STATIC "Link against static libraries" OFF)
 MESSAGE(STATUS "LINK_STATIC:  " ${LINK_STATIC})
 
@@ -29,10 +34,19 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
 
+if (MSVC)
+    add_compile_options(/wd4819)
+endif ()
+
 if (LINK_STATIC)
-    find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+    if (MSVC)
+        find_library(PULSAR_LIBRARY NAMES pulsarWithDeps.lib)
+    else ()
+        find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+    endif ()
+    add_definitions("-DPULSAR_STATIC")
 else()
-    find_library(PULSAR_LIBRARY NAMES libpulsar.so  libpulsar.dylib)
+    find_library(PULSAR_LIBRARY NAMES pulsar libpulsar)
 endif()
 message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
 
@@ -44,10 +58,23 @@ SET(CMAKE_CXX_STANDARD 11)
 find_package (Python3 REQUIRED COMPONENTS Development.Module)
 MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})
 
+find_package(Boost REQUIRED ${Boost_INCLUDE_DIRS})
+message(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
+
 SET(Boost_USE_STATIC_LIBS   ${LINK_STATIC})
-find_package(Boost REQUIRED COMPONENTS python3)
-MESSAGE(STATUS "Boost Python3: " ${Boost_PYTHON3_LIBRARY})
-MESSAGE(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
+
+set(BOOST_PYTHON_NAME_LIST python3 python310 python39 python38 python37)
+foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
+    find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
+    if (${Boost_FOUND})
+        set(BOOST_PYTHON_COMPONENT_FOUND ${BOOST_PYTHON_NAME})
+        message(STATUS "Found Boost COMPONENTS " ${BOOST_PYTHON_COMPONENT_FOUND})
+        break ()
+    endif ()
+endforeach ()
+if (NOT BOOST_PYTHON_COMPONENT_FOUND)
+    message(FATAL_ERROR "Could not find Boost Python library")
+endif ()
 
 ########################################################################################################################
 
@@ -68,8 +95,11 @@ ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
                            src/utils.cc
         )
 
-SET(CMAKE_SHARED_LIBRARY_PREFIX )
-SET(CMAKE_SHARED_LIBRARY_SUFFIX .so)
+if (MSVC)
+    set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd)
+else ()
+    set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
+endif ()
 
 if (NOT APPLE AND NOT MSVC)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_PYTHON}")
@@ -80,12 +110,17 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
 endif()
 
 # Try all possible boost-python variable namings
-set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
-                        ${Boost_PYTHON3_LIBRARY})
+set(PYTHON_WRAPPER_LIBS
+    ${PULSAR_LIBRARY}
+    Boost::${BOOST_PYTHON_COMPONENT_FOUND}
+)
+if (MSVC)
+    set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module)
+endif ()
 
 message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
 
-if (LINK_STATIC)
+if (LINK_STATIC AND NOT MSVC)
     set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
 
     # We need to include all the static libs individually because we cannot easily create a universal2 libpulsar.a
@@ -125,9 +160,14 @@ if (LINK_STATIC)
         endif()
         target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
     endif ()
+elseif (LINK_STATIC) # MSVC
+    set_property(TARGET _pulsar PROPERTY
+        MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+    target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
 else()
     target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
 endif ()
+install(TARGETS _pulsar DESTINATION ${CMAKE_SOURCE_DIR})
 
 find_package(ClangTools)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
diff --git a/README.md b/README.md
@@ -31,20 +31,48 @@
 
 ## Install the Python wheel
 
+### Windows (with Vcpkg)
+
+First, install the dependencies via [Vcpkg](https://github.com/microsoft/vcpkg).
+
+```PowerShell
+vcpkg install --feature-flags=manifests --triplet x64-windows
+```
+
+> NOTE: For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/tree/master/triplets) for all available triplets.
+
+Then, build and install the Python wheel.
+
+```PowerShell
+# Assuming the Pulsar C++ client has been installed under the `PULSAR_CPP` directory.
+cmake -B build -DUSE_VCPKG=ON -DCMAKE_PREFIX_PATH="$env:PULSAR_CPP" -DLINK_STATIC=ON
+cmake --build build --config Release
+cmake --install build
+py setup.py bdist_wheel
+py -m pip install ./dist/pulsar_client-*.whl
+```
+
+Since the Python client links to Boost.Python dynamically, you have to copy the dll (e.g. `boost_python310-vc142-mt-x64-1_80.dll`) into the system path (the `PATH` environment variable). If the `-DLINK_STATIC=ON` option is not specified, you have to copy the `pulsar.dll` into the system path as well.
+
+### Linux or macOS
+
+Assuming the Pulsar C++ client and Boost.Python have been installed under the system path.
+
 ```bash
 cmake -B build
 cmake --build build -j8
-cp build/_pulsar.so .
+cmake --install build
 ./setup.py bdist_wheel
 pip3 install dist/pulsar_client-*.whl --force-reinstall
-rm _pulsar.so
 ```
 
 > **NOTE**
 >
 > 1. Here a separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` is under the project directory.
 > 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before.
 
+## Running examples
+
 You can run `python3 -c 'import pulsar'` to see whether the wheel has been installed successfully. If it failed, check whether dependencies (e.g. `libpulsar.so`) are in the system path. If not, make sure the dependencies are in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS).
 
 Then you can run examples as a simple end-to-end test.
diff --git a/setup.py b/setup.py
@@ -21,6 +21,7 @@
 from setuptools import setup
 from distutils.core import Extension
 from os import environ, path
+import platform
 
 from distutils.command import build_ext
 
@@ -58,7 +59,13 @@ def build_extension(self, ext):
         except OSError as e:
             if e.errno != 17:  # already exists
                 raise
-        shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name))
+        if 'Windows' in platform.platform():
+            shutil.copyfile('_pulsar.pyd', self.get_ext_fullpath(ext.name))
+        else:
+            try:
+                shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name))
+            except FileNotFoundError:
+                shutil.copyfile('lib_pulsar.so', self.get_ext_fullpath(ext.name))
 
 
 # Core Client dependencies
diff --git a/vcpkg b/vcpkg
@@ -0,0 +1 @@
+Subproject commit 253704407ae68efa37bf8f5b59b3e06dd40d3d3f
diff --git a/vcpkg-3.10.json b/vcpkg-3.10.json
@@ -0,0 +1,12 @@
+{
+  "name": "pulsar-python",
+  "version": "3.0.0",
+  "description": "Pulsar Python SDK (Python 3.10)",
+  "dependencies": [
+     {
+       "name": "boost-python",
+       "version>=": "1.79.0"
+     }
+  ],
+  "builtin-baseline": "c266859544a3cdcfd952d218039c55a268863740"
+}
diff --git a/vcpkg-3.7.json b/vcpkg-3.7.json
@@ -0,0 +1,18 @@
+{
+  "name": "pulsar-python",
+  "version": "3.0.0",
+  "description": "Pulsar Python SDK (Python 3.7)",
+  "dependencies": [
+     {
+       "name": "boost-python",
+       "version>=": "1.76.0"
+     }
+  ],
+  "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379",
+  "overrides": [
+     {
+       "name": "python3",
+       "version": "3.7.3"
+     }
+  ]
+}
diff --git a/vcpkg-3.8.json b/vcpkg-3.8.json
@@ -0,0 +1,18 @@
+{
+  "name": "pulsar-python",
+  "version": "3.0.0",
+  "description": "Pulsar Python SDK (Python 3.8)",
+  "dependencies": [
+     {
+       "name": "boost-python",
+       "version>=": "1.76.0"
+     }
+  ],
+  "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379",
+  "overrides": [
+     {
+       "name": "python3",
+       "version": "3.8.3"
+     }
+  ]
+}
diff --git a/vcpkg-3.9.json b/vcpkg-3.9.json
@@ -0,0 +1,12 @@
+{
+  "name": "pulsar-python",
+  "version": "3.0.0",
+  "description": "Pulsar Python SDK (Python 3.9)",
+  "dependencies": [
+     {
+       "name": "boost-python",
+       "version>=": "1.76.0"
+     }
+  ],
+  "builtin-baseline": "35312384e7701760ed7855961eff41a63f9cc379"
+}
diff --git a/vcpkg.json b/vcpkg.json
@@ -0,0 +1,6 @@
+{
+  "name": "pulsar-python",
+  "version": "3.0.0",
+  "description": "Pulsar Python SDK",
+  "dependencies": ["boost-python"]
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[submodule "vcpkg"]`
	`2`	`+ path = vcpkg`
	`3`	`+ url = https://github.com/microsoft/vcpkg.git`