Skip to content

Commit f09a67e

Browse files
felixschlegelfelixschlegel
authored
librdkafka as submodule (swift-server#50)
* Build librdkafka as submodule Modifications: * add `librdkafka` submodule to project * create a hardcoded `config.h` file that contains information on how `librdkafka` should be compiled (normally this is done through a the `librdkafka/configure.sh` script, though we cannot invoke this at buildtime with SPM) * add package dependencies to `zstd` and `swift-nio-ssl` * build `librdkafka` package with SPM instead of relying on a `.systemLibary` target * create bridge between `<openssl/*>` headers and `swift-nio-ssl` with `custom` include folder * update Docker to install zlib * remove librdkafka from Docker * create hard-coded config.h file that is used to compile librdkafka * automatically update submodule from Docker * Update README to mention new submodule * Empty dummy folder was not tracked by git * * Create OS+Arch specific config.h files for building librdkafka * Include NOTICE about using librdkafka * Make build OS depend on target, not on host machine * Symbolic link for rdkafka.h * Depend on OpenSSL system library * Update librdkafka to v2.1.1 * Merge multiple config.h files into one Modifications: * merge `config.h` files used to compile `librdkafka` into single `config.h` file by making use of compiler directives * Run SwiftFormat * Exclude everything in the Crdkafka folder from soundess check * Fix failing CI builds Modifications: * remove `git submodule init` from `Dockerfile` as Jenkins will take care of submodules and this crashes our builds * adjust Docker build/test targets to Apple standards * config.h: "#elif __xx" -> "#elif defined(__xx"
1 parent 6e24f0e commit f09a67e

18 files changed

+306
-102
lines changed

‎.dockerignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
11
.build
2-
.git

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "Sources/Crdkafka/librdkafka"]
2+
path = Sources/Crdkafka/librdkafka
3+
url = https://github.com/confluentinc/librdkafka

‎NOTICE.txt

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
2+
The Swift Kafka Client Project
3+
====================================
4+
5+
Please visit the Swift Kafka Client web site for more information:
6+
7+
* https://github.com/swift-server/swift-kafka-gsoc
8+
9+
Copyright 2023 The Swift Kafka Client Project
10+
11+
The Swift Kafka Client Project licenses this file to you under the Apache License,
12+
version 2.0 (the "License"); you may not use this file except in compliance
13+
with the License. You may obtain a copy of the License at:
14+
15+
https://www.apache.org/licenses/LICENSE-2.0
16+
17+
Unless required by applicable law or agreed to in writing, software
18+
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
19+
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
20+
License for the specific language governing permissions and limitations
21+
under the License.
22+
23+
Also, please refer to each LICENSE.<component>.txt file, which is located in
24+
the 'license' directory of the distribution file, for the license terms of the
25+
components that this product depends on.
26+
27+
-------------------------------------------------------------------------------
28+
29+
This product is based on librdkafka - the Apache Kafka C driver library.
30+
31+
* LICENSE (BSD-2):
32+
* https://opensource.org/license/BSD-2-Clause
33+
* HOMEPAGE:
34+
* https://github.com/confluentinc/librdkafka
35+
36+
---
37+
38+
This product uses zstd.
39+
40+
* LICENSE (BSD-3):
41+
* https://opensource.org/license/BSD-3-Clause
42+
* HOMEPAGE:
43+
* https://github.com/facebook/zstd

‎Package.swift

+38-4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,17 @@
1515

1616
import PackageDescription
1717

18+
let rdkafkaExclude = [
19+
"./librdkafka/src/CMakeLists.txt",
20+
"./librdkafka/src/Makefile",
21+
"./librdkafka/src/generate_proto.sh",
22+
"./librdkafka/src/librdkafka_cgrp_synch.png",
23+
"./librdkafka/src/statistics_schema.json",
24+
"./librdkafka/src/rdkafka_sasl_win32.c",
25+
"./librdkafka/src/rdwin32.h",
26+
"./librdkafka/src/win32_config.h",
27+
]
28+
1829
let package = Package(
1930
name: "swift-kafka-gsoc",
2031
platforms: [
@@ -32,8 +43,31 @@ let package = Package(
3243
dependencies: [
3344
.package(url: "https://github.com/apple/swift-nio.git", from: "2.43.1"),
3445
.package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
46+
// The zstd Swift package produces warnings that we cannot resolve:
47+
// https://github.com/facebook/zstd/issues/3328
48+
.package(url: "https://github.com/facebook/zstd.git", from: "1.5.0"),
3549
],
3650
targets: [
51+
.target(
52+
name: "Crdkafka",
53+
dependencies: [
54+
"COpenSSL",
55+
.product(name: "libzstd", package: "zstd"),
56+
],
57+
exclude: rdkafkaExclude,
58+
sources: ["./librdkafka/src/"],
59+
publicHeadersPath: "./include",
60+
cSettings: [
61+
// dummy folder, because config.h is included as "../config.h" in librdkafka
62+
.headerSearchPath("./custom/config/dummy"),
63+
.headerSearchPath("./librdkafka/src"),
64+
],
65+
linkerSettings: [
66+
.linkedLibrary("curl"),
67+
.linkedLibrary("sasl2"),
68+
.linkedLibrary("z"), // zlib
69+
]
70+
),
3771
.target(
3872
name: "SwiftKafka",
3973
dependencies: [
@@ -43,11 +77,11 @@ let package = Package(
4377
]
4478
),
4579
.systemLibrary(
46-
name: "Crdkafka",
47-
pkgConfig: "rdkafka",
80+
name: "COpenSSL",
81+
pkgConfig: "openssl",
4882
providers: [
49-
.brew(["librdkafka"]),
50-
.apt(["librdkafka-dev"]),
83+
.brew(["libressl"]),
84+
.apt(["libssl-dev"]),
5185
]
5286
),
5387
.testTarget(

‎README.md

+15-60
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,8 @@
11
# 🚧WIP🚧: SwiftKafka
22

3-
SwiftKafka is a Swift Package in development that provides a convenient way to communicate with [Apache Kafka](https://kafka.apache.org) servers. The main goal was to create an API that leverages [Swift's new concurrency features](https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html). Under the hood, this package uses the [`librdkafka`](github.com/edenhill/librdkafka) C library.
3+
SwiftKafka is a Swift Package in development that provides a convenient way to communicate with [Apache Kafka](https://kafka.apache.org) servers. The main goal was to create an API that leverages [Swift's new concurrency features](https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html). Under the hood, this package uses the [`librdkafka`](https://github.com/confluentinc/librdkafka) C library.
44

5-
## Getting Started
6-
7-
### Installing Dependencies
8-
9-
Please make sure to have the [`librdkafka`](https://github.com/edenhill/librdkafka) library installed before building.
10-
11-
#### macOS
12-
13-
```bash
14-
brew install librdkafka
15-
```
16-
17-
#### Linux
18-
19-
The default `apt-get` package for the library is outdated. Therefore, we recommend installing [`librdkafka`](https://github.com/edenhill/librdkafka) from the [official Confluent package repository](https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software).
20-
21-
```bash
22-
wget -qO - http://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add -
23-
sudo add-apt-repository "deb http://packages.confluent.io/deb/ $(lsb_release -cs) main"
24-
apt-get install librdkafka-dev
25-
```
26-
27-
### Building the Project
28-
29-
#### macOS
30-
31-
We rely on the `openssl` package, which is *keg-only*. This means that Swift Package Manager will not find it by default. To build, we have to explicitly make the `PKG_CONFIG_PATH` environment variable point to the location of our `openssl` installation.
32-
33-
##### Building from Command Line
34-
35-
```bash
36-
env PKG_CONFIG_PATH=$(brew --prefix)/opt/[email protected]/lib/pkgconfig swift build
37-
```
38-
39-
##### Opening & Building the Package in Xcode
40-
41-
> **Note**
42-
>
43-
> Please make sure that Xcode is **not** running already. Otherwise, Xcode will not open the project in the specified environment.
44-
45-
```bash
46-
env PKG_CONFIG_PATH=$(brew --prefix)/opt/[email protected]/lib/pkgconfig xed .
47-
```
48-
49-
#### Linux
50-
51-
```bash
52-
swift build
53-
```
54-
55-
### Docker
56-
57-
We also provide a Docker environment for this package. This will automatically start a local Kafka server and run the package tests.
58-
59-
```bash
60-
docker-compose -f docker/docker-compose.yaml run swift-kafka-gsoc
61-
```
62-
63-
## Overview
5+
## Usage
646

657
### Producer API
668

@@ -166,3 +108,16 @@ for await messageResult in consumer.messages {
166108
}
167109
}
168110
```
111+
112+
## librdkafka
113+
114+
The Package depends on [the librdkafka library](https://github.com/confluentinc/librdkafka), which is included as a git submodule.
115+
It has source files that are excluded in `Package.swift`.
116+
117+
## Development Setup
118+
119+
We provide a Docker environment for this package. This will automatically start a local Kafka server and run the package tests.
120+
121+
```bash
122+
docker-compose -f docker/docker-compose.yaml run swift-kafka-gsoc
123+
```

‎Sources/COpenSSL/module.modulemap

Whitespace-only changes.

‎Sources/Crdkafka/custom/config/config.h

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#ifndef _CONFIG_H_
2+
#define _CONFIG_H_
3+
4+
// distro
5+
#ifdef __APPLE__
6+
#define SOLIB_EXT ".dylib"
7+
#elif defined(__linux__)
8+
#define SOLIB_EXT ".so"
9+
#endif
10+
11+
#ifdef __x86_64__
12+
#define ARCH "x86_64"
13+
#elif defined(__arm64__) || defined(__arm__) || defined(__aarch64__)
14+
#define ARCH "arm64"
15+
#endif
16+
17+
#define CPU "generic"
18+
#define WITHOUT_OPTIMIZATION 0
19+
20+
#define ENABLE_ZLIB 1
21+
#define ENABLE_ZSTD 1
22+
#define ENABLE_SSL 1
23+
#define ENABLE_GSSAPI 1
24+
#define ENABLE_CURL 1
25+
#define ENABLE_DEVEL 0
26+
#define ENABLE_VALGRIND 0
27+
#define ENABLE_REFCNT_DEBUG 0
28+
#define ENABLE_LZ4_EXT 1
29+
#define ENABLE_LZ4_EXT 1
30+
#define ENABLE_REGEX_EXT 1
31+
#define ENABLE_C11THREADS "try"
32+
#define ENABLE_ZLIB 1
33+
#define ENABLE_ZSTD 1
34+
#define ENABLE_SSL 1
35+
#define ENABLE_GSSAPI 1
36+
#define ENABLE_CURL 1
37+
#define ENABLE_LZ4_EXT 1
38+
#define WITH_STATIC_LINKING 1
39+
#define MKL_APP_NAME "librdkafka"
40+
#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library"
41+
42+
#ifdef __APPLE__
43+
#define WITH_STRIP 0
44+
#define ENABLE_SYSLOG 1
45+
#endif
46+
47+
#ifdef __APPLE__
48+
// gcc
49+
#define WITH_GCC 1
50+
// gxx
51+
#define WITH_GXX 1
52+
#elif defined(__linux__)
53+
// ccenv
54+
#define WITH_CC 1
55+
// cxxenv
56+
#define WITH_CXX 1
57+
#endif
58+
59+
// pkgconfig
60+
#if !(defined(__linux__) && (defined(__arm64__) || defined(__arm__) || defined(__aarch64__)))
61+
#define WITH_PKGCONFIG 1
62+
#endif
63+
64+
#ifdef __linux__
65+
// install
66+
#define WITH_INSTALL 1
67+
// gnulib
68+
#define WITH_GNULD 1
69+
#endif
70+
71+
#ifdef __APPLE__
72+
// osxlib
73+
#define WITH_OSXLD 1
74+
// syslog
75+
#define WITH_SYSLOG 1
76+
#endif
77+
78+
// crc32chw
79+
#ifdef __x86_64__
80+
#define WITH_CRC32C_HW 1
81+
#endif
82+
83+
#ifdef __APPLE__
84+
// rand_r
85+
#define HAVE_RAND_R 1
86+
// strlcpy
87+
#define HAVE_STRLCPY 1
88+
// strcasestr
89+
#define HAVE_STRCASESTR 1
90+
// pthread_setname_darwin
91+
#define HAVE_PTHREAD_SETNAME_DARWIN 1
92+
// getrusage
93+
#define HAVE_GETRUSAGE 1
94+
#endif
95+
96+
#ifdef __linux__
97+
// pthread_setname_gnu
98+
#define HAVE_PTHREAD_SETNAME_GNU 1
99+
#endif
100+
101+
// Common identifiers
102+
// PIC
103+
#define HAVE_PIC 1
104+
// __atomic_32
105+
#define HAVE_ATOMICS_32 1
106+
// __atomic_32
107+
#define HAVE_ATOMICS_32_ATOMIC 1
108+
// atomic_32
109+
#define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
110+
// __atomic_64
111+
#define HAVE_ATOMICS_64 1
112+
// __atomic_64
113+
#define HAVE_ATOMICS_64_ATOMIC 1
114+
// atomic_64
115+
#define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
116+
// atomic_64
117+
#define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
118+
// parseversion
119+
#define RDKAFKA_VERSION_STR "2.1.0"
120+
// parseversion
121+
#define MKL_APP_VERSION "2.1.0"
122+
// libdl
123+
#define WITH_LIBDL 1
124+
// WITH_PLUGINS
125+
#define WITH_PLUGINS 1
126+
// zlib
127+
#define WITH_ZLIB 1
128+
// libssl
129+
#define WITH_SSL 1
130+
// libcrypto
131+
#define OPENSSL_SUPPRESS_DEPRECATED "OPENSSL_SUPPRESS_DEPRECATED"
132+
// libsasl2
133+
#define WITH_SASL_CYRUS 1
134+
// libzstd
135+
#define WITH_ZSTD 1
136+
// libcurl
137+
#define WITH_CURL 1
138+
// WITH_HDRHISTOGRAM
139+
#define WITH_HDRHISTOGRAM 1
140+
// WITH_SNAPPY
141+
#define WITH_SNAPPY 1
142+
// WITH_SOCKEM
143+
#define WITH_SOCKEM 1
144+
// WITH_SASL_SCRAM
145+
#define WITH_SASL_SCRAM 1
146+
// WITH_SASL_OAUTHBEARER
147+
#define WITH_SASL_OAUTHBEARER 1
148+
// WITH_OAUTHBEARER_OIDC
149+
#define WITH_OAUTHBEARER_OIDC 1
150+
// regex
151+
#define HAVE_REGEX 1
152+
// strndup
153+
#define HAVE_STRNDUP 1
154+
// strerror_r
155+
#define HAVE_STRERROR_R 1
156+
157+
// BUILT_WITH
158+
#ifdef __APPLE__
159+
#ifdef __x86_64__
160+
#define BUILT_WITH "STATIC_LINKING GCC GXX PKGCONFIG OSXLD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD CURL HDRHISTOGRAM SYSLOG SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER OAUTHBEARER_OIDC CRC32C_HW"
161+
#elif defined(__arm64__) || defined(__arm__) || (__aarch64__)
162+
#define BUILT_WITH "STATIC_LINKING GCC GXX PKGCONFIG OSXLD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD CURL HDRHISTOGRAM SYSLOG SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER OAUTHBEARER_OIDC"
163+
#endif
164+
#elif defined(__linux__)
165+
#ifdef __x86_64__
166+
#define BUILT_WITH "STATIC_LINKING CC CXX PKGCONFIG INSTALL GNULD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD CURL HDRHISTOGRAM SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER OAUTHBEARER_OIDC CRC32C_HW"
167+
#elif defined(__arm64__) || defined(__arm__) || defined(__aarch64__)
168+
#define BUILT_WITH "STATIC_LINKING CC CXX INSTALL GNULD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD CURL HDRHISTOGRAM SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER OAUTHBEARER_OIDC"
169+
#endif
170+
#endif
171+
172+
#endif /* _CONFIG_H_ */

‎Sources/Crdkafka/custom/config/dummy/empty

Whitespace-only changes.

‎Sources/Crdkafka/include/rdkafka.h

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../librdkafka/src/rdkafka.h

‎Sources/Crdkafka/librdkafka

Submodule librdkafka added at c282ba2

‎Sources/Crdkafka/module.modulemap

-5
This file was deleted.

0 commit comments

Comments
 (0)