feat: node 22 support, latest rdkafka

Author: AVVS

.commitlintrc.js renamed to .commitlintrc.cjs

@@ -18,3 +18,4 @@ package-lock.json
 pnpm-lock.yaml
 .env
 *.tgz
+.idea

.gitignore

@@ -1,4 +1 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
 "`npm x -- mdep bin commitlint`" --edit $1

.husky/commit-msg

@@ -8,19 +8,18 @@ agent:
 global_job_config:
   prologue:
     commands:
-      - sem-version node 20
-      - curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=8.10.2 sh -
-      - source /home/semaphore/.bashrc
-      - pnpm config set store-dir=~/.pnpm-store
+      - sem-version node --lts
+      - corepack enable
+      - corepack install --global [email protected]
       - checkout
       - git submodule init
       - git submodule update
       - cache restore node-$(checksum pnpm-lock.yaml)
       - pnpm i --frozen-lockfile --prefer-offline --ignore-scripts
-      - cache store node-$(checksum pnpm-lock.yaml) $(pnpm config get store-dir)
+      - cache store node-$(checksum pnpm-lock.yaml) $(pnpm store path)
   env_vars:
   - name: BUILD_LIBRDKAFKA
-    value: '0'
+    value: '1'
 
 blocks:
   - name: verify
@@ -45,7 +44,7 @@ blocks:
       - name: pre-build & publish binaries
         matrix:
         - env_var: NODE_VER
-          values: ["18", "20", "21"]
+          values: ["22", "23"]
         - env_var: platform
           values: ["-rdkafka", "-debian-rdkafka"]
         commands:

.semaphore/semaphore.yml

@@ -13,6 +13,7 @@ so if you feel something is missing feel free to send a pull request.
   * [Contributor Agreement](#contributor-agreement)
 
 [How Can I Contribute?](#how-can-i-contribute)
+  * [Setting up the repository](#setting-up-the-repository)
   * [Reporting Bugs](#reporting-bugs)
   * [Suggesting Enhancements](#suggesting-enhancements)
   * [Pull Requests](#pull-requests)
@@ -37,6 +38,14 @@ Not currently required.
 
 ## How can I contribute?
 
+### Setting up the repository
+
+To set up the library locally, do the following:
+
+1) Clone this repository.
+2) Install librdkafka with `git submodule update --init --recursive`
+3) Install the dependencies `npm install`
+
 ### Reporting Bugs
 
 Please use __Github Issues__ to report bugs. When filling out an issue report,
@@ -164,7 +173,7 @@ I began using Visual Studio code to develop on `node-rdkafka`. If you use it you
             ],
             "defines": [],
             "macFrameworkPath": [
-                "/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks"
+                "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks"
             ],
             "compilerPath": "/usr/bin/clang",
             "cStandard": "c11",

CONTRIBUTING.md

@@ -64,7 +64,7 @@ check: node_modules/.dirstamp
 	@$(NODE) util/test-compile.js
 
 e2e: $(E2E_TESTS)
-	./node_modules/.bin/mocha --exit --timeout 120000 $(TEST_REPORTER) $(E2E_TESTS) $(TEST_OUTPUT)
+	./node_modules/.bin/mocha --exit --bail --timeout 30000 $(TEST_REPORTER) $(E2E_TESTS) $(TEST_OUTPUT)
 
 define release
 	NEXT_VERSION=$(shell node -pe 'require("semver").inc("$(VERSION)", "$(1)")')

Makefile

@@ -17,7 +17,7 @@ I am looking for *your* help to make this project even better! If you're interes
 
 The `node-rdkafka` library is a high-performance NodeJS client for [Apache Kafka](http://kafka.apache.org/) that wraps the native  [librdkafka](https://github.com/edenhill/librdkafka) library.  All the complexity of balancing writes across partitions and managing (possibly ever-changing) brokers should be encapsulated in the library.
 
-__This library currently uses `librdkafka` version `2.3.0`.__
+__This library currently uses `librdkafka` version `2.6.1`.__
 
 ## Reference Docs
 
@@ -34,7 +34,7 @@ Play nice; Play fair.
 ## Requirements
 
 * Apache Kafka >=0.9
-* Node.js >=4
+* Node.js >=16
 * Linux/Mac
 * Windows?! See below
 * OpenSSL
@@ -60,7 +60,7 @@ Using Alpine Linux? Check out the [docs](https://github.com/Blizzard/node-rdkafk
 
 ### Windows
 
-Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.2.3.0.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
+Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.2.6.1.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
 
 Requirements:
  * [node-gyp for Windows](https://github.com/nodejs/node-gyp#on-windows)
@@ -97,7 +97,7 @@ const Kafka = require('node-rdkafka');
 
 ## Configuration
 
-You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.3.0/CONFIGURATION.md)
+You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.6.1/CONFIGURATION.md)
 
 Configuration keys that have the suffix `_cb` are designated as callbacks. Some
 of these keys are informational and you can choose to opt-in (for example, `dr_cb`). Others are callbacks designed to
@@ -132,7 +132,7 @@ You can also get the version of `librdkafka`
 const Kafka = require('node-rdkafka');
 console.log(Kafka.librdkafkaVersion);
 
-// #=> 2.3.0
+// #=> 2.6.1
 ```
 
 ## Sending Messages
@@ -145,7 +145,7 @@ const producer = new Kafka.Producer({
 });
 ```
 
-A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.3.0/CONFIGURATION.md) file described previously.
+A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.6.1/CONFIGURATION.md) file described previously.
 
 The following example illustrates a list with several `librdkafka` options set.
 
@@ -511,6 +511,7 @@ The following table lists events for this API.
 |`data` | When using the Standard API consumed messages are emitted in this event. |
 |`partition.eof` | When using Standard API and the configuration option `enable.partition.eof` is set, `partition.eof` events are emitted in this event. The event contains `topic`, `partition` and `offset` properties. |
 |`warning` | The event is emitted in case of `UNKNOWN_TOPIC_OR_PART` or `TOPIC_AUTHORIZATION_FAILED` errors when consuming in *Flowing mode*. Since the consumer will continue working if the error is still happening, the warning event should reappear after the next metadata refresh. To control the metadata refresh rate set `topic.metadata.refresh.interval.ms` property. Once you resolve the error, you can manually call `getMetadata` to speed up consumer recovery. |
+|`rebalance` | The `rebalance` event is emitted when the consumer group is rebalanced. <br><br>This event is only emitted if the `rebalance_cb` configuration is set to a function or set to `true` |
 |`disconnected` | The `disconnected` event is emitted when the broker disconnects. <br><br>This event is only emitted when `.disconnect` is called. The wrapper will always try to reconnect otherwise. |
 |`ready` | The `ready` event is emitted when the `Consumer` is ready to read messages. |
 |`event` | The `event` event is emitted when `librdkafka` reports an event (if you opted in via the `event_cb` option).|

README.md

@@ -14,8 +14,8 @@ set -ex
 if [ x"$CI" = x"true" ]; then
   cp ~/.env.aws-s3-credentials .env
 fi
-env IMAGE_TAG=${NODE_VER}${platform} UID=${UID} PNPM_STORE="$(pnpm config get store-dir)" docker-compose up -d
-env UID=${UID} docker-compose exec -u ${UID} tester pnpm i --frozen-lockfile --prefer-offline --ignore-scripts
+env IMAGE_TAG=${NODE_VER}${platform} UID=${UID} PNPM_STORE="$(dirname `pnpm store path`)" docker-compose up -d
+env UID=${UID} docker-compose exec -u ${UID} tester pnpm i --frozen-lockfile --offline --ignore-scripts
 env UID=${UID} docker-compose exec -u ${UID} tester pnpm binary:build
 env UID=${UID} docker-compose exec -u ${UID} tester pnpm binary:package
 env UID=${UID} docker-compose exec -u ${UID} tester pnpm binary:test

ci/build-and-publish.sh

@@ -4,11 +4,11 @@ set -ex
 
 if git log --oneline -n 1 | grep -v "chore(release)" > /dev/null; then
   touch .env
-  env UID=${UID} PNPM_STORE=$(pnpm config get store-dir) docker-compose --profile tests up -d
-  docker-compose exec tester pnpm i
-  docker-compose exec tester pnpm binary:build
-  docker-compose exec tester pnpm test
-  docker-compose exec tester pnpm test:e2e
+  env UID=${UID} PNPM_STORE=$(dirname `pnpm store path`) docker-compose --profile tests up -d
+  env UID=${UID} docker-compose exec -u ${UID} tester pnpm i --frozen-lockfile --offline --ignore-scripts
+  env UID=${UID} docker-compose exec -u ${UID} tester pnpm binary:build
+  env UID=${UID} docker-compose exec -u ${UID} tester pnpm test
+  env UID=${UID} docker-compose exec -u ${UID} tester pnpm test:e2e
   pnpm semantic-release
 else
   echo "skipped commit: `git log --oneline -n 1`"

ci/semantic-release.sh

@@ -1,4 +1,4 @@
-// ====== Generated from librdkafka 2.3.0 file CONFIGURATION.md ======
+// ====== Generated from librdkafka 2.6.1 file CONFIGURATION.md ======
 // Code that generated this is a derivative work of the code from Nam Nguyen
 // https://gist.github.com/ntgn81/066c2c8ec5b4238f85d1e9168a04e3fb
 
@@ -620,12 +620,33 @@ export interface GlobalConfig {
     "client.rack"?: string;
 
     /**
-     * Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to `resolve_canonical_bootstrap_servers_only`, each entry will be resolved and expanded into a list of canonical names. NOTE: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname.
+     * The backoff time in milliseconds before retrying a protocol request, this is the first backoff time, and will be backed off exponentially until number of retries is exhausted, and it's capped by retry.backoff.max.ms.
+     *
+     * @default 100
+     */
+    "retry.backoff.ms"?: number;
+
+    /**
+     * The max backoff time in milliseconds before retrying a protocol request, this is the atmost backoff allowed for exponentially backed off requests.
+     *
+     * @default 1000
+     */
+    "retry.backoff.max.ms"?: number;
+
+    /**
+     * Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to `resolve_canonical_bootstrap_servers_only`, each entry will be resolved and expanded into a list of canonical names. **WARNING**: `resolve_canonical_bootstrap_servers_only` must only be used with `GSSAPI` (Kerberos) as `sasl.mechanism`, as it's the only purpose of this configuration value. **NOTE**: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname.
      *
      * @default use_all_dns_ips
      */
     "client.dns.lookup"?: 'use_all_dns_ips' | 'resolve_canonical_bootstrap_servers_only';
 
+    /**
+     * Whether to enable pushing of client metrics to the cluster, if the cluster has a client metrics subscription which matches this client
+     *
+     * @default true
+     */
+    "enable.metrics.push"?: boolean;
+
     /**
      * Enables or disables `event.*` emitting.
      *
@@ -703,20 +724,6 @@ export interface ProducerGlobalConfig extends GlobalConfig {
      */
     "retries"?: number;
 
-    /**
-     * The backoff time in milliseconds before retrying a protocol request, this is the first backoff time, and will be backed off exponentially until number of retries is exhausted, and it's capped by retry.backoff.max.ms.
-     *
-     * @default 100
-     */
-    "retry.backoff.ms"?: number;
-
-    /**
-     * The max backoff time in milliseconds before retrying a protocol request, this is the atmost backoff allowed for exponentially backed off requests.
-     *
-     * @default 1000
-     */
-    "retry.backoff.max.ms"?: number;
-
     /**
      * The threshold of outstanding not yet transmitted broker requests needed to backpressure the producer's message accumulator. If the number of not yet transmitted requests equals or exceeds this number, produce request creation that would have otherwise been triggered (for example, in accordance with linger.ms) will be delayed. A lower number yields larger and more effective batches. A higher value can improve latency when using compression on slow machines.
      *
@@ -810,12 +817,24 @@ export interface ConsumerGlobalConfig extends GlobalConfig {
     "heartbeat.interval.ms"?: number;
 
     /**
-     * Group protocol type. NOTE: Currently, the only supported group protocol type is `consumer`.
+     * Group protocol type for the `classic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`.
      *
      * @default consumer
      */
     "group.protocol.type"?: string;
 
+    /**
+     * Group protocol to use. Use `classic` for the original protocol and `consumer` for the new protocol introduced in KIP-848. Available protocols: classic or consumer. Default is `classic`, but will change to `consumer` in next releases.
+     *
+     * @default classic
+     */
+    "group.protocol"?: 'classic' | 'consumer';
+
+    /**
+     * Server side assignor to use. Keep it null to make server select a suitable assignor for the group. Available assignors: uniform or range. Default is null
+     */
+    "group.remote.assignor"?: string;
+
     /**
      * How often to query for the current client group coordinator. If the currently assigned coordinator is down the configured query interval will be divided by ten to more quickly recover in case of coordinator reassignment.
      *

config.d.ts

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   zookeeper:
     profiles: ["tests"]
@@ -27,13 +25,13 @@ services:
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_DEFAULT_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_NUM_NETWORK_THREADS: 4
-      KAFKA_NUM_IO_THREADS: 4
-      KAFKA_BACKGROUND_THREADS: 4
+      KAFKA_NUM_NETWORK_THREADS: 2
+      KAFKA_NUM_IO_THREADS: 2
+      KAFKA_BACKGROUND_THREADS: 2
       KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m"
 
   tester:
-    image: makeomatic/node:${IMAGE_TAG:-16-rdkafka-tester}
+    image: makeomatic/node:${IMAGE_TAG:-22-rdkafka-tester}
     volumes:
       - ${PWD}:/src
       - ${PWD}/ci/npmrc:/usr/local/etc/npmrc:ro
@@ -43,6 +41,7 @@ services:
     env_file:
       - .env
     environment:
+      - BUILD_LIBRDKAFKA=0
       - UV_THREADPOOL_SIZE=16
       - CI=true
       - UID=${UID:-1000}