From e8f20f6ebd2f8c56d4058fe6b52765a0e41058cf Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 5 Oct 2020 19:25:45 +0200 Subject: [PATCH 01/10] remove topic conf, deprecated --- pkg/rdkafka/RdKafkaContext.php | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/rdkafka/RdKafkaContext.php b/pkg/rdkafka/RdKafkaContext.php index cc4915062..362037dd2 100644 --- a/pkg/rdkafka/RdKafkaContext.php +++ b/pkg/rdkafka/RdKafkaContext.php @@ -184,20 +184,18 @@ public static function getLibrdKafkaVersion(): string private function getConf(): Conf { if (null === $this->conf) { - $topicConf = new TopicConf(); + $this->conf = new Conf(); if (isset($this->config['topic']) && is_array($this->config['topic'])) { foreach ($this->config['topic'] as $key => $value) { - $topicConf->set($key, $value); + $this->conf->set($key, $value); } } if (isset($this->config['partitioner'])) { - $topicConf->setPartitioner($this->config['partitioner']); + $this->conf->set('partitioner', $this->config['partitioner']); } - $this->conf = new Conf(); - if (isset($this->config['global']) && is_array($this->config['global'])) { foreach ($this->config['global'] as $key => $value) { $this->conf->set($key, $value); @@ -219,8 +217,6 @@ private function getConf(): Conf if (isset($this->config['stats_cb'])) { $this->conf->setStatsCb($this->config['stats_cb']); } - - $this->conf->setDefaultTopicConf($topicConf); } return $this->conf; From c02b57dc243142db7e5f837132b28c325d389dc9 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 5 Oct 2020 19:46:40 +0200 Subject: [PATCH 02/10] remove unused import --- pkg/rdkafka/RdKafkaContext.php | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/rdkafka/RdKafkaContext.php b/pkg/rdkafka/RdKafkaContext.php index 362037dd2..9930b56ee 100644 --- a/pkg/rdkafka/RdKafkaContext.php +++ b/pkg/rdkafka/RdKafkaContext.php @@ -19,7 +19,6 @@ use RdKafka\Conf; use RdKafka\KafkaConsumer; use RdKafka\Producer as VendorProducer; -use RdKafka\TopicConf; class RdKafkaContext implements Context { From b1a0bbc1b2cac7d9a9dff5402a4c91a67f1b3456 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 5 Oct 2020 21:14:06 +0200 Subject: [PATCH 03/10] test improvements --- .../Tests/Spec/RdKafkaSendToAndReceiveFromTopicTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/rdkafka/Tests/Spec/RdKafkaSendToAndReceiveFromTopicTest.php b/pkg/rdkafka/Tests/Spec/RdKafkaSendToAndReceiveFromTopicTest.php index c755b8740..d756f23e2 100644 --- a/pkg/rdkafka/Tests/Spec/RdKafkaSendToAndReceiveFromTopicTest.php +++ b/pkg/rdkafka/Tests/Spec/RdKafkaSendToAndReceiveFromTopicTest.php @@ -30,7 +30,8 @@ public function test() $context->createProducer()->send($topic, $context->createMessage($expectedBody)); - $message = $consumer->receive(10000); // 10 sec + // Initial balancing can take some time, so we want to make sure the timeout is high enough + $message = $consumer->receive(15000); // 15 sec $this->assertInstanceOf(Message::class, $message); $consumer->acknowledge($message); @@ -47,7 +48,7 @@ protected function createContext() 'enable.auto.commit' => 'false', ], 'topic' => [ - 'auto.offset.reset' => 'beginning', + 'auto.offset.reset' => 'earliest', ], ]; From 093cddfc4a458e5b465f9d9e6a16a30926c28858 Mon Sep 17 00:00:00 2001 From: nick Date: Thu, 8 Oct 2020 21:35:32 +0200 Subject: [PATCH 04/10] add exception for unsupported librdkafka version --- docker/Dockerfile | 2 +- pkg/rdkafka/RdKafkaConnectionFactory.php | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ad483b3d4..f41c22953 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ RUN set -x && \ mkdir -p $HOME/librdkafka && \ cd $HOME/librdkafka && \ git clone https://github.com/edenhill/librdkafka.git . && \ - git checkout v0.11.6 && \ + git checkout v1.0.0 && \ ./configure && make && make install && \ pecl install rdkafka && \ echo "extension=rdkafka.so" > /etc/php/7.2/cli/conf.d/10-rdkafka.ini && \ diff --git a/pkg/rdkafka/RdKafkaConnectionFactory.php b/pkg/rdkafka/RdKafkaConnectionFactory.php index a55c5c617..24d60c9b6 100644 --- a/pkg/rdkafka/RdKafkaConnectionFactory.php +++ b/pkg/rdkafka/RdKafkaConnectionFactory.php @@ -39,6 +39,10 @@ class RdKafkaConnectionFactory implements ConnectionFactory */ public function __construct($config = 'kafka:') { + if (version_compare(RdKafkaContext::getLibrdKafkaVersion(), '1.0.0', '<')) { + throw new \RuntimeException('You must install librdkafka:1.0.0 or higher'); + } + if (empty($config) || 'kafka:' === $config) { $config = []; } elseif (is_string($config)) { From 2a1ca16dda7bdac39c5ae8c48a3ca23e43991b7d Mon Sep 17 00:00:00 2001 From: nick Date: Thu, 8 Oct 2020 21:50:26 +0200 Subject: [PATCH 05/10] bump php-amqp --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f41c22953..91af4ed05 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN set -x && \ apt-get install -y --no-install-recommends --no-install-suggests php-dev librabbitmq-dev make && \ mkdir -p $HOME/php-amqp && \ cd $HOME/php-amqp && \ - git clone https://github.com/pdezwart/php-amqp.git . && git checkout v1.9.3 && \ + git clone https://github.com/pdezwart/php-amqp.git . && git checkout v1.10.2 && \ phpize --clean && phpize && ./configure && make install ## librdkafka From c8e6c561bff28eaf9b057e479d254cf4b8e80c5e Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 9 Oct 2020 10:26:15 +0200 Subject: [PATCH 06/10] make sure we use the latest image --- bin/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/test.sh b/bin/test.sh index f392e616b..12c9734bd 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -3,4 +3,5 @@ set -x set -e +docker pull enqueue/dev docker-compose run --workdir="/mqdev" --rm dev ./docker/bin/test.sh $@ From c16bdb4e10aa1b0849f16e7d192933a98d4881cd Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 9 Oct 2020 10:50:20 +0200 Subject: [PATCH 07/10] install librdkafka locally --- .travis.yml | 1 + bin/install-librdkafka.sh | 6 ++++++ bin/test.sh | 1 - 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 bin/install-librdkafka.sh diff --git a/.travis.yml b/.travis.yml index 33535e57e..978a96644 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ install: - echo "memory_limit=2048M" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini - php ./bin/fix-symfony-version.php "$SYMFONY_VERSION" - composer install + - ./bin/install-librdkafka.sh - if [ "$PREPARE_CONTAINER" = true ]; then docker --version; fi - if [ "$PREPARE_CONTAINER" = true ]; then docker-compose --version; fi - if [ "$PREPARE_CONTAINER" = true ]; then bin/dev -b; fi diff --git a/bin/install-librdkafka.sh b/bin/install-librdkafka.sh new file mode 100755 index 000000000..cb696a9e7 --- /dev/null +++ b/bin/install-librdkafka.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +git clone --depth 1 --branch v1.0.0 https://github.com/edenhill/librdkafka.git +cd librdkafka +./configure +make +sudo make install diff --git a/bin/test.sh b/bin/test.sh index 12c9734bd..f392e616b 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -3,5 +3,4 @@ set -x set -e -docker pull enqueue/dev docker-compose run --workdir="/mqdev" --rm dev ./docker/bin/test.sh $@ From 16c537cd3c89e8346c974d47c79612ba9e2eec07 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 9 Oct 2020 11:08:29 +0200 Subject: [PATCH 08/10] try to fix lib problem --- bin/install-librdkafka.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/install-librdkafka.sh b/bin/install-librdkafka.sh index cb696a9e7..4248975d4 100755 --- a/bin/install-librdkafka.sh +++ b/bin/install-librdkafka.sh @@ -4,3 +4,5 @@ cd librdkafka ./configure make sudo make install +sudo ldconfig + From 07c0fb725fe612bee0c1835d528167ade0216dcc Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 9 Oct 2020 11:34:19 +0200 Subject: [PATCH 09/10] fix stub --- .travis.yml | 2 +- bin/install-librdkafka.sh | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100755 bin/install-librdkafka.sh diff --git a/.travis.yml b/.travis.yml index 978a96644..7c3c68762 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ install: - echo "memory_limit=2048M" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini - php ./bin/fix-symfony-version.php "$SYMFONY_VERSION" - composer install - - ./bin/install-librdkafka.sh + - sed -i 's/525568/16777471/' vendor/kwn/php-rdkafka-stubs/stubs/constants.php - if [ "$PREPARE_CONTAINER" = true ]; then docker --version; fi - if [ "$PREPARE_CONTAINER" = true ]; then docker-compose --version; fi - if [ "$PREPARE_CONTAINER" = true ]; then bin/dev -b; fi diff --git a/bin/install-librdkafka.sh b/bin/install-librdkafka.sh deleted file mode 100755 index 4248975d4..000000000 --- a/bin/install-librdkafka.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -git clone --depth 1 --branch v1.0.0 https://github.com/edenhill/librdkafka.git -cd librdkafka -./configure -make -sudo make install -sudo ldconfig - From a71fd0995373cdc1ea46b98f23ee857355f79815 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 9 Oct 2020 11:56:41 +0200 Subject: [PATCH 10/10] make check more safe --- pkg/rdkafka/RdKafkaProducer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/rdkafka/RdKafkaProducer.php b/pkg/rdkafka/RdKafkaProducer.php index 2b987bb96..db21b9311 100644 --- a/pkg/rdkafka/RdKafkaProducer.php +++ b/pkg/rdkafka/RdKafkaProducer.php @@ -47,7 +47,8 @@ public function send(Destination $destination, Message $message): void // Headers in payload are maintained for backwards compatibility with apps that might run on lower phprdkafka version if (method_exists($topic, 'producev')) { // Phprdkafka <= 3.1.0 will fail calling `producev` on librdkafka >= 1.0.0 causing segfault - if (version_compare(RdKafkaContext::getLibrdKafkaVersion(), '1.0.0', '>=') + // Since we are forcing to use at least librdkafka:1.0.0, no need to check the lib version anymore + if (false !== phpversion('rdkafka') && version_compare(phpversion('rdkafka'), '3.1.0', '<=')) { trigger_error( 'Phprdkafka <= 3.1.0 is incompatible with librdkafka 1.0.0 when calling `producev`. '.