Skip to content

Commit 268f3f5

Browse files
acogoluegnesacogoluegnes
authored
Merge pull request #1549 from rabbitmq/3-node-cluster-test
Use 3-node cluster in test suite
2 parents 68dcc73 + 46b245f commit 268f3f5

22 files changed

+384
-460
lines changed

Diff for: ‎.github/workflows/test-rabbitmq-alphas.yml

+12-12
Original file line numberDiff line numberDiff line change
@@ -43,19 +43,19 @@ jobs:
4343
run: make deps
4444
- name: Test with NIO
4545
run: |
46-
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
47-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
46+
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
47+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
48+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
49+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4850
-Dmaven.javadoc.skip=true \
49-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
5051
--no-transfer-progress
5152
- name: Test with blocking IO
5253
run: |
53-
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
54-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
55-
-Dmaven.javadoc.skip=true \
56-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
57-
--no-transfer-progress
58-
- name: Stop broker A
59-
run: docker stop rabbitmq && docker rm rabbitmq
60-
- name: Stop broker B
61-
run: docker stop hare && docker rm hare
54+
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
55+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
56+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
57+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
58+
-Dmaven.javadoc.skip=true \
59+
--no-transfer-progress
60+
- name: Stop cluster
61+
run: docker compose --file ci/cluster/docker-compose.yml down

Diff for: ‎.github/workflows/test-supported-java-versions-5.x.yml

+8-6
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,19 @@ jobs:
4545
run: |
4646
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
4747
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
48-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
48+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
49+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4950
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
5051
--no-transfer-progress \
5152
-Dnet.bytebuddy.experimental=true
5253
- name: Test with blocking IO
5354
run: |
5455
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
55-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
56-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
57-
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
58-
--no-transfer-progress \
59-
-Dnet.bytebuddy.experimental=true
56+
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
57+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
58+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
59+
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
60+
--no-transfer-progress \
61+
-Dnet.bytebuddy.experimental=true
6062
- name: Stop broker
6163
run: docker stop rabbitmq && docker rm rabbitmq

Diff for: ‎.github/workflows/test-supported-java-versions-main.yml

+8-6
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,19 @@ jobs:
4343
run: |
4444
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
4545
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
46-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
46+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
47+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4748
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
4849
--no-transfer-progress \
4950
-Dnet.bytebuddy.experimental=true
5051
- name: Test with blocking IO
5152
run: |
5253
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
53-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
54-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
55-
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
56-
--no-transfer-progress \
57-
-Dnet.bytebuddy.experimental=true
54+
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
55+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
56+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
57+
-Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
58+
--no-transfer-progress \
59+
-Dnet.bytebuddy.experimental=true
5860
- name: Stop broker
5961
run: docker stop rabbitmq && docker rm rabbitmq

Diff for: ‎.github/workflows/test.yml

+13-12
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,23 @@ jobs:
4141
run: make deps
4242
- name: Test with NIO
4343
run: |
44-
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
45-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
44+
./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
45+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
46+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
47+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
4648
-Dmaven.javadoc.skip=true \
47-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
4849
--no-transfer-progress
4950
- name: Test with blocking IO
5051
run: |
51-
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
52-
-Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
53-
-Dmaven.javadoc.skip=true \
54-
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
55-
--no-transfer-progress
56-
- name: Stop broker A
57-
run: docker stop rabbitmq && docker rm rabbitmq
58-
- name: Stop broker B
59-
run: docker stop hare && docker rm hare
52+
./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
53+
-Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
54+
-Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
55+
-Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
56+
-Dmaven.javadoc.skip=true \
57+
-Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
58+
--no-transfer-progress
59+
- name: Stop cluster
60+
run: docker compose --file ci/cluster/docker-compose.yml down
6061
- name: Publish snapshot
6162
if: ${{ github.event_name != 'pull_request' }}
6263
run: ./mvnw clean deploy -Psnapshots -DskipITs -DskipTests --no-transfer-progress

Diff for: ‎ci/_start-cluster.sh

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env bash
2+
3+
LOCAL_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
4+
5+
RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0}
6+
7+
wait_for_message() {
8+
while ! docker logs "$1" | grep -q "$2";
9+
do
10+
sleep 5
11+
echo "Waiting 5 seconds for $1 to start..."
12+
done
13+
}
14+
15+
make -C "${PWD}"/tls-gen/basic
16+
17+
mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
18+
mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
19+
mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
20+
mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
21+
22+
rm -rf rabbitmq-configuration
23+
mkdir -p rabbitmq-configuration/tls
24+
25+
cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
26+
chmod -R o+r rabbitmq-configuration/tls/*
27+
chmod -R g+r rabbitmq-configuration/tls/*
28+
./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
29+
cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
30+
cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
31+
32+
echo "Running RabbitMQ ${RABBITMQ_IMAGE}"
33+
34+
docker rm -f rabbitmq 2>/dev/null || echo "rabbitmq was not running"
35+
docker run -d --name rabbitmq \
36+
--network host \
37+
-v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
38+
--env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
39+
--env RABBITMQ_NODENAME=rabbit@$(hostname) \
40+
--env RABBITMQ_NODE_PORT=5672 \
41+
--env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
42+
"${RABBITMQ_IMAGE}"
43+
44+
# for CLI commands to share the same cookie
45+
docker exec rabbitmq bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
46+
docker exec rabbitmq chmod 0600 /var/lib/rabbitmq/.erlang.cookie
47+
48+
wait_for_message rabbitmq "completed with"
49+
50+
docker run -d --name hare \
51+
--network host \
52+
-v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
53+
--env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
54+
--env RABBITMQ_NODENAME=hare@$(hostname) \
55+
--env RABBITMQ_NODE_PORT=5673 \
56+
--env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
57+
"${RABBITMQ_IMAGE}"
58+
59+
# for CLI commands to share the same cookie
60+
docker exec hare bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
61+
docker exec hare chmod 0600 /var/lib/rabbitmq/.erlang.cookie
62+
63+
wait_for_message hare "completed with"
64+
65+
docker exec hare rabbitmqctl --node hare@$(hostname) status
66+
67+
docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) is_running
68+
docker exec hare rabbitmq-diagnostics --node hare@$(hostname) is_running
69+
70+
docker exec hare rabbitmqctl --node hare@$(hostname) stop_app
71+
docker exec hare rabbitmqctl --node hare@$(hostname) join_cluster rabbit@$(hostname)
72+
docker exec hare rabbitmqctl --node hare@$(hostname) start_app
73+
74+
sleep 10
75+
76+
docker exec hare rabbitmqctl --node hare@$(hostname) await_startup
77+
78+
docker exec hare rabbitmqctl --node hare@$(hostname) enable_feature_flag --opt-in khepri_db
79+
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) enable_feature_flag --opt-in khepri_db
80+
81+
docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) erlang_version
82+
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) version
83+
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) status
84+
docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) cluster_status

Diff for: ‎ci/cluster/configuration/rabbitmq.conf

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
2+
cluster_formation.classic_config.nodes.1 = rabbit@node0
3+
cluster_formation.classic_config.nodes.2 = rabbit@node1
4+
cluster_formation.classic_config.nodes.3 = rabbit@node2
5+
loopback_users = none
6+
7+
listeners.ssl.default = 5671
8+
9+
ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
10+
ssl_options.certfile = /etc/rabbitmq/tls/server_certificate.pem
11+
ssl_options.keyfile = /etc/rabbitmq/tls/server_key.pem
12+
ssl_options.verify = verify_peer
13+
ssl_options.fail_if_no_peer_cert = false
14+
ssl_options.honor_cipher_order = true
15+
16+
auth_mechanisms.1 = PLAIN
17+
auth_mechanisms.2 = ANONYMOUS
18+
auth_mechanisms.3 = AMQPLAIN
19+
auth_mechanisms.4 = EXTERNAL
20+
auth_mechanisms.5 = RABBIT-CR-DEMO

Diff for: ‎ci/cluster/docker-compose.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
services:
2+
node0:
3+
environment:
4+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
5+
networks:
6+
- rabbitmq-cluster
7+
hostname: node0
8+
container_name: rabbitmq0
9+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
10+
pull_policy: always
11+
ports:
12+
- "5672:5672"
13+
- "5671:5671"
14+
tty: true
15+
volumes:
16+
- ./configuration/:/etc/rabbitmq/
17+
- ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
18+
node1:
19+
environment:
20+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
21+
networks:
22+
- rabbitmq-cluster
23+
hostname: node1
24+
container_name: rabbitmq1
25+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
26+
pull_policy: always
27+
ports:
28+
- "5673:5672"
29+
tty: true
30+
volumes:
31+
- ./configuration/:/etc/rabbitmq/
32+
- ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
33+
node2:
34+
environment:
35+
- RABBITMQ_ERLANG_COOKIE='secret_cookie'
36+
networks:
37+
- rabbitmq-cluster
38+
hostname: node2
39+
container_name: rabbitmq2
40+
image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
41+
pull_policy: always
42+
ports:
43+
- "5674:5672"
44+
tty: true
45+
volumes:
46+
- ./configuration/:/etc/rabbitmq/
47+
- ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
48+
networks:
49+
rabbitmq-cluster:

Diff for: ‎ci/start-broker.sh

+23-11
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,33 @@ wait_for_message() {
1212
done
1313
}
1414

15-
make -C "${PWD}"/tls-gen/basic
15+
rm -rf rabbitmq-configuration
16+
mkdir -p rabbitmq-configuration/tls
1617

17-
mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
18-
mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
19-
mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
20-
mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
18+
make -C "${PWD}"/tls-gen/basic
2119

2220
rm -rf rabbitmq-configuration
2321
mkdir -p rabbitmq-configuration/tls
24-
25-
cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
26-
chmod -R o+r rabbitmq-configuration/tls/*
27-
chmod -R g+r rabbitmq-configuration/tls/*
28-
./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
29-
cp target/test-classes/[email protected] rabbitmq-configuration/rabbitmq.config
22+
cp -R "${PWD}"/tls-gen/basic/result/* rabbitmq-configuration/tls
23+
chmod o+r rabbitmq-configuration/tls/*
24+
chmod g+r rabbitmq-configuration/tls/*
25+
26+
echo "loopback_users = none
27+
28+
listeners.ssl.default = 5671
29+
30+
ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
31+
ssl_options.certfile = /etc/rabbitmq/tls/server_$(hostname)_certificate.pem
32+
ssl_options.keyfile = /etc/rabbitmq/tls/server_$(hostname)_key.pem
33+
ssl_options.verify = verify_peer
34+
ssl_options.fail_if_no_peer_cert = false
35+
ssl_options.honor_cipher_order = true
36+
37+
auth_mechanisms.1 = PLAIN
38+
auth_mechanisms.2 = ANONYMOUS
39+
auth_mechanisms.3 = AMQPLAIN
40+
auth_mechanisms.4 = EXTERNAL
41+
auth_mechanisms.5 = RABBIT-CR-DEMO" >> rabbitmq-configuration/rabbitmq.conf
3042

3143
echo "Running RabbitMQ ${RABBITMQ_IMAGE}"
3244

0 commit comments

Comments
 (0)