Skip to content

Commit 3d05c55

Browse files
lingtinglingting
authored
feat: More convenient local code testing (#2388)
* Update .gitignore * feat(docker): add basic image * feat(docker): Use local code run in docker * docks: use en * fix(docker): remove test service * feat(docker): add task buildLocal * refactor(docker): remove basic image. use eclipse-temurin:17-jdk-noble * fix(docker): remove test service * Revert "feat(docker): add task buildLocal" This reverts commit f1d58fc. * refactor(docker): use releaseTarGz task to run docker
1 parent d34b894 commit 3d05c55

File tree

5 files changed

+202
-0
lines changed

5 files changed

+202
-0
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ TAGS
3131
Vagrantfile.local
3232
/logs
3333
.DS_Store
34+
local.properties
3435

3536
config/server-*
3637
config/zookeeper-*

‎docker/README.md

+23
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,29 @@ Run `pip install -r requirements.txt` to get all the requirements for running th
114114

115115
Make sure you have docker installed with support for buildx enabled. (For pushing multi-architecture image to docker registry)
116116

117+
Use local code run in docker
118+
---------------------------------------
119+
120+
- command run in project root folders
121+
122+
1. generate tgz
123+
```shell
124+
# For example only, can be modified based on your compilation requirements
125+
./gradlew releaseTarGz -x test -x check
126+
```
127+
2. run
128+
```shell
129+
docker-compose -f docker/local/docker-compose.yml up -d
130+
```
131+
132+
- After modifying your code, simply regenerate the tgz and restart the specified service.
133+
```shell
134+
# For example only, can be modified based on your compilation requirements
135+
./gradlew releaseTarGz -x test -x check
136+
# eg: restart broker
137+
docker-compose -f docker/local/docker-compose.yml up broker1 broker2 -d --force-recreate
138+
```
139+
117140
Building image and running tests locally
118141
---------------------------------------
119142

‎docker/local/docker-compose.yml

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
2+
version: "3.8"
3+
4+
services:
5+
localstack:
6+
container_name: "${LOCALSTACK_DOCKER_NAME-localstack}"
7+
hostname: "${LOCALSTACK_DOCKER_NAME-localstack}"
8+
image: localstack/localstack:3.3.0
9+
ports:
10+
- "4566:4566" # LocalStack Gateway
11+
- "4510-4559:4510-4559" # external services port range
12+
environment:
13+
- DOCKER_HOST=unix:///var/run/docker.sock
14+
volumes:
15+
- s3_data:/var/lib/localstack
16+
- /var/run/docker.sock:/var/run/docker.sock
17+
# use a static ip
18+
networks:
19+
automq_net:
20+
ipv4_address: 10.6.0.2
21+
22+
# create needed buckets
23+
aws-cli:
24+
container_name: "${AWS_CLI_DOCKER_NAME-aws-cli}"
25+
hostname: "${AWS_CLI_DOCKER_NAME-aws-cli}"
26+
image: amazon/aws-cli:2.15.37
27+
environment:
28+
- AWS_ACCESS_KEY_ID=test
29+
- AWS_SECRET_ACCESS_KEY=test
30+
- AWS_DEFAULT_REGION=us-east-1
31+
command: s3api create-bucket --bucket ko3 --endpoint=http://10.6.0.2:4566
32+
depends_on:
33+
localstack:
34+
condition: service_healthy
35+
networks:
36+
- automq_net
37+
38+
# Only comment out, do not remove. For easier future testing
39+
# test:
40+
# container_name: test
41+
# hostname: test
42+
# # image: amazoncorretto:17.0.14
43+
# # image: gradle:jdk17
44+
# image: eclipse-temurin:17-jdk-noble
45+
# working_dir: /opt/automq
46+
# volumes:
47+
# - ../../core/build/distributions:/opt/volume_libs:ro
48+
# - ../scripts:/opt/volume_scripts:ro
49+
# command:
50+
# - bash
51+
# - -c
52+
# - |
53+
# /opt/volume_scripts/pre_start.sh && \
54+
# tail -f /dev/null
55+
56+
controller:
57+
container_name: "${CONTROLLER_DOCKER_NAME-controller}"
58+
hostname: "${CONTROLLER_DOCKER_NAME-controller}"
59+
stop_grace_period: 2m
60+
image: eclipse-temurin:17-jdk-noble
61+
environment:
62+
- KAFKA_S3_ACCESS_KEY=test
63+
- KAFKA_S3_SECRET_KEY=test
64+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m
65+
volumes:
66+
- ../../core/build/distributions:/opt/volume_libs:ro
67+
- ../scripts:/opt/volume_scripts:ro
68+
command:
69+
- bash
70+
- -c
71+
- |
72+
/opt/volume_scripts/pre_start.sh && \
73+
/opt/automq/scripts/start.sh up --process.roles controller --node.id 0 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
74+
networks:
75+
- automq_net
76+
depends_on:
77+
- localstack
78+
- aws-cli
79+
80+
broker1:
81+
container_name: "${BROKER1_DOCKER_NAME-broker1}"
82+
hostname: "${BROKER1_DOCKER_NAME-broker1}"
83+
stop_grace_period: 2m
84+
image: eclipse-temurin:17-jdk-noble
85+
ports:
86+
- "9094:9094"
87+
environment:
88+
- KAFKA_S3_ACCESS_KEY=test
89+
- KAFKA_S3_SECRET_KEY=test
90+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
91+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_IN_CAPACITY=5120
92+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_OUT_CAPACITY=5120
93+
- KAFKA_CFG_AUTOBALANCER_REPORTER_METRICS_REPORTING_INTERVAL_MS=5000
94+
# override listener settings
95+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,EXTERNAL://:9094
96+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://broker1:9092,EXTERNAL://localhost:9094
97+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
98+
volumes:
99+
- ../../core/build/distributions:/opt/volume_libs:ro
100+
- ../scripts:/opt/volume_scripts:ro
101+
command:
102+
- bash
103+
- -c
104+
- |
105+
/opt/volume_scripts/pre_start.sh && \
106+
/opt/automq/scripts/start.sh up --process.roles broker --node.id 1 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
107+
networks:
108+
- automq_net
109+
depends_on:
110+
- localstack
111+
- aws-cli
112+
- controller
113+
114+
broker2:
115+
container_name: "${BROKER2_DOCKER_NAME-broker2}"
116+
hostname: "${BROKER2_DOCKER_NAME-broker2}"
117+
stop_grace_period: 2m
118+
image: eclipse-temurin:17-jdk-noble
119+
ports:
120+
- "9095:9095"
121+
environment:
122+
- KAFKA_S3_ACCESS_KEY=test
123+
- KAFKA_S3_SECRET_KEY=test
124+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
125+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_IN_CAPACITY=5120
126+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_OUT_CAPACITY=5120
127+
- KAFKA_CFG_AUTOBALANCER_REPORTER_METRICS_REPORTING_INTERVAL_MS=5000
128+
# override listener settings
129+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,EXTERNAL://:9095
130+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://broker2:9092,EXTERNAL://localhost:9095
131+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
132+
volumes:
133+
- ../../core/build/distributions:/opt/volume_libs:ro
134+
- ../scripts:/opt/volume_scripts:ro
135+
command:
136+
- bash
137+
- -c
138+
- |
139+
/opt/volume_scripts/pre_start.sh && \
140+
/opt/automq/scripts/start.sh up --process.roles broker --node.id 2 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
141+
networks:
142+
- automq_net
143+
depends_on:
144+
- localstack
145+
- aws-cli
146+
- controller
147+
148+
volumes:
149+
s3_data:
150+
driver: local
151+
152+
networks:
153+
automq_net:
154+
name: automq_net
155+
driver: bridge
156+
ipam:
157+
driver: default
158+
config:
159+
- subnet: "10.6.0.0/16"
160+
gateway: "10.6.0.1"

‎docker/scripts/pre_start.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
echo "[PreStart] mkdir"
4+
rm -rf /opt/automq
5+
mkdir -p /opt/kafka || exit 1
6+
ln -s /opt/kafka /opt/automq || exit 1
7+
echo "[PreStart] file"
8+
for f in /opt/volume_libs/*.tgz; do
9+
tar -xzf "$f" -C /opt/kafka --one-top-level=kafka --strip-components=1 --overwrite
10+
done
11+
cp -r /opt/volume_scripts /opt/kafka/scripts || exit 1
12+
find /opt/kafka -type f -name "*.sh" -exec chmod a+x {} \;
13+
echo "[PreStart] env"
14+
echo "export DEBIAN_FRONTEND=noninteractive" >> ~/.bashrc
15+
echo "export AWS_DEFAULT_REGION=us-east-1" >> ~/.bashrc
16+
echo "export KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseZGC -XX:ZCollectionInterval=5\"" >> ~/.bashrc

‎docker/scripts/start.sh

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717

18+
source ~/.bashrc
19+
1820
script_path="${0}"
1921

2022
# The absolute path to the directory which this script is in.

0 commit comments

Comments
 (0)