Skip to content

Commit a41ee9e

Browse files
HarshCasperHarshCasper
authored
add docs for self-managed Kafka cluster (#130)
* DOCS: Add docs for self-managed Kafka cluster * Update index.md
1 parent c17efbc commit a41ee9e

File tree

2 files changed

+179
-0
lines changed

2 files changed

+179
-0
lines changed

‎content/en/integrations/kafka/docker-compose.yml

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
version: '3.9'
2+
services:
3+
zookeeper:
4+
image: confluentinc/cp-zookeeper:6.0.2
5+
container_name: zookeeper
6+
hostname: zookeeper
7+
ports:
8+
- "2181:2181"
9+
environment:
10+
ZOOKEEPER_CLIENT_PORT: 2181
11+
ZOOKEEPER_TICK_TIME: 2000
12+
13+
kafka:
14+
image: confluentinc/cp-kafka:6.0.2
15+
container_name: kafka
16+
hostname: kafka
17+
restart: always
18+
depends_on:
19+
- zookeeper
20+
ports:
21+
- "9092:9092"
22+
- "9101:9101"
23+
environment:
24+
KAFKA_BROKER_ID: 1
25+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
26+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
27+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
28+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
29+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
30+
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
31+
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
32+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
33+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
34+
KAFKA_JMX_PORT: 9101
35+
KAFKA_JMX_HOSTNAME: localhost
36+
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
37+
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
38+
CONFLUENT_METRICS_ENABLE: 'true'
39+
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
40+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
41+
42+
kowl:
43+
image: quay.io/cloudhut/kowl:v1.3.1
44+
container_name: kowl
45+
restart: always
46+
ports:
47+
- "8080:8080"
48+
depends_on:
49+
- kafka
50+
environment:
51+
- KAFKA_BROKERS=kafka:29092
52+
53+
localstack:
54+
container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
55+
image: localstack/localstack
56+
network_mode: bridge
57+
ports:
58+
- "4566:4566"
59+
depends_on:
60+
- kafka
61+
- kowl
62+
environment:
63+
- SERVICES=lambda,secretsmanager
64+
- DEBUG=${DEBUG- }
65+
- DATA_DIR=${DATA_DIR- }
66+
- LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR- }
67+
- KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- }
68+
- DOCKER_HOST=unix:///var/run/docker.sock
69+
- HOST_TMP_FOLDER=${TMPDIR}
70+
volumes:
71+
- "${TMPDIR:-/tmp/localstack}:/tmp/localstack"
72+
- "/var/run/docker.sock:/var/run/docker.sock"

‎content/en/integrations/kafka/index.md

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
---
2+
title: "Self-managed Kafka cluster"
3+
tags: ["kafka", "self-managed"]
4+
categories: []
5+
weight: 11
6+
description: >
7+
Using Localstack lambda with self-managed Kafka cluster
8+
---
9+
10+
Localstack does not currently support AWS MSK out of the box, but you can run your own self-managed Kafka cluster and integrate it with your own applications.
11+
12+
## Running self-managed Kafka
13+
14+
You can find the [example Docker Compose](docker-compose.yml) file which contains a single-noded ZooKeeper and a Kafka cluster and a simple LocalStack setup as well as [Kowl](https://github.com/cloudhut/kowl), an Apache Kafka Web UI.
15+
16+
1. Run Docker Compose:
17+
18+
{{< command >}}
19+
$ docker-compose up -d
20+
{{< / command >}}
21+
22+
2. Create the Lambda function:
23+
24+
{{< command >}}
25+
$ awslocal lambda create-function \
26+
--function-name fun1 \
27+
--handler lambda.handler \
28+
--runtime python3.8 \
29+
--role r1 \
30+
--zip-file fileb://lambda.zip
31+
{
32+
"FunctionName": "fun1",
33+
"FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1",
34+
"Runtime": "python3.8",
35+
"Role": "r1",
36+
"Handler": "lambda.handler",
37+
"CodeSize": 294,
38+
"Description": "",
39+
"Timeout": 3,
40+
"LastModified": "2021-05-19T02:01:06.617+0000",
41+
"CodeSha256": "/GPsiNXaq4tBA4QpxPCwgpeVfP7j+1tTH6zdkJ3jiU4=",
42+
"Version": "$LATEST",
43+
"VpcConfig": {},
44+
"TracingConfig": {
45+
"Mode": "PassThrough"
46+
},
47+
"RevisionId": "d85469d2-8558-4d75-bc0e-5926f373e12c",
48+
"State": "Active",
49+
"LastUpdateStatus": "Successful",
50+
"PackageType": "Zip"
51+
}
52+
{{< / command >}}
53+
54+
3. Create an example secret:
55+
56+
{{< command >}}
57+
$ awslocal secretsmanager create-secret --name localstack
58+
{
59+
"ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI",
60+
"Name": "localstack",
61+
"VersionId": "32bbb8e2-46ee-4322-b3d5-b6459d54513b"
62+
}
63+
{{< / command >}}
64+
65+
4. Create an example Kafka topic:
66+
67+
{{< command >}}
68+
$ docker exec -ti kafka kafka-topics --zookeeper zookeeper:2181 --create --replication-factor 1 --partitions 1 --topic t1
69+
Created topic t1.
70+
{{< / command >}}
71+
72+
5. Create the event source mapping to your local kafka cluster:
73+
74+
{{< command >}}
75+
$ awslocal lambda create-event-source-mapping \
76+
--topics t1 \
77+
--source-access-configuration Type=SASL_SCRAM_512_AUTH,URI=arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI \
78+
--function-name arn:aws:lambda:us-east-1:000000000000:function:fun1 \
79+
--self-managed-event-source '{"Endpoints":{"KAFKA_BOOTSTRAP_SERVERS":["localhost:9092"]}}'
80+
{
81+
"UUID": "4a2b0ea6-960c-4847-8684-465876dd6dbd",
82+
"BatchSize": 100,
83+
"FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1",
84+
"LastModified": "2021-05-19T04:02:49+02:00",
85+
"LastProcessingResult": "OK",
86+
"State": "Enabled",
87+
"StateTransitionReason": "User action",
88+
"Topics": [
89+
"t1"
90+
],
91+
"SourceAccessConfigurations": [
92+
{
93+
"Type": "SASL_SCRAM_512_AUTH",
94+
"URI": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI"
95+
}
96+
],
97+
"SelfManagedEventSource": {
98+
"Endpoints": {
99+
"KAFKA_BOOTSTRAP_SERVERS": [
100+
"localhost:9092"
101+
]
102+
}
103+
}
104+
}
105+
{{< / command >}}
106+
107+
6. Aditionally visit `http://localhost:8080` for Kowl's UI.

0 commit comments

Comments
 (0)