diff --git a/content/en/integrations/kafka/docker-compose.yml b/content/en/integrations/kafka/docker-compose.yml new file mode 100644 index 0000000000..42ff397be0 --- /dev/null +++ b/content/en/integrations/kafka/docker-compose.yml @@ -0,0 +1,72 @@ +version: '3.9' +services: + zookeeper: + image: confluentinc/cp-zookeeper:6.0.2 + container_name: zookeeper + hostname: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:6.0.2 + container_name: kafka + hostname: kafka + restart: always + depends_on: + - zookeeper + ports: + - "9092:9092" + - "9101:9101" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'true' + CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + + kowl: + image: quay.io/cloudhut/kowl:v1.3.1 + container_name: kowl + restart: always + ports: + - "8080:8080" + depends_on: + - kafka + environment: + - KAFKA_BROKERS=kafka:29092 + + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: localstack/localstack + network_mode: bridge + ports: + - "4566:4566" + depends_on: + - kafka + - kowl + environment: + - SERVICES=lambda,secretsmanager + - DEBUG=${DEBUG- } + - DATA_DIR=${DATA_DIR- } + - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR- } + - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- } + - DOCKER_HOST=unix:///var/run/docker.sock + - HOST_TMP_FOLDER=${TMPDIR} + volumes: + - "${TMPDIR:-/tmp/localstack}:/tmp/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/content/en/integrations/kafka/index.md b/content/en/integrations/kafka/index.md new file mode 100644 index 0000000000..86bb904c78 --- /dev/null +++ b/content/en/integrations/kafka/index.md @@ -0,0 +1,107 @@ +--- +title: "Self-managed Kafka cluster" +tags: ["kafka", "self-managed"] +categories: [] +weight: 11 +description: > + Using Localstack lambda with self-managed Kafka cluster +--- + +Localstack does not currently support AWS MSK out of the box, but you can run your own self-managed Kafka cluster and integrate it with your own applications. + +## Running self-managed Kafka + +You can find the [example Docker Compose](docker-compose.yml) file which contains a single-noded ZooKeeper and a Kafka cluster and a simple LocalStack setup as well as [Kowl](https://github.com/cloudhut/kowl), an Apache Kafka Web UI. + +1. Run Docker Compose: + +{{< command >}} +$ docker-compose up -d +{{< / command >}} + +2. Create the Lambda function: + +{{< command >}} +$ awslocal lambda create-function \ + --function-name fun1 \ + --handler lambda.handler \ + --runtime python3.8 \ + --role r1 \ + --zip-file fileb://lambda.zip +{ + "FunctionName": "fun1", + "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1", + "Runtime": "python3.8", + "Role": "r1", + "Handler": "lambda.handler", + "CodeSize": 294, + "Description": "", + "Timeout": 3, + "LastModified": "2021-05-19T02:01:06.617+0000", + "CodeSha256": "/GPsiNXaq4tBA4QpxPCwgpeVfP7j+1tTH6zdkJ3jiU4=", + "Version": "$LATEST", + "VpcConfig": {}, + "TracingConfig": { + "Mode": "PassThrough" + }, + "RevisionId": "d85469d2-8558-4d75-bc0e-5926f373e12c", + "State": "Active", + "LastUpdateStatus": "Successful", + "PackageType": "Zip" +} +{{< / command >}} + +3. Create an example secret: + +{{< command >}} +$ awslocal secretsmanager create-secret --name localstack +{ + "ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI", + "Name": "localstack", + "VersionId": "32bbb8e2-46ee-4322-b3d5-b6459d54513b" +} +{{< / command >}} + +4. Create an example Kafka topic: + +{{< command >}} +$ docker exec -ti kafka kafka-topics --zookeeper zookeeper:2181 --create --replication-factor 1 --partitions 1 --topic t1 +Created topic t1. +{{< / command >}} + +5. Create the event source mapping to your local kafka cluster: + +{{< command >}} +$ awslocal lambda create-event-source-mapping \ + --topics t1 \ + --source-access-configuration Type=SASL_SCRAM_512_AUTH,URI=arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI \ + --function-name arn:aws:lambda:us-east-1:000000000000:function:fun1 \ + --self-managed-event-source '{"Endpoints":{"KAFKA_BOOTSTRAP_SERVERS":["localhost:9092"]}}' +{ + "UUID": "4a2b0ea6-960c-4847-8684-465876dd6dbd", + "BatchSize": 100, + "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1", + "LastModified": "2021-05-19T04:02:49+02:00", + "LastProcessingResult": "OK", + "State": "Enabled", + "StateTransitionReason": "User action", + "Topics": [ + "t1" + ], + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI" + } + ], + "SelfManagedEventSource": { + "Endpoints": { + "KAFKA_BOOTSTRAP_SERVERS": [ + "localhost:9092" + ] + } + } +} +{{< / command >}} + +6. Aditionally visit `http://localhost:8080` for Kowl's UI.