Decompose docker-compose files so that depends-on can be used. (spring-attic#4915)

Author: Corneil du Plessis

spring-cloud-dataflow-server/README.adoc

@@ -5,6 +5,7 @@ The Data Flow Server deploys Streams by delegating to Spring Cloud Skipper and d
 
 NOTE: The local version of the Data Flow server should only be used for Stream development but can be used in a production environment for Task deployment as a replacement for the Spring Cloud Batch Admin Server.
 
+
 == Getting Started
 
 This shows how to start the Data Flow Server and Shell to create the `time | log` stream.
@@ -140,6 +141,8 @@ To disable the cloud config server set the `spring.cloud.config.enabled` propert
 
 == Docker-Compose Integration Tests
 
+NOTE: The docker-compose files have been decomposed so that you have to specify the database and messaging engine separately. Previously mysql and kafka was the default.
+
 To run the docker compose integration, enable the `-Pfailsafe` maven profile and the `-Dgroups="docker-compose"` JUnit tag.
 
 . You can run just the integration tests like this:
@@ -153,7 +156,7 @@ SCDF, Skipper servers installed, or the versions Stream and Task apps:
 
 ----
 ./mvnw clean test-compile failsafe:integration-test -pl spring-cloud-dataflow-server -Pfailsafe -Dgroups="docker-compose" \
-   -Dtest.docker.compose.paths=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-influxdb.yml,../src/docker-compose/docker-compose-postgres.yml,../src/docker-compose/docker-compose-rabbitmq.yml \
+   -Dtest.docker.compose.paths="../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-influxdb.yml,../src/docker-compose/docker-compose-postgres.yml,../src/docker-compose/docker-compose-rabbitmq.yml" \
    -Dtest.docker.compose.stream.apps.uri=https://dataflow.spring.io/rabbitmq-maven-latest \
    -Dtest.docker.compose.dataflow.version=2.8.0-SNAPSHOT \
    -Dtest.docker.compose.skipper.version=2.7.0-SNAPSHOT \
@@ -163,7 +166,11 @@ The `test.docker.compose.paths` property accepts comma separated list of docker
 
 ----
 ./mvnw clean test-compile integration-test -pl spring-cloud-dataflow-server -Pfailsafe -Dgroups="docker-compose" \
-   -Dtest.docker.compose.paths=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-dood.yml,../src/docker-compose/docker-compose-prometheus.yml \
+   -Dtest.docker.compose.paths=""../src/docker-compose/docker-compose.yml,\
+    ../src/docker-compose/docker-compose-mysql.yml, \
+    ../src/docker-compose/docker-compose-kafka.yml, \
+    ../src/docker-compose/docker-compose-dood.yml, \
+    ../src/docker-compose/docker-compose-prometheus.yml" \
    -Dtest.docker.compose.stream.apps.uri=https://dataflow.spring.io/kafka-docker-latest \
    -Dtest.docker.compose.task.apps.uri=https://dataflow.spring.io/task-docker-latest \
    -Dtest.docker.compose.dataflow.version=2.8.0-SNAPSHOT \
@@ -186,7 +193,10 @@ Use the `-Dit.test=class-name#method-name` to filter only selected tests.
 
 ----
 ./mvnw clean test-compile integration-test -pl spring-cloud-dataflow-server -Pfailsafe -Dgroups="docker-compose" \
-   -Dtest.docker.compose.paths=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-ssl.yml \
+   -Dtest.docker.compose.paths=""../src/docker-compose/docker-compose.yml,\
+    ../src/docker-compose/docker-compose-mysql.yml, \
+    ../src/docker-compose/docker-compose-kafka.yml, \
+    ../src/docker-compose/docker-compose-ssl.yml" \
    -Dtest.docker.compose.dataflow.uri=https://dataflow-server:9393 \
    -Dtest.docker.compose.skipper.uri=https://skipper-server:7577 \
    -Dtest.docker.compose.waiting.for.service.format=https://$HOST:$EXTERNAL_PORT \
@@ -198,7 +208,10 @@ Use the `-Dit.test=class-name#method-name` to filter only selected tests.
 
 Use the following env. variables to run an IT test from IDE:
 ----
-TEST_DOCKER_COMPOSE_PATHS=../src/docker-compose/docker-compose.yml,../src/docker-compose/docker-compose-ssl.yml
+TEST_DOCKER_COMPOSE_PATHS=../src/docker-compose/docker-compose.yml,\
+    ../src/docker-compose/docker-compose-mysql.yml, \
+    ../src/docker-compose/docker-compose-kafka.yml, \
+    ../src/docker-compose/docker-compose-ssl.yml
 TEST_DOCKER_COMPOSE_SKIPPER_URI=https://skipper-server:7577
 TEST_DOCKER_COMPOSE_DATAFLOW_URI=https://dataflow-server:9393
 TEST_DOCKER_COMPOSE_WAITING_FOR_SERVICE_FORMAT=https://$HOST:$EXTERNAL_PORT
@@ -220,7 +233,10 @@ Create HTTPS docker-compose from shell:
 ----
 export DATAFLOW_URI=https://dataflow-server:9393
 export SKIPPER_URI=https://skipper-server:7577
-docker-compose -f ./src/docker-compose/docker-compose.yml -f ./src/docker-compose/docker-compose-ssl.yml
+docker-compose -f ./src/docker-compose/docker-compose.yml \
+               -f ./src/docker-compose/docker-compose-mysql.yml, \
+               -f ./src/docker-compose/docker-compose-kafka.yml, \
+               -f ./src/docker-compose/docker-compose-ssl.yml
 ----
 
 

src/docker-compose/docker-compose-kafka.yml

@@ -0,0 +1,56 @@
+version: '3'
+
+# Configuration environment variables:
+# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
+# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
+# - HOST_MOUNT_PATH and DOCKER_MOUNT_PATH are used to set the host and docker mount folders.
+#   If not set HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started.
+#   If not set DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers.
+#   Example to mount the local Maven repository: HOST_MOUNT_PATH=~/.m2 DOCKER_MOUNT_PATH=/home/cnb/.m2  docker-compose up
+# - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE
+#   must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container.
+#   The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that
+#   doesn't overlap with the APPS_PORT_RANGE.
+#
+# Exposed container ports:
+# - 9393:9393               - Data Flow server port (http://localhost:9393/dashboard)
+# - 7577:7577               - Skipper server port (http://localhost:7577/api)
+# - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps).
+#                             That means you can reach the application's actuator endpoints from your host machine.
+#                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
+#                             can be reached on the ports they expose.
+services:
+
+  kafka-broker:
+    image: confluentinc/cp-kafka:5.5.2
+    container_name: dataflow-kafka
+    expose:
+      - "9092"
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.2
+    container_name: dataflow-kafka-zookeeper
+    expose:
+      - "2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+
+  dataflow-server:
+    environment:
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181
+
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
+    depends_on:
+      - kafka-broker

src/docker-compose/docker-compose-mariadb.yml

@@ -0,0 +1,48 @@
+version: '3'
+
+# Configuration environment variables:
+# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
+# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
+# - HOST_MOUNT_PATH and DOCKER_MOUNT_PATH are used to set the host and docker mount folders.
+#   If not set HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started.
+#   If not set DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers.
+#   Example to mount the local Maven repository: HOST_MOUNT_PATH=~/.m2 DOCKER_MOUNT_PATH=/home/cnb/.m2  docker-compose up
+# - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE
+#   must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container.
+#   The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that
+#   doesn't overlap with the APPS_PORT_RANGE.
+#
+# Exposed container ports:
+# - 9393:9393               - Data Flow server port (http://localhost:9393/dashboard)
+# - 7577:7577               - Skipper server port (http://localhost:7577/api)
+# - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps).
+#                             That means you can reach the application's actuator endpoints from your host machine.
+#                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
+#                             can be reached on the ports they expose.
+services:
+  mariadb:
+    image: mariadb:10.4.22
+    container_name: dataflow-mariadb
+    environment:
+      MARIADB_DATABASE: dataflow
+      MARIADB_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  dataflow-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
+    depends_on:
+      - mariadb
+
+  skipper-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
+    depends_on:
+      - mariadb

src/docker-compose/docker-compose-mysql.yml

@@ -0,0 +1,49 @@
+version: '3'
+
+# Configuration environment variables:
+# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
+# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
+# - HOST_MOUNT_PATH and DOCKER_MOUNT_PATH are used to set the host and docker mount folders.
+#   If not set HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started.
+#   If not set DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers.
+#   Example to mount the local Maven repository: HOST_MOUNT_PATH=~/.m2 DOCKER_MOUNT_PATH=/home/cnb/.m2  docker-compose up
+# - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE
+#   must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container.
+#   The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that
+#   doesn't overlap with the APPS_PORT_RANGE.
+#
+# Exposed container ports:
+# - 9393:9393               - Data Flow server port (http://localhost:9393/dashboard)
+# - 7577:7577               - Skipper server port (http://localhost:7577/api)
+# - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps).
+#                             That means you can reach the application's actuator endpoints from your host machine.
+#                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
+#                             can be reached on the ports they expose.
+services:
+  mysql:
+    image: mysql:5.7.25
+    container_name: dataflow-mysql
+    environment:
+      MYSQL_DATABASE: dataflow
+      MYSQL_USER: root
+      MYSQL_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  dataflow-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
+    depends_on:
+      - mysql
+
+  skipper-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
+    depends_on:
+      - mysql

src/docker-compose/docker-compose-postgres.yml

@@ -4,9 +4,6 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
 services:
 
-  mariadb:
-    entrypoint: ["echo", "MariaDB service is disabled"]
-
   postgres:
     image: postgres:10
     command: postgres -c 'max_connections=300'
@@ -27,11 +24,13 @@ services:
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-
+    depends_on:
+      - postgres
   skipper-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-
+    depends_on:
+      - postgres

src/docker-compose/docker-compose-rabbitmq.yml

@@ -19,6 +19,8 @@ services:
   dataflow-server:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
+    depends_on:
+      - rabbitmq
 
   app-import-stream:
     command: >

src/docker-compose/docker-compose.yml

@@ -20,38 +20,6 @@ version: '3'
 #                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
 #                             can be reached on the ports they expose.
 services:
-  mariadb:
-    image: mariadb:10.4.22
-    container_name: dataflow-mariadb
-    environment:
-      MARIADB_DATABASE: dataflow
-      MARIADB_ROOT_PASSWORD: rootpw
-    expose:
-      - 3306
-
-  kafka-broker:
-    image: confluentinc/cp-kafka:5.5.2
-    container_name: dataflow-kafka
-    expose:
-      - "9092"
-    environment:
-      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
-      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
-      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
-      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
-      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
-    depends_on:
-      - zookeeper
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:5.5.2
-    container_name: dataflow-kafka-zookeeper
-    expose:
-      - "2181"
-    environment:
-      - ZOOKEEPER_CLIENT_PORT=2181
-
   dataflow-server:
     user: root
     image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.10.0-SNAPSHOT}${BP_JVM_VERSION:-}
@@ -70,25 +38,14 @@ services:
 
       - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
 
-      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
-      - SPRING_DATASOURCE_USERNAME=root
-      - SPRING_DATASOURCE_PASSWORD=rootpw
-      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
 
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
     depends_on:
-      - kafka-broker
       - skipper-server
-    entrypoint: >
-      /bin/sh -c "
-         apt-get update && apt-get install --no-install-recommends -y wget &&
-         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
-         chmod a+x /tmp/wait-for-it.sh &&
-         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
       - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}
@@ -128,17 +85,7 @@ services:
     environment:
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
-      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
-      - SPRING_DATASOURCE_USERNAME=root
-      - SPRING_DATASOURCE_PASSWORD=rootpw
-      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
-    entrypoint: >
-      /bin/sh -c "
-         apt-get update && apt-get install --no-install-recommends -y wget &&
-         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
-         chmod a+x /tmp/wait-for-it.sh &&
-         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
-      - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}
+      - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}