Update k8s deployment files

Author: scdf-build-bot

src/docker-compose/docker-compose-dood.yml

@@ -40,10 +40,10 @@ services:
       - SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
 
       # Override the CTR form maven to docker
-      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
+      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
       - SPRING_CLOUD_DATAFLOW_SERVER_URI=${DATAFLOW_URI:-http://dataflow-server:9393}
     entrypoint: >
-      /bin/sh -c "
+      bin/sh -c "
          apt-get update && apt-get install --no-install-recommends -y wget &&
          wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
          tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
@@ -65,7 +65,7 @@ services:
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_LOW=20200
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
     entrypoint: >
-      /bin/sh -c "
+      bin/sh -c "
          apt-get update && apt-get install --no-install-recommends -y wget &&
          wget --no-check-certificate -P /tmp/ https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz &&
          tar -xvf /tmp/docker-latest.tgz --directory /tmp/ &&
@@ -85,7 +85,7 @@ services:
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=docker:springcloudstream/log-sink-kafka:3.0.1';
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=docker:springcloudstream/log-sink-kafka:2.1.5.RELEASE';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-@project.version@}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}';
         echo 'Docker Stream apps imported'"
 
   app-import-task:

src/docker-compose/docker-compose-influxdb.yml

@@ -19,15 +19,13 @@ services:
         }
 
   influxdb:
-    platform: linux/amd64
     image: influxdb:1.7.7
     container_name: influxdb
     ports:
       - '8086:8086'
 
   grafana:
-    platform: linux/amd64
-    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:[email protected]@}
     container_name: grafana
     ports:
       - '3000:3000'

src/docker-compose/docker-compose-postgres.yml

@@ -4,37 +4,34 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
 services:
 
+  mariadb:
+    entrypoint: ["echo", "MariaDB service is disabled"]
+
   postgres:
-    platform: linux/amd64
     image: postgres:10
     command: postgres -c 'max_connections=300'
     container_name: dataflow-postgres
     restart: always
     environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
       POSTGRES_DB: dataflow
       POSTGRES_USER: root
       POSTGRES_PASSWORD: rootpw
     expose:
       - 5432
     ports:
-      - '5432:5432'
+      - 5432:5432
 
   dataflow-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+
   skipper-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+

src/docker-compose/docker-compose-prometheus.yml

@@ -22,7 +22,6 @@ services:
       #- SPRING_APPLICATION_JSON={"spring.jpa.properties.hibernate.generate_statistics":true}
 
   prometheus-rsocket-proxy:
-    platform: linux/amd64
     image: micrometermetrics/prometheus-rsocket-proxy:1.5.0
     container_name: prometheus-rsocket-proxy
     expose:
@@ -39,15 +38,13 @@ services:
       - micrometer.prometheus-proxy.tcp-port=7001
 
   grafana:
-    platform: linux/amd64
-    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:[email protected]@}
     container_name: grafana
     ports:
       - '3000:3000'
 
   prometheus:
-    platform: linux/amd64
-    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:[email protected]@}
     container_name: prometheus
     ports:
       - '9090:9090'

src/docker-compose/docker-compose-rabbitmq.yml

@@ -4,22 +4,21 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-rabbitmq.yml up
 services:
 
+  kafka-broker:
+    entrypoint: ["echo", "kafka-broker service is disabled"]
+
+  zookeeper:
+    entrypoint: ["echo", "zookeeper service is disabled"]
+
   rabbitmq:
-    platform: linux/amd64
     image: rabbitmq:3.7
     container_name: dataflow-rabbitmq
-    environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
     expose:
       - '5672'
 
   dataflow-server:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
 
   app-import-stream:
     command: >
@@ -28,6 +27,6 @@ services:
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:3.0.1';
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:2.1.5.RELEASE';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-@project.version@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:-@project.version@}';
         echo 'Stream apps imported'"
 

src/docker-compose/docker-compose-zipkin.yml

@@ -6,7 +6,6 @@ version: '3'
 services:
 
   zipkin-server:
-    platform: linux/amd64
     image: openzipkin/zipkin:2.23.2
     container_name: zipkin-server
     expose:

src/docker-compose/docker-compose.yml

@@ -1,4 +1,5 @@
 version: '3'
+
 # Configuration environment variables:
 # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
 # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
@@ -19,40 +20,94 @@ version: '3'
 #                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
 #                             can be reached on the ports they expose.
 services:
+  mariadb:
+    image: mariadb:10.4.22
+    container_name: dataflow-mariadb
+    environment:
+      MARIADB_DATABASE: dataflow
+      MARIADB_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  kafka-broker:
+    image: confluentinc/cp-kafka:5.5.2
+    container_name: dataflow-kafka
+    expose:
+      - "9092"
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.2
+    container_name: dataflow-kafka-zookeeper
+    expose:
+      - "2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+
   dataflow-server:
     user: root
-    platform: linux/amd64
-    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
+    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:[email protected]@}${BP_JVM_VERSION:-}
     container_name: dataflow-server
     ports:
       - "9393:9393"
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181
+
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
       # Set CLOSECONTEXTENABLED=true to ensure that the CRT launcher is closed.
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_TASK_SPRING_CLOUD_TASK_CLOSECONTEXTENABLED=true
+
       - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
+
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
+
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
     depends_on:
+      - kafka-broker
       - skipper-server
+    entrypoint: >
+      /bin/sh -c "
+         apt-get update && apt-get install --no-install-recommends -y wget &&
+         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
+         chmod a+x /tmp/wait-for-it.sh &&
+         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
       - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}
 
   app-import-stream:
-    platform: linux/amd64
     image: springcloud/baseimage:1.0.1
     container_name: dataflow-app-import-stream
     depends_on:
       - dataflow-server
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:3.0.1';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:2.1.5.RELEASE';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:[email protected]@}';
+        echo 'Maven Stream apps imported'"
 
   app-import-task:
-    platform: linux/amd64
     image: springcloud/baseimage:1.0.1
     container_name: dataflow-app-import-task
     depends_on:
@@ -65,19 +120,25 @@ services:
 
   skipper-server:
     user: root
-    platform: linux/amd64
-    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
+    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:[email protected]@}${BP_JVM_VERSION:-}
     container_name: skipper
     ports:
       - "7577:7577"
       - ${APPS_PORT_RANGE:-20000-20195:20000-20195}
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
+    entrypoint: >
+      /bin/sh -c "
+         apt-get update && apt-get install --no-install-recommends -y wget &&
+         wget --no-check-certificate -P /tmp/ https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh &&
+         chmod a+x /tmp/wait-for-it.sh &&
+         /tmp/wait-for-it.sh mariadb:3306 -- /cnb/process/web"
     restart: always
     volumes:
       - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}

src/kubernetes/kafka/kafka-deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: kafka-broker
-  namespace: kafka
   labels:
     app: kafka
     component: kafka-broker
@@ -21,7 +20,6 @@ spec:
       containers:
       - name: kafka
         image: confluentinc/cp-kafka:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9092
         env:
@@ -32,9 +30,9 @@ spec:
           - name: JDK_JAVA_OPTIONS
             value: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
           - name: KAFKA_ADVERTISED_LISTENERS
-            value: "PLAINTEXT://kafka-broker.kafka:9092"
+            value: "PLAINTEXT://kafka-broker:9092"
           - name: KAFKA_ZOOKEEPER_CONNECT
-            value: kafka-zk.kafka:2181
+            value: kafka-zk:2181
           - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
             value: "1"
           - name: KAFKA_LOG4J_ROOT_LOGLEVEL

src/kubernetes/kafka/kafka-svc.yaml

@@ -2,11 +2,11 @@ apiVersion: v1
 kind: Service
 metadata:
   name: kafka-broker
-  namespace: kafka
   labels:
     app: kafka
     component: kafka-broker
 spec:
+  clusterIP: None
   ports:
   - port: 9092
     name: kafka-port

src/kubernetes/kafka/kafka-zk-deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kafka-zk
-  namespace: kafka
   labels:
     app: kafka
     component: kafka-zk
@@ -20,7 +19,6 @@ spec:
       containers:
       - name: kafka-zk
         image: confluentinc/cp-zookeeper:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 2181
         env:

src/kubernetes/kafka/kafka-zk-svc.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Service
 metadata:
   name: kafka-zk
-  namespace: kafka
   labels:
     app: kafka
     component: kafka-zk

src/kubernetes/mariadb/mariadb-deployment.yaml

@@ -16,16 +16,17 @@ spec:
     spec:
       containers:
       - image: mariadb:10.4.22
-        imagePullPolicy: IfNotPresent
         name: mariadb
         env:
+          - name: LANG
+            value: en_US.utf8
+          - name: LC_ALL
+            value: en_US.utf8
           - name: MARIADB_ROOT_PASSWORD
             valueFrom:
               secretKeyRef:
                 key: database-password
                 name: mariadb
-          - name: MARIADB_DATABASE
-            value: 'dataflow'
         ports:
           - containerPort: 3306
             name: mariadb

src/kubernetes/prometheus-proxy/prometheus-proxy-clusterrolebinding.yaml

@@ -7,7 +7,7 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: prometheus-rsocket-proxy
-    namespace: prometheus
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: cluster-admin