Update k8s deployment files

Author: scdf-build-bot

src/docker-compose/docker-compose-dood.yml

@@ -40,7 +40,7 @@ services:
       - SPRING_CLOUD_DATAFLOW_TASK_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_DOCKER_PORTRANGE_HIGH=61000
 
       # Override the CTR form maven to docker
-      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
+      - SPRING_CLOUD_DATAFLOW_TASK_COMPOSEDTASKRUNNER_URI=docker://springcloud/spring-cloud-dataflow-composed-task-runner:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
       - SPRING_CLOUD_DATAFLOW_SERVER_URI=${DATAFLOW_URI:-http://dataflow-server:9393}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
@@ -64,15 +64,13 @@ services:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';               
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-docker-latest&force=true}';
+                wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-@project.version@}' --no-check-certificate --post-data='uri=docker:springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}';
         echo 'Docker Stream apps imported'"
 
   app-import-task:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate  --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-docker-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-task:3.0.0';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=docker:springcloudtask/timestamp-batch-task:3.0.0';
         echo 'Docker Task apps imported'"

src/docker-compose/docker-compose-influxdb.yml

@@ -19,13 +19,13 @@ services:
         }
 
   influxdb:
-    image: influxdb:1.8
+    image: influxdb:1.7.7
     container_name: influxdb
     ports:
       - '8086:8086'
 
   grafana:
-    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-influxdb:${DATAFLOW_VERSION:-@project.version@}
     container_name: grafana
     ports:
       - '3000:3000'

src/docker-compose/docker-compose-postgres.yml

@@ -4,36 +4,34 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
 services:
 
+  mariadb:
+    entrypoint: ["echo", "MariaDB service is disabled"]
+
   postgres:
     image: postgres:12
     command: postgres -c 'max_connections=300'
     container_name: dataflow-postgres
     restart: always
     environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
       POSTGRES_DB: dataflow
       POSTGRES_USER: root
       POSTGRES_PASSWORD: rootpw
     expose:
       - 5432
     ports:
-      - '5432:5432'
+      - 5432:5432
 
   dataflow-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+
   skipper-server:
     environment:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-    depends_on:
-      - postgres
+

src/docker-compose/docker-compose-prometheus.yml

@@ -38,13 +38,13 @@ services:
       - micrometer.prometheus-proxy.tcp-port=7001
 
   grafana:
-    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:-@project.version@}
     container_name: grafana
     ports:
       - '3000:3000'
 
   prometheus:
-    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}
+    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:-@project.version@}
     container_name: prometheus
     ports:
       - '9090:9090'

src/docker-compose/docker-compose-rabbitmq.yml

@@ -4,33 +4,29 @@ version: '3'
 # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-rabbitmq.yml up
 services:
 
+  kafka-broker:
+    entrypoint: ["echo", "kafka-broker service is disabled"]
+
+  zookeeper:
+    entrypoint: ["echo", "zookeeper service is disabled"]
+
   rabbitmq:
-    image: rabbitmq:3.8
+    image: rabbitmq:3.7
     container_name: dataflow-rabbitmq
-    environment:
-      LANG: en_US.utf8
-      LC_ALL: en_US.utf8
-      JDK_JAVA_OPTIONS: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
     expose:
       - '5672'
 
   dataflow-server:
     environment:
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
-
-  skipper-server:
-    environment:
-      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_RABBITMQ_HOST=rabbitmq
-    depends_on:
-      - rabbitmq
 
   app-import-stream:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:3.0.1';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-rabbit:2.1.5.RELEASE';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-rabbit:${DATAFLOW_VERSION:[email protected]@}';
         echo 'Stream apps imported'"
 

src/docker-compose/docker-compose-zipkin.yml

@@ -6,7 +6,7 @@ version: '3'
 services:
 
   zipkin-server:
-    image: openzipkin/zipkin:2
+    image: openzipkin/zipkin:2.23.2
     container_name: zipkin-server
     expose:
       - '9411'

src/docker-compose/docker-compose.yml

@@ -1,4 +1,5 @@
 version: '3'
+
 # Configuration environment variables:
 # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
 # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
@@ -19,25 +20,65 @@ version: '3'
 #                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
 #                             can be reached on the ports they expose.
 services:
+  mariadb:
+    image: mariadb:11
+    container_name: dataflow-mariadb
+    environment:
+      MARIADB_DATABASE: dataflow
+      MARIADB_ROOT_PASSWORD: rootpw
+    expose:
+      - 3306
+
+  kafka-broker:
+    image: confluentinc/cp-kafka:5.5.2
+    container_name: dataflow-kafka
+    expose:
+      - "9092"
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_ADVERTISED_HOST_NAME=kafka-broker
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.5.2
+    container_name: dataflow-kafka-zookeeper
+    expose:
+      - "2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+
   dataflow-server:
     user: root
-    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
+    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-@project.version@}${BP_JVM_VERSION:-}
     container_name: dataflow-server
     ports:
       - "9393:9393"
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181
+      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
       # Set CLOSECONTEXTENABLED=true to ensure that the CRT launcher is closed.
       - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_TASK_SPRING_CLOUD_TASK_CLOSECONTEXTENABLED=true
       - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
+
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
       - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
     depends_on:
+      - kafka-broker
       - skipper-server
     restart: always
     volumes:
@@ -48,6 +89,12 @@ services:
     container_name: dataflow-app-import-stream
     depends_on:
       - dataflow-server
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 360 dataflow-server:9393;
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
+        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/dataflow-tasklauncher/${DATAFLOW_VERSION:[email protected]@}' --no-check-certificate --post-data='uri=maven://org.springframework.cloud:spring-cloud-dataflow-tasklauncher-sink-kafka:${DATAFLOW_VERSION:[email protected]@}';
+        echo 'Maven Stream apps imported'"
 
   app-import-task:
     image: springcloud/baseimage:1.0.4
@@ -58,24 +105,22 @@ services:
       /bin/sh -c "
         ./wait-for-it.sh -t 360 dataflow-server:9393;
         wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-task:3.0.0';
-        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/timestamp-batch3' --no-check-certificate --post-data='bootVersion=3&uri=maven://uri=maven:io.spring:timestamp-batch:3.0.0';
         echo 'Maven Task apps imported'"
 
   skipper-server:
     user: root
-    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.0-SNAPSHOT}${BP_JVM_VERSION:-}
-    container_name: skipper-server
+    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-@spring-cloud-skipper.version@}${BP_JVM_VERSION:-}
+    container_name: skipper
     ports:
       - "7577:7577"
       - ${APPS_PORT_RANGE:-20000-20195:20000-20195}
     environment:
-      - LANG=en_US.utf8
-      - LC_ALL=en_US.utf8
-      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
-      - SERVER_PORT=7577
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
       - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
+      - SPRING_DATASOURCE_URL=jdbc:mariadb://mariadb:3306/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
       - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
     restart: always
     volumes:

src/kubernetes/grafana/grafana-configmap.yaml

@@ -12,7 +12,7 @@ data:
       type: prometheus
       access: proxy
       org_id: 1
-      url: http://localhost:9090
+      url: http://prometheus:9090
       is_default: true
       version: 5
       editable: true

src/kubernetes/kafka/kafka-deployment.yaml

@@ -20,7 +20,6 @@ spec:
       containers:
       - name: kafka
         image: confluentinc/cp-kafka:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9092
         env:
@@ -31,9 +30,9 @@ spec:
           - name: JDK_JAVA_OPTIONS
             value: '-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8'
           - name: KAFKA_ADVERTISED_LISTENERS
-            value: "PLAINTEXT://kafka-broker.kafka:9092"
+            value: "PLAINTEXT://kafka-broker:9092"
           - name: KAFKA_ZOOKEEPER_CONNECT
-            value: kafka-zk.kafka:2181
+            value: kafka-zk:2181
           - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
             value: "1"
           - name: KAFKA_LOG4J_ROOT_LOGLEVEL

src/kubernetes/kafka/kafka-svc.yaml

@@ -6,6 +6,7 @@ metadata:
     app: kafka
     component: kafka-broker
 spec:
+  clusterIP: None
   ports:
   - port: 9092
     name: kafka-port

src/kubernetes/kafka/kafka-zk-deployment.yaml

@@ -19,7 +19,6 @@ spec:
       containers:
       - name: kafka-zk
         image: confluentinc/cp-zookeeper:5.5.2
-        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 2181
         env:

src/kubernetes/mariadb/mariadb-deployment.yaml

@@ -16,16 +16,17 @@ spec:
     spec:
       containers:
       - image: mariadb:11
-        imagePullPolicy: IfNotPresent
         name: mariadb
         env:
+          - name: LANG
+            value: en_US.utf8
+          - name: LC_ALL
+            value: en_US.utf8
           - name: MARIADB_ROOT_PASSWORD
             valueFrom:
               secretKeyRef:
                 key: database-password
                 name: mariadb
-          - name: MARIADB_DATABASE
-            value: 'dataflow'
         ports:
           - containerPort: 3306
             name: mariadb

src/kubernetes/prometheus-proxy/prometheus-proxy-clusterrolebinding.yaml

@@ -7,6 +7,7 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: prometheus-rsocket-proxy
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: cluster-admin

src/kubernetes/prometheus-proxy/prometheus-proxy-deployment.yaml

@@ -31,9 +31,6 @@ spec:
             requests:
               cpu: 0.5
               memory: 1024Mi
-          env:
-            - name: LOGGING_LEVEL_IO_MICROMETER
-              value: DEBUG
       securityContext:
         fsGroup: 2000
         runAsNonRoot: true

src/kubernetes/prometheus-proxy/prometheus-proxy-serviceaccount.yaml

@@ -4,4 +4,4 @@ metadata:
   name: prometheus-rsocket-proxy
   labels:
     app: prometheus-rsocket-proxy
-
+  namespace: default

src/kubernetes/prometheus/prometheus-clusterrolebinding.yaml

@@ -11,4 +11,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: prometheus
-
+  namespace: default