Improve deployment scripts (#5725)

[skip ci]

Author: Corneil du Plessis

spring-cloud-dataflow-docs/src/main/asciidoc/appendix-extend-classpath.adoc

@@ -6,52 +6,42 @@ Users may require the addition of dependencies to the existing Stream applicatio
 
 NOTE: The Spring Cloud Dataflow repository contains scripts to help with this task. The examples below assume you have cloned the `spring-cloud-dataflow` repository and are executing the scripts from `src/templates/add-deps`.
 
-
-==== Containers
-
-In order to add dependencies to existing container the following steps will be the approach.
-
-* Create a folder with the extra dependencies.
-* Create a new container image while copying the files to the libraries folder.
-* Push the image to a private registry.
-
-===== Environmental variables
-
-* `DEPS_FOLDER` should be a full filename or path expression for files to copy to the container.
-* `CONTAINER_REPO` the source docker image name.
-* `CONTAINER_TAG` the tag of source image.
-* `PRIVATE_REGISTRY` the host name of the private registry.
-
-===== Examples
-
-[source,shell]
-....
-export CONTAINER_REPO="springcloud/spring-cloud-dataflow-server"
-export CONTAINER_TAG="2.9.5-jdk17"
-export PRIVATE_REGISTRY="our.private.registry"
-export DEPS_FOLDER="./extra-libs/"
-docker build -f Dockerfile -t "$PRIVATE_REGISTRY/$CONTAINER_REPO:$CONTAINER_TAG"
-docker push "$PRIVATE_REGISTRY/$CONTAINER_REPO:$CONTAINER_TAG"
-....
-
-NOTE: As pointed out above, the `Dockerfile` lives in the `spring-cloud-dataflow` repository under `src/add-deps`.
-
 ==== JAR File
 
-When using CloudFoundry or local deployment you will need to update jar before publishing it to a private registry or Maven Local.
+_We suggest you publish the updated jar it to a private Maven repository and that the Maven Coordinates of the private registry is then used to register application with SCDF._
 
 ===== Example
 
-This example adds the dependencies and then installs the jar to Maven local.
+This example:
+* assumes the jar is downloaded to `${appFolder}/${appName}-${appVersion}.jar`
+* adds the dependencies and then publishes the jar to Maven local.
 
 [source,shell]
 ....
 ./gradlew -i publishToMavenLocal \
     -P appFolder="." \
     -P appGroup="org.springframework.cloud" \
     -P appName="spring-cloud-dataflow-server" \
-    -P appVersion="2.9.5" \
+    -P appVersion="2.11.2" \
     -P depFolder="./extra-libs"
 ....
 
 NOTE: Use the `publishMavenPublicationToMavenRepository` task to publish to a remote repository. Update the `gradle.properties` with the remote repository details. Alternatively move `repoUser` and `repoPassword` to ~/.gradle/gradle.properties
+
+==== Containers
+
+In order to create a container we suggest using https://buildpacks.io/docs/for-platform-operators/how-to/integrate-ci/pack[paketo pack cli] to create a container from the jar created in previous step.
+
+[source, shell]
+....
+REPO=springcloud/spring-cloud-dataflow-server
+TAG=2.11.2
+JAR=build/spring-cloud-dataflow-server-${TAG}.jar
+JAVA_VERSION=8
+pack build --builder gcr.io/paketo-buildpacks/builder:base \
+            --path "$JAR" \
+            --trust-builder --verbose \
+            --env BP_JVM_VERSION=${JAVA_VERSION} "$REPO:$TAG-jdk${JAVA_VERSION}-extra"
+....
+
+NOTE: Publish the container to a private container registry and register the application docker uri with SCDF.

src/add-deps/Dockerfile

@@ -10,7 +10,7 @@ task combinedJar(type: Jar) {
     into('BOOT-INF/lib') {
         from file(depsFolder)
     }
-    archiveFileName = "${appName}.jar"
+    archiveFileName = "${appName}-${appVersion}.jar"
     destinationDirectory = project.buildDir
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
 }

src/add-deps/build.gradle

@@ -1,6 +1,6 @@
 appGroup=org.springframework.cloud
 appName=spring-cloud-dataflow-server
-appVersion=2.10.2
+appVersion=2.11.2
 appFile=spring-cloud-dataflow-server.jar
 appFolder=.
 repoUrl=https://my.private.repo

src/add-deps/gradle.properties

@@ -42,6 +42,9 @@ case $2 in
 "mysql57")
     DATABASE=mysql57
     ;;
+"oracle")
+    DATABASE=oracle
+    ;;
 *)
     echo "Unsupported or invalid database $2"
     exit 1
@@ -62,6 +65,9 @@ case $DATABASE in
 "postgresql")
     JDBC_DRIVER_CLASS=org.postgresql.Driver
     ;;
+"oracle")
+    JDBC_DRIVER_CLASS=oracle.jdbc.OracleDriver
+    ;;
 *)
     echo "Unsupported $DATABASE."
     ;;

src/deploy/carvel/configure-database.sh

@@ -12,13 +12,22 @@ if [ "$1" = "" ]; then
     echo "<database> must be provided. Choose one of postgresql or mariadb"
     exit 1
 fi
+JDBC_URL="jdbc:$DATABASE://$DATABASE.$DATABASE/dataflow"
 case $1 in
 "postgresql" | "postgres")
     DATABASE=postgresql
     ;;
 "mariadb" | "maria")
     DATABASE=mariadb
     ;;
+#"oracle")
+#    DATABASE=oracle
+#    JDBC_URL="jdbc:oracle:thin:@oracle.oracle:1521"
+#    ;;
+"mysql57")
+    DATABASE=mysql57
+    JDBC_URL="jdbc:mysql://$DATABASE.$DATABASE/dataflow?permitMysqlScheme"
+    ;;
 *)
     echo "Unsupported or invalid database $1"
     exit 1
@@ -37,7 +46,6 @@ set -e
 kubectl rollout status deployment --namespace "$DATABASE" $DATABASE
 set +e
 
-JDBC_URL="jdbc:$DATABASE://$DATABASE.$DATABASE/dataflow"
 "$SCDIR/configure-database.sh" dataflow $DATABASE "$JDBC_URL" $DATABASE database-username database-password
 "$SCDIR/configure-database.sh" skipper $DATABASE "$JDBC_URL" $DATABASE database-username database-password
 export DATABASE

src/deploy/carvel/deploy-local-database.sh

@@ -58,7 +58,7 @@ case "$K8S_DRIVER" in
     echo "Creating Minikube cluster with $K8S_DRIVER and k8s=$K8S_VERSION"
     # K8S_DRIVER=kvm2, docker, vmware, virtualbox, podman, vmwarefusion or hyperkit
     if [ "$MK_ARGS" = "" ]; then
-        MK_ARGS="--cpus=8 --memory=12g"
+        MK_ARGS="--cpus=8 --memory=16g --disk-size=50g"
     fi
     minikube start $MK_ARGS "--driver=$K8S_DRIVER" "--kubernetes-version=$K8S_VERSION"
     echo -e "Please run ${bold}'minikube tunnel'${end} in a separate shell to ensure a LoadBalancer is active."

src/deploy/k8s/configure-k8s.sh

@@ -91,7 +91,11 @@ if [ "$K8S_DRIVER" != "tmc" ] && [ "$K8S_DRIVER" != "gke" ] ; then
     else
         echo "Loading:$IMAGE:$DOCKER_IDS"
     fi
-    minikube image load --pull=$PULL "$IMAGE"
+    OPTIONS="--overwrite true --daemon true --pull $PULL"
+    if [ "$PULL" == "false" ]; then
+        OPTIONS="$OPTIONS --remote false"
+    fi
+    minikube image load "$IMAGE" $OPTIONS
     MK_IDS=$(minikube image ls --format table | grep -F "$NAME" | grep -F "$TAG" | awk '{print $6}')
     for did in $DOCKER_IDS; do
       for mid in $MK_IDS; do

src/deploy/k8s/load-image.sh

@@ -10,5 +10,5 @@ scdf-type:
     maintenance-snapshot: '1.5.4-SNAPSHOT'
 default:
   scdf-type: 'oss'
-  version: 'snapshot'
-  package-version: '2.11.3-SNAPSHOT'
+  version: 'release'
+  package-version: '2.11.2'