[CI] Update carvel documentation

Author: Corneil du Plessis

spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc

@@ -1,7 +1,8 @@
-:source-highlighter: rouge
 [[configuration-carvel]]
 == Deployment using Carvel
 
+Deployment of a carvel package requires the installation of tools and specific Kubernetes controllers. Then you will add the package repository to the cluster and install the application.
+
 == Required Tools
 
 * `kubectl` - Kubernetes CLI (Install with `brew install kubectl`)
@@ -37,8 +38,8 @@ These scripts assume you are connected to a Kubernetes cluster and `kubectl` is
 |Name | Arguments |Descriptions
 
 | start-deploy.sh
-| <broker> [scdf-type] [namespace]
-| Configures environmental variables needs for the rest of the scripts. `BROKER`, `NS` and `SCDF_TYPE` are set. The default `NS` is `scdf`. The namespace will be created if it doesn't exist by `setup-scdf-repo.sh`. The default `SCDF_TYPE` is `oss`
+| <broker> [scdf-type] [namespace] [release\|snapshot]
+| Configures environmental variables needs for the rest of the scripts. `BROKER`, `NS` and `SCDF_TYPE` are set. The default `NS` is `scdf`. The namespace will be created if it doesn't exist by `setup-scdf-repo.sh`. The default `SCDF_TYPE` is `oss`. _release\|snapshot_ and _scdf-type_ will determine the value of `PACKAGE_VERSION` set.
 
 | prepare-cluster.sh
 | N/A
@@ -95,14 +96,15 @@ Executing the following script will configure the environmental variables needed
 
 [source,shell]
 ....
-source ./start-deploy.sh <broker> <namespace> [scdf-type]
+source ./start-deploy.sh <broker> <namespace> [scdf-type] [release\|snapshot]
 ....
 
 Where:
 
 * `broker` is one of rabbitmq or kafka
 * `namespace` A valid Kubernetes namespace other than `default`
 * `scdf-type` One of oss or pro. oss is the default.
+* `release\|snapshot` and `scdf-type` will determine the value of `PACKAGE_VERSION`.
 
 
 The environmental variables can also be configured manually to override the values.

spring-cloud-dataflow-package/pom.xml

@@ -16,6 +16,20 @@
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-assembly-plugin</artifactId>
 				<version>3.5.0</version>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+						<configuration>
+							<appendAssemblyId>false</appendAssemblyId>
+							<descriptors>
+								<descriptor>src/main/assembly/zip.xml</descriptor>
+							</descriptors>
+						</configuration>
+					</execution>
+				</executions>
 			</plugin>
 		</plugins>
 	</build>

spring-cloud-dataflow-package/scr/assembly/assembly.xml

@@ -0,0 +1,22 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.1"
+		  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		  xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.1 https://maven.apache.org/xsd/assembly-2.1.1.xsd">
+	<id>distribution</id>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<formats>
+		<format>tar.gz</format>
+	</formats>
+	<fileSets>
+		<fileSet>
+			<directory>${project.basedir}/../src/deploy</directory>
+			<outputDirectory>deploy</outputDirectory>
+			<excludes>
+				<exclude>*.adoc</exclude>
+			</excludes>
+		</fileSet>
+		<fileSet>
+			<directory>${project.basedir}/../src/kubernetes</directory>
+			<outputDirectory>deploy/kubernetes</outputDirectory>
+		</fileSet>
+	</fileSets>
+</assembly>

spring-cloud-dataflow-package/src/main/assembly/zip.xml

@@ -1,6 +1,6 @@
 = Deployment to Kubernetes
 
-This document describes how to use the Carvel packages to deploy Spring Cloud Data Flow, and it also has a section on creating local development cluster using minikube or kind.
+This document describes how to use the Carvel packages to deploy Spring Cloud Data Flow, and it also has a xref:#local-k8s-development[section] on creating local development cluster using minikube or kind.
 
 include::../../spring-cloud-dataflow-docs/src/main/asciidoc/configuration-carvel.adoc[]
 

src/deploy/README.adoc

@@ -30,7 +30,7 @@ check_env NS
 kubectl create namespace $NS
 kubectl create namespace secrets-ns
 $SCDIR/add-roles.sh "system:aggregate-to-edit" "system:aggregate-to-admin" "system:aggregate-to-view"
-PRESENT=$(kubectl get serviceaccount --namespace $NS --output=json | count_kind serviceaccount "$NS-sa")
+PRESENT=$(kubectl get serviceaccount --namespace $NS --output=json | count_kind serviceaccount "$SA")
 if ((PRESENT > 0)); then
     kubectl delete serviceaccount "$SA" --namespace $NS
 fi

src/deploy/README.html

@@ -2,7 +2,6 @@
 if [ "$DEBUG" == "true" ]; then
     echo "DEBUG: configure-database.sh $*"
 fi
-
 SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 if [ "$4" = "" ]; then
     echo "<app> <database> <url> <username/secret-name> [password/secret-username-key] [secret-password-key]"

src/deploy/README.pdf

@@ -12,25 +12,21 @@ function check_env() {
   fi
 }
 check_env NS
-
-if [ "$SCDF_TYPE" == "" ]; then
-    SCDF_TYPE=pro
-fi
+check_env PACKAGE_VERSION
 if [ "$1" != "" ]; then
   SCDF_TYPE=$1
 fi
+check_env SCDF_TYPE
 
 case $SCDF_TYPE in
 "pro")
   APP_NAME=scdf-pro-app
-  PACKAGE_VERSION=1.5.3-SNAPSHOT
   PACKAGE_NAME=scdf-pro.tanzu.vmware.com
   REGISTRY=dev.registry.pivotal.io
   REPO_NAME="p-scdf-for-kubernetes/scdf-pro-repo"
   ;;
 "oss")
   APP_NAME=scdf-oss-app
-  PACKAGE_VERSION=2.11.0-SNAPSHOT
   PACKAGE_NAME=scdf.tanzu.vmware.com
   REGISTRY=index.docker.io
   REPO_NAME="springcloud/scdf-oss-repo"

src/deploy/carvel/carvel-prepare-namespaces.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 function count_kind() {
     jq --arg kind $1 --arg name $2 '.items | .[] | select(.kind == $kind) | .metadata | select(.name == $name) | .name' | grep -c -F "$2"
 }
@@ -22,8 +25,10 @@ case $BROKER in
     exit 1
     ;;
 esac
-
-K8S=$(realpath $SCDIR/../../kubernetes)
+K8S=$(realpath $SCDIR/../kubernetes)
+if [ ! -d "$K8S" ]; then
+  K8S=$(realpath $SCDIR/../../kubernetes)
+fi
 $SCDIR/prepare-local-namespace.sh "$BROKER-sa" $BROKER
 kubectl create --namespace $BROKER -f $K8S/$BROKER/
 if [ "$BROKER" = "rabbitmq" ]; then
@@ -39,7 +44,7 @@ export BROKER
 yq ".scdf.binder.type=\"$BINDER_NAME\"" -i ./scdf-values.yml
 
 if [ "$BROKER" = "rabbitmq" ]; then
-    yq ".scdf.binder.rabbit.host=\"rabbitmq\"" -i ./scdf-values.yml
+    yq ".scdf.binder.rabbit.host=\"rabbitmq.rabbitmq\"" -i ./scdf-values.yml
     yq ".scdf.binder.rabbit.port=5672" -i ./scdf-values.yml
 else
     yq ".scdf.binder.type=\"kafka\"" -i ./scdf-values.yml

src/deploy/carvel/configure-database.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 start_time=$(date +%s)
 if [ "$1" = "" ]; then
@@ -18,7 +21,10 @@ case $1 in
     ;;
 esac
 
-K8S=$(realpath $SCDIR/../../kubernetes)
+K8S=$(realpath $SCDIR/../kubernetes)
+if [ ! -d "$K8S" ]; then
+  K8S=$(realpath $SCDIR/../../kubernetes)
+fi
 set +e
 $SCDIR/prepare-local-namespace.sh "$DATABASE-sa" $DATABASE
 

src/deploy/carvel/delete-scdf.sh

@@ -1,7 +1,13 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 start_time=$(date +%s)
-K8S=$(realpath $SCDIR/../../kubernetes)
+K8S=$(realpath $SCDIR/../kubernetes)
+if [ ! -d "$K8S" ]; then
+  K8S=$(realpath $SCDIR/../../kubernetes)
+fi
 
 $SCDIR/prepare-local-namespace.sh prometheus prometheus
 kubectl create serviceaccount prometheus-rsocket-proxy --namespace prometheus

src/deploy/carvel/deploy-local-broker.sh

@@ -1,11 +1,27 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
+function check_env() {
+  eval ev='$'$1
+  if [ "$ev" == "" ]; then
+    echo "env var $1 not defined"
+    if ((sourced != 0)); then
+      return 1
+    else
+      exit 1
+    fi
+  fi
+}
+if [ "$SCDF_TYPE" == "" ]; then
+    echo "Environmental variable SCDF_TYPE must be set to one of oss or pro."
+fi
+check_env NS
+check_env PACKAGE_VERSION
 SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 start_time=$(date +%s)
 # the following names are your choice.
-if [ "$NS" = "" ]; then
-    echo "Expected env var NS"
-    exit 1
-fi
+
 COUNT=$(kubectl get namespace $NS | grep -c "$NS")
 if ((COUNT == 0)); then
     echo "Expected namespace $NS"
@@ -14,23 +30,13 @@ else
     echo "Namespace $NS exists"
 fi
 
-if [ "$SCDF_TYPE" == "" ]; then
-    echo "SCDF_TYPE must be set to one of oss or pro."
-fi
-
 case $SCDF_TYPE in
 "pro")
     APP_NAME=scdf-pro-app
-    if [ "$PACKAGE_VERSION" = "" ]; then
-        PACKAGE_VERSION=1.6.0-SNAPSHOT
-    fi
     PACKAGE_NAME=scdf-pro.tanzu.vmware.com
     ;;
 "oss")
     APP_NAME=scdf-oss-app
-    if [ "$PACKAGE_VERSION" = "" ]; then
-        PACKAGE_VERSION=2.11.0-SNAPSHOT
-    fi
     PACKAGE_NAME=scdf.tanzu.vmware.com
     ;;
 *)

src/deploy/carvel/deploy-local-database.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 CM_VERSION=v1.11.1
 SGC_VER=v0.14.2
 KC_VER=v0.45.0

src/deploy/carvel/deploy-local-prometheus.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 start_time=$(date +%s)
 function register_app() {

src/deploy/carvel/deploy-scdf.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 function check_env() {
     eval ev='$'$1
     if [ "$ev" == "" ]; then
@@ -16,6 +19,7 @@ start_time=$(date +%s)
 # the following names are your choice.
 check_env NS
 check_env SCDF_TYPE
+check_env PACKAGE_VERSION
 check_env DOCKER_HUB_USERNAME
 check_env DOCKER_HUB_PASSWORD
 
@@ -24,9 +28,6 @@ $SCDIR/carvel-prepare-namespaces.sh $NS
 
 case $SCDF_TYPE in
 "pro")
-    if [ "$PACKAGE_VERSION" = "" ]; then
-        PACKAGE_VERSION=1.6.0-SNAPSHOT
-    fi
     PACKAGE_NAME=scdf-pro.tanzu.vmware.com
     if [ "$PACKAGE_REPO" = "" ]; then
         PACKAGE_REPO="dev.registry.pivotal.io/p-scdf-for-kubernetes"
@@ -36,9 +37,6 @@ case $SCDF_TYPE in
     fi
     ;;
 "oss")
-    if [ "$PACKAGE_VERSION" = "" ]; then
-        PACKAGE_VERSION=2.11.0-SNAPSHOT
-    fi
     PACKAGE_NAME=scdf.tanzu.vmware.com
     if [ "$PACKAGE_REPO" = "" ]; then
         PACKAGE_REPO="index.docker.io/springcloud"

src/deploy/carvel/prepare-cluster.sh

@@ -1,22 +1,43 @@
 #!/usr/bin/env bash
+bold="\033[1m"
+dim="\033[2m"
+end="\033[0m"
 (return 0 2>/dev/null) && sourced=1 || sourced=0
 if [ "$sourced" = "0" ]; then
     echo "This script must be invoked using: source $0 $*"
     exit 1
 fi
 if [ "$1" = "" ]; then
-    echo "Usage: <broker> [scdf-type] [namespace]"
+    echo "Usage: <broker> [scdf-type] [namespace] [release|snapshot]"
     echo "Where:"
     echo "  broker is one of kafka or rabbitmq"
     echo "  scdf-type is one of oss or pro. The default is 'oss'"
     echo "  namespace is a valid k8s namespace other than 'default'. The default is 'scdf'."
+    echo "  release or snapshot and scdf-type will determine PACKAGE_VERSION. The default is latest snapshot."
     return 0
 fi
-DATABASE=postgresql
+
 SCDF_TYPE=oss
 NS=scdf
+
 while [ "$1" != "" ]; do
     case $1 in
+    "snapshot")
+        if [ "$SCDF_TYPE" = "oss" ]; then
+            PACKAGE_VERSION=2.11.0-SNAPSHOT
+        else
+            PACKAGE_VERSION=1.6.0-SNAPSHOT
+        fi
+        export PACKAGE_VERSION
+        ;;
+    "release")
+        if [ "$SCDF_TYPE" = "oss" ]; then
+            PACKAGE_VERSION=2.10.3
+        else
+            PACKAGE_VERSION=1.5.3
+        fi
+        export PACKAGE_VERSION
+        ;;
     "rabbitmq" | "rabbit")
         BROKER=rabbitmq
         ;;
@@ -44,6 +65,13 @@ fi
 export BROKER
 export SCDF_TYPE
 export NS
+
 echo "Broker: $BROKER"
 echo "SCDF Type: $SCDF_TYPE"
+if [ "$PACKAGE_VERSION" = "" ]; then
+    echo "Package version: N/A"
+    echo -e "${bold}Configure environmental variable PACKAGE_VERSION to specific value.${end}"
+else
+    echo "Package version: $PACKAGE_VERSION"
+fi
 echo "NS: $NS"