Use staging Devfile registry for PR tests (#7129)

rm3l · web-flow · commit 5d4a6ef32551 · 2023-10-27T07:54:14.000Z
* Revert using a DEVFILE_PROXY env var

There is no proxy deployed in the internal test cluster.
As such, this env var no longer makes sense.

* To help troubleshoot, display the resolved Devfile registry URL

* Make interactive tests more resilient with stack versions

They are now able to determine if the "Select version" prompt
should be asked by "odo init" or not:

* Make sure doc automation tests do not rely on hard-coded namespaces

* Allow to run doc automation tests with more parallel Ginkgo nodes

This is possible now that those tests no longer
depend on a single hard-coded namespace.

* Remove occurrences of the DEVFILE_REGISTRY env var in IBM Pipelines scripts

* Reuse logic for determining the Devfile Registry URL in "odo registry" tests

* Clarify what openshiftci-config.sh is used for
diff --git a/.ibm/pipelines/kubernetes-tests.sh b/.ibm/pipelines/kubernetes-tests.sh
@@ -15,8 +15,6 @@ cleanup_namespaces
 export SKIP_USER_LOGIN_TESTS=true
 (
     set -e
-    export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' || true)"
-    echo Using Devfile proxy: ${DEVFILE_PROXY}
     make install
     make test-integration-cluster
 ) |& tee "/tmp/${LOGFILE}"
diff --git a/.ibm/pipelines/nocluster-tests.sh b/.ibm/pipelines/nocluster-tests.sh
@@ -13,7 +13,6 @@ ibmcloud target -r "${IBM_REGION}"
 (
     set -e
     make install
-    echo Using Devfile Registry ${DEVFILE_REGISTRY}
     make test-integration-no-cluster
 ) |& tee "/tmp/${LOGFILE}"
 
diff --git a/.ibm/pipelines/openshift-tests.sh b/.ibm/pipelines/openshift-tests.sh
@@ -16,8 +16,6 @@ cleanup_namespaces
 
 (
     set -e
-    export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' || true)"
-    echo Using Devfile proxy: ${DEVFILE_PROXY}
     make install
     make test-integration-cluster
     make test-e2e
diff --git a/.ibm/pipelines/windows-test-script.ps1 b/.ibm/pipelines/windows-test-script.ps1
@@ -60,7 +60,6 @@ function Run-Test {
     [Environment]::SetEnvironmentVariable("SKIP_SERVICE_BINDING_TESTS","$SKIP_SERVICE_BINDING_TESTS" )
     # Integration tests detecting key press when running DevSession are not working on Windows
     [Environment]::SetEnvironmentVariable("SKIP_KEY_PRESS","true")
-    [Environment]::SetEnvironmentVariable("DEVFILE_REGISTRY", "$DEVFILE_REGISTRY") 
 
     Shout "Login IBMcloud"
     ibmcloud login --apikey ${API_KEY}
@@ -71,14 +70,6 @@ function Run-Test {
     oc login -u apikey -p ${API_KEY} ${IBM_OPENSHIFT_ENDPOINT}
     Check-ExitCode $LASTEXITCODE
 
-    Shout "Getting Devfile proxy address"
-    $DEVFILE_PROXY=$(oc get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
-    if ( $LASTEXITCODE -eq 0 )
-    {
-        Shout "Using Devfile proxy: $DEVFILE_PROXY"
-        [Environment]::SetEnvironmentVariable("DEVFILE_PROXY", "$DEVFILE_PROXY")
-    }
-
     Shout "Create Binary"
     make install 
     Shout "Running test"
@@ -109,8 +100,7 @@ $LOGFILE=$args[4]
 $REPO=$args[5]
 $CLUSTER_ID=$args[6]
 $TEST_EXEC_NODES=$args[7]
-$DEVFILE_REGISTRY=$args[8]
-$SKIP_SERVICE_BINDING_TESTS=$args[9]
+$SKIP_SERVICE_BINDING_TESTS=$args[8]
 Shout "Args Recived"
 
 
diff --git a/.ibm/pipelines/windows-test.sh b/.ibm/pipelines/windows-test.sh
@@ -18,7 +18,7 @@ sshpass -p $WINDOWS_PASSWORD scp -o StrictHostKeyChecking=no ./.ibm/pipelines/wi
 
 #execute test from the test script
 export TEST_EXEC_NODES=${TEST_EXEC_NODES:-"16"}
-sshpass -p $WINDOWS_PASSWORD ssh Administrator@$WINDOWS_IP -o StrictHostKeyChecking=no powershell /tmp/windows-test-script.ps1 "${GIT_PR_NUMBER}" "${BUILD_NUMBER}" "${API_KEY_QE}" "${IBM_OPENSHIFT_ENDPOINT}" "${LOGFILE}" "${REPO}" "${CLUSTER_ID}" "${TEST_EXEC_NODES}" "${DEVFILE_REGISTRY}" "${SKIP_SERVICE_BINDING_TESTS}"
+sshpass -p $WINDOWS_PASSWORD ssh Administrator@$WINDOWS_IP -o StrictHostKeyChecking=no powershell /tmp/windows-test-script.ps1 "${GIT_PR_NUMBER}" "${BUILD_NUMBER}" "${API_KEY_QE}" "${IBM_OPENSHIFT_ENDPOINT}" "${LOGFILE}" "${REPO}" "${CLUSTER_ID}" "${TEST_EXEC_NODES}" "${SKIP_SERVICE_BINDING_TESTS}"
 RESULT=$?
 echo "RESULT: $RESULT"
 
diff --git a/Makefile b/Makefile
@@ -238,7 +238,7 @@ test-e2e:
 
 .PHONY: test-doc-automation
 test-doc-automation:
-	$(RUN_GINKGO) $(GINKGO_FLAGS_ONE) --junit-report="test-doc-automation.xml"  tests/documentation/...
+	$(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-doc-automation.xml"  tests/documentation/...
 
 
 # Generate OpenAPISpec library based on ododevapispec.yaml inside pkg/apiserver-gen; this will only generate interfaces
diff --git a/docs/website/docs/command-reference/docs-mdx/init/interactive_mode_empty_directory_output.mdx b/docs/website/docs/command-reference/docs-mdx/init/interactive_mode_empty_directory_output.mdx
@@ -15,7 +15,8 @@ Interactive mode enabled, please answer the following questions:
 ? Select architectures to filter by: amd64
 ? Select language: Java
 ? Select project type: Maven Java
- ✓  Downloading devfile "java-maven" from registry "DefaultDevfileRegistry" [4s]
+? Select version:  1.2.0
+ ✓  Downloading devfile "java-maven:1.2.0" from registry "DefaultDevfileRegistry" [4s]
 
 ↪ Container Configuration "tools":
   OPEN PORTS:
@@ -30,7 +31,7 @@ Interactive mode enabled, please answer the following questions:
  ✓  Downloading starter project "springbootproject" [1s]
 
 You can automate this command by executing:
-   odo init --name my-java-maven-app --devfile java-maven --devfile-registry DefaultDevfileRegistry --starter springbootproject
+   odo init --name my-java-maven-app --devfile java-maven --devfile-registry DefaultDevfileRegistry --devfile-version 1.2.0 --starter springbootproject
 
 Your new component 'my-java-maven-app' is ready in the current directory.
 To start editing your component, use 'odo dev' and open this folder in your favorite IDE.
diff --git a/docs/website/docs/overview/configure.md b/docs/website/docs/overview/configure.md
@@ -189,7 +189,6 @@ Options here are mostly used for debugging and testing `odo` behavior.
 | `ODO_DISABLE_TELEMETRY`             | Useful for disabling [telemetry collection](https://github.com/redhat-developer/odo/blob/main/USAGE_DATA.md). **Deprecated in v3.2.0**. Use `ODO_TRACKING_CONSENT` instead.                                                                                                                                                                                                    | v2.1.0        | `true`                                     |
 | `GLOBALODOCONFIG`                   | Useful for setting a different location of global preference file `preference.yaml`.                                                                                                                                                                                                                                                                                           | v0.0.19       | `~/.config/odo/preference.yaml`            |
 | `ODO_DEBUG_TELEMETRY_FILE`          | Useful for debugging [telemetry](https://github.com/redhat-developer/odo/blob/main/USAGE_DATA.md). When set it will save telemetry data to a file instead of sending it to the server.                                                                                                                                                                                         | v3.0.0-alpha1 | `/tmp/telemetry_data.json`                 |
-| `DEVFILE_PROXY`                     | Integration tests will use this address as Devfile registry instead of `registry.stage.devfile.io`                                                                                                                                                                                                                                                                             | v3.0.0-beta3  | `my-registry.example.com`                  |
 | `TELEMETRY_CALLER`                  | Caller identifier passed to [telemetry](https://github.com/redhat-developer/odo/blob/main/USAGE_DATA.md). Case-insensitive. Acceptable values: `vscode`, `intellij`, `jboss`.                                                                                                                                                                                                  | v3.1.0        | `intellij`                                 |
 | `ODO_TRACKING_CONSENT`              | Useful for controlling [telemetry](https://github.com/redhat-developer/odo/blob/main/USAGE_DATA.md). Acceptable values: `yes` ([enables telemetry](https://github.com/redhat-developer/odo/blob/main/USAGE_DATA.md) and skips consent prompt), `no` (disables telemetry and consent prompt). Takes precedence over the [`ConsentTelemetry`](#preference-key-table) preference. | v3.2.0        | `yes`                                      |
 | `ODO_PUSH_IMAGES`                   | Whether to push the images once built; this is used only when applying Devfile image components as part of a Dev Session running on Podman; this is useful for integration tests running on Podman. `true` by default                                                                                                                                                          | v3.7.0        | `false`                                    |
diff --git a/docs/website/docs/user-guides/quickstart/docs-mdx/dotnet/dotnet_odo_init_output.mdx b/docs/website/docs/user-guides/quickstart/docs-mdx/dotnet/dotnet_odo_init_output.mdx
@@ -12,7 +12,7 @@ Based on the files in the current directory odo detected
 Supported architectures: all
 Language: .NET
 Project type: dotnet
-The devfile "dotnet50:1.0.3" from the registry "DefaultDevfileRegistry" will be downloaded.
+The devfile "dotnet60:1.0.2" from the registry "DefaultDevfileRegistry" will be downloaded.
 ? Is this correct? No
 ? Select architectures to filter by:  [Use arrows to move, space to select, <right> to all, <left> to none, type to filter]
 > [x]  amd64
diff --git a/docs/website/docs/user-guides/quickstart/docs-mdx/java/java_odo_init_output.mdx b/docs/website/docs/user-guides/quickstart/docs-mdx/java/java_odo_init_output.mdx
@@ -12,9 +12,9 @@ Based on the files in the current directory odo detected
 Supported architectures: all
 Language: Java
 Project type: springboot
-The devfile "java-springboot:1.2.0" from the registry "DefaultDevfileRegistry" will be downloaded.
+The devfile "java-springboot:1.3.0" from the registry "DefaultDevfileRegistry" will be downloaded.
 ? Is this correct? Yes
- ✓  Downloading devfile "java-springboot:1.2.0" from registry "DefaultDevfileRegistry" [3s]
+ ✓  Downloading devfile "java-springboot:1.3.0" from registry "DefaultDevfileRegistry" [3s]
 
 ↪ Container Configuration "tools":
   OPEN PORTS:
@@ -27,7 +27,7 @@ The devfile "java-springboot:1.2.0" from the registry "DefaultDevfileRegistry" w
 ? Enter component name: my-java-app
 
 You can automate this command by executing:
-   odo init --name my-java-app --devfile java-springboot --devfile-registry DefaultDevfileRegistry --devfile-version 1.2.0
+   odo init --name my-java-app --devfile java-springboot --devfile-registry DefaultDevfileRegistry --devfile-version 1.3.0
 
 Your new component 'my-java-app' is ready in the current directory.
 To start editing your component, use 'odo dev' and open this folder in your favorite IDE.
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -8,7 +8,6 @@ import (
 )
 
 type Configuration struct {
-	DevfileProxy                  *string       `env:"DEVFILE_PROXY,noinit"`
 	DockerCmd                     string        `env:"DOCKER_CMD,default=docker"`
 	Globalodoconfig               *string       `env:"GLOBALODOCONFIG,noinit"`
 	OdoDebugTelemetryFile         *string       `env:"ODO_DEBUG_TELEMETRY_FILE,noinit"`
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -18,7 +18,6 @@ func TestDefaultValues(t *testing.T) {
 	checkDefaultBoolValue(t, "OdoExperimentalMode", cfg.OdoExperimentalMode, false)
 
 	// Use noinit to set non initialized value as nil instead of zero-value
-	checkNilString(t, "DevfileProxy", cfg.DevfileProxy)
 	checkNilString(t, "Globalodoconfig", cfg.Globalodoconfig)
 	checkNilString(t, "Globalodoconfig", cfg.Globalodoconfig)
 	checkNilString(t, "OdoDebugTelemetryFile", cfg.OdoDebugTelemetryFile)
diff --git a/scripts/openshiftci-config.sh b/scripts/openshiftci-config.sh
@@ -1 +1,4 @@
+# This is used in openshiftci-presubmit-all-tests.sh, used by Interop tests.
+# Since these interop tests are managed by a different team, we intentionally want to use a stable Devfile registry.
+# But our own internal tests make use of the staging Devfile registry.
 export DEVFILE_REGISTRY=https://devfile-registry-ci-devfile-registry.odo-test-kubernete-449701-49529fc6e6a4a9fe7ebba9a3db5b55c4-0000.eu-de.containers.appdomain.cloud/
diff --git a/tests/documentation/command-reference/doc_command_reference_createnamespace_test.go b/tests/documentation/command-reference/doc_command_reference_createnamespace_test.go
@@ -8,18 +8,20 @@ import (
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
 	"github.com/redhat-developer/odo/tests/helper"
 )
 
 var _ = Describe("doc command reference odo create namespace", func() {
 	var commonVar helper.CommonVar
 	var commonPath = filepath.Join("command-reference", "docs-mdx", "create-namespace")
 	var outputStringFormat = "```console\n$ odo %s\n%s```\n"
+	var ns string
 
 	BeforeEach(func() {
 		commonVar = helper.CommonBeforeEach()
 		helper.Chdir(commonVar.Context)
-		Expect(helper.VerifyFileExists(".odo/env/env.yaml")).To(BeFalse())
+		ns = helper.GenerateProjectName()
 	})
 
 	AfterEach(func() {
@@ -29,23 +31,27 @@ var _ = Describe("doc command reference odo create namespace", func() {
 	Context("To create a namespace resource", func() {
 
 		AfterEach(func() {
-			commonVar.CliRunner.DeleteNamespaceProject("odo-dev", true)
+			if commonVar.CliRunner.HasNamespaceProject(ns) {
+				commonVar.CliRunner.DeleteNamespaceProject(ns, false)
+			}
 		})
 
 		It("Creates a namespace resource for a kubernetes cluster", func() {
-			args := []string{"create", "namespace", "odo-dev"}
+			args := []string{"create", "namespace", ns}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
 			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
+			got = strings.ReplaceAll(got, ns, "odo-dev")
 			file := "create_namespace.mdx"
 			want := helper.GetMDXContent(filepath.Join(commonPath, file))
 			diff := cmp.Diff(want, got)
 			Expect(diff).To(BeEmpty(), file)
 		})
 
 		It("Creates a project resource for a kubernetes cluster", func() {
-			args := []string{"create", "project", "odo-dev"}
+			args := []string{"create", "project", ns}
 			out := helper.Cmd("odo", args...).ShouldPass().Out()
 			got := fmt.Sprintf(outputStringFormat, strings.Join(args, " "), helper.StripSpinner(out))
+			got = strings.ReplaceAll(got, ns, "odo-dev")
 			file := "create_project.mdx"
 			want := helper.GetMDXContent(filepath.Join(commonPath, file))
 			diff := cmp.Diff(want, got)
diff --git a/tests/documentation/command-reference/doc_command_reference_deletenamespace_test.go b/tests/documentation/command-reference/doc_command_reference_deletenamespace_test.go
@@ -3,6 +3,7 @@ package docautomation
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
@@ -15,11 +16,12 @@ var _ = Describe("doc command reference odo delete namespace", func() {
 	var commonVar helper.CommonVar
 	var commonPath = filepath.Join("command-reference", "docs-mdx", "delete-namespace")
 	var outputStringFormat = "```console\n$ odo %s\n%s```\n"
+	var ns string
 
 	BeforeEach(func() {
 		commonVar = helper.CommonBeforeEach()
 		helper.Chdir(commonVar.Context)
-		Expect(helper.VerifyFileExists(".odo/env/env.yaml")).To(BeFalse())
+		ns = helper.GenerateProjectName()
 	})
 
 	AfterEach(func() {
@@ -29,42 +31,45 @@ var _ = Describe("doc command reference odo delete namespace", func() {
 	Context("To delete a namespace resource", func() {
 
 		BeforeEach(func() {
-			helper.Cmd("odo", "create", "namespace", "odo-dev").ShouldPass()
-
+			helper.Cmd("odo", "create", "namespace", ns).ShouldPass()
 		})
 
 		AfterEach(func() {
-			commonVar.CliRunner.DeleteNamespaceProject("odo-dev", true)
+			if commonVar.CliRunner.HasNamespaceProject(ns) {
+				commonVar.CliRunner.DeleteNamespaceProject(ns, false)
+			}
 		})
 
 		It("Deletes a namespace resource for a kubernetes cluster", func() {
-			args := []string{"odo", "delete", "namespace", "odo-dev"}
+			args := []string{"odo", "delete", "namespace", ns}
 			out, err := helper.RunInteractive(args, []string{"ODO_LOG_LEVEL=0"}, func(ctx helper.InteractiveContext) {
-				helper.ExpectString(ctx, "? Are you sure you want to delete namespace \"odo-dev\"?")
+				helper.ExpectString(ctx, fmt.Sprintf("? Are you sure you want to delete namespace %q?", ns))
 				helper.SendLine(ctx, "Yes")
 
 			})
 			Expect(err).To(BeNil())
 			got := helper.StripAnsi(out)
 			got = helper.StripInteractiveQuestion(got)
 			got = fmt.Sprintf(outputStringFormat, args[1], helper.StripSpinner(got))
+			got = strings.ReplaceAll(got, ns, "odo-dev")
 			file := "delete_namespace.mdx"
 			want := helper.GetMDXContent(filepath.Join(commonPath, file))
 			diff := cmp.Diff(want, got)
 			Expect(diff).To(BeEmpty(), file)
 		})
 
 		It("Deletes a project resource for a openshift cluster", func() {
-			args := []string{"odo", "delete", "project", "odo-dev"}
+			args := []string{"odo", "delete", "project", ns}
 			out, err := helper.RunInteractive(args, []string{"ODO_LOG_LEVEL=0"}, func(ctx helper.InteractiveContext) {
-				helper.ExpectString(ctx, "? Are you sure you want to delete project \"odo-dev\"?")
+				helper.ExpectString(ctx, fmt.Sprintf("? Are you sure you want to delete project %q?", ns))
 				helper.SendLine(ctx, "Yes")
 
 			})
 			Expect(err).To(BeNil())
 			got := helper.StripAnsi(out)
 			got = helper.StripInteractiveQuestion(got)
 			got = fmt.Sprintf(outputStringFormat, args[1], helper.StripSpinner(got))
+			got = strings.ReplaceAll(got, ns, "odo-dev")
 			file := "delete_project.mdx"
 			want := helper.GetMDXContent(filepath.Join(commonPath, file))
 			diff := cmp.Diff(want, got)
diff --git a/tests/documentation/command-reference/doc_command_reference_init_test.go b/tests/documentation/command-reference/doc_command_reference_init_test.go
@@ -41,6 +41,11 @@ var _ = Describe("doc command reference odo init", Label(helper.LabelNoCluster),
 				helper.ExpectString(ctx, "Select project type")
 				helper.SendLine(ctx, "")
 
+				if helper.HasAtLeastTwoVersions("", "java-maven") {
+					helper.ExpectString(ctx, "Select version")
+					helper.SendLine(ctx, "")
+				}
+
 				helper.ExpectString(ctx, "Select container for which you want to change configuration?")
 				helper.SendLine(ctx, "")
 
diff --git a/tests/documentation/user-guides/doc_user_guides_quickstart_test.go b/tests/documentation/user-guides/doc_user_guides_quickstart_test.go
diff --git a/tests/e2escenarios/e2e_test.go b/tests/e2escenarios/e2e_test.go
diff --git a/tests/helper/helper_generic.go b/tests/helper/helper_generic.go
diff --git a/tests/helper/helper_registry.go b/tests/helper/helper_registry.go
diff --git a/tests/integration/cmd_devfile_registry_test.go b/tests/integration/cmd_devfile_registry_test.go
diff --git a/tests/integration/interactive_dev_test.go b/tests/integration/interactive_dev_test.go
diff --git a/tests/integration/interactive_init_test.go b/tests/integration/interactive_init_test.go

Original file line number	Diff line number	Diff line change
`@@ -15,8 +15,6 @@ cleanup_namespaces`
`15`	`15`	`export SKIP_USER_LOGIN_TESTS=true`
`16`	`16`	`(`
`17`	`17`	`set -e`
`18`		`- export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' \|\| true)"`
`19`		`- echo Using Devfile proxy: ${DEVFILE_PROXY}`
`20`	`18`	`make install`
`21`	`19`	`make test-integration-cluster`
`22`	`20`	`) \|& tee "/tmp/${LOGFILE}"`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,6 @@ ibmcloud target -r "${IBM_REGION}"`
`13`	`13`	`(`
`14`	`14`	`set -e`
`15`	`15`	`make install`
`16`		`- echo Using Devfile Registry ${DEVFILE_REGISTRY}`
`17`	`16`	`make test-integration-no-cluster`
`18`	`17`	`) \|& tee "/tmp/${LOGFILE}"`
`19`	`18`
Original file line number	Diff line number	Diff line change
`@@ -16,8 +16,6 @@ cleanup_namespaces`
`16`	`16`
`17`	`17`	`(`
`18`	`18`	`set -e`
`19`		`- export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' \|\| true)"`
`20`		`- echo Using Devfile proxy: ${DEVFILE_PROXY}`
`21`	`19`	`make install`
`22`	`20`	`make test-integration-cluster`
`23`	`21`	`make test-e2e`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,6 @@ import (`
`8`	`8`	`)`
`9`	`9`
`10`	`10`	`type Configuration struct {`
`11`		- DevfileProxy *string `env:"DEVFILE_PROXY,noinit"`
`12`	`11`	DockerCmd string `env:"DOCKER_CMD,default=docker"`
`13`	`12`	Globalodoconfig *string `env:"GLOBALODOCONFIG,noinit"`
`14`	`13`	OdoDebugTelemetryFile *string `env:"ODO_DEBUG_TELEMETRY_FILE,noinit"`