test/e2e/fail-forward: use internal catalogs, change failure mode

Previously, these tests intedend to cause failing InstallPlans by not
providiing the ServiceAccount on the OperatorGroup sufficient
permissions. Due to unrelated reasons, we've had to make insufficient
permissions not a terminal failure mode (the user may always add them
in...).

Now, we achieve failing InstallPlans by using invalid Kubernetes objects
in the bundles. We also use the internal CatalogSource instead of
hosting our catalog data in Quay, as this allows us to skip waiting for
image pulls from the registry and keep our test data in version control.

Signed-off-by: Steve Kuznetsov <[email protected]>

Author: stevekuznetsov

test/e2e/fail_forward_e2e_test.go

@@ -53,6 +53,10 @@ const (
 	interval = time.Millisecond * 100
 )
 
+const (
+	subscriptionTestDataBaseDir = "subscription/"
+)
+
 var _ = Describe("Subscription", func() {
 	var (
 		generatedNamespace corev1.Namespace
@@ -2425,7 +2429,7 @@ var _ = Describe("Subscription", func() {
 
 		BeforeEach(func() {
 			By("deploying the testing catalog")
-			provider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, failForwardTestDataBaseDir, "example-operator.v0.1.0.yaml"))
+			provider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, subscriptionTestDataBaseDir, "example-operator.v0.1.0.yaml"))
 			Expect(err).To(BeNil())
 			catalogSourceName = fmt.Sprintf("%s-catsrc", generatedNamespace.GetName())
 			magicCatalog = NewMagicCatalog(ctx.Ctx().Client(), generatedNamespace.GetName(), catalogSourceName, provider)
@@ -2481,7 +2485,7 @@ var _ = Describe("Subscription", func() {
 				addBundleUnpackTimeoutOGAnnotation(context.Background(), ctx.Ctx().Client(), ogNN, "1s")
 
 				By("updating the catalog with a broken v0.2.0 bundle image")
-				brokenProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, failForwardTestDataBaseDir, "example-operator.v0.2.0-non-existent-tag.yaml"))
+				brokenProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, subscriptionTestDataBaseDir, "example-operator.v0.2.0-non-existent-tag.yaml"))
 				Expect(err).To(BeNil())
 				err = magicCatalog.UpdateCatalog(context.Background(), brokenProvider)
 				Expect(err).To(BeNil())
@@ -2516,7 +2520,7 @@ var _ = Describe("Subscription", func() {
 				addBundleUnpackTimeoutOGAnnotation(context.Background(), ctx.Ctx().Client(), ogNN, "5m")
 
 				By("updating the catalog with a fixed v0.2.0 bundle image")
-				brokenProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, failForwardTestDataBaseDir, "example-operator.v0.2.0.yaml"))
+				brokenProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, subscriptionTestDataBaseDir, "example-operator.v0.2.0.yaml"))
 				Expect(err).To(BeNil())
 				err = magicCatalog.UpdateCatalog(context.Background(), brokenProvider)
 				Expect(err).To(BeNil())
@@ -2685,7 +2689,7 @@ properties:
 
 		It("should not retry successful unpack jobs", func() {
 			By("deploying the testing catalog")
-			provider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, failForwardTestDataBaseDir, "example-operator.v0.1.0.yaml"))
+			provider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, subscriptionTestDataBaseDir, "example-operator.v0.1.0.yaml"))
 			Expect(err).To(BeNil())
 			catalogSourceName := fmt.Sprintf("%s-catsrc", generatedNamespace.GetName())
 			magicCatalog := NewMagicCatalog(ctx.Ctx().Client(), generatedNamespace.GetName(), catalogSourceName, provider)

test/e2e/subscription_e2e_test.go

@@ -0,0 +1,37 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: example-operator.v0.1.0
+  namespace: placeholder
+spec:
+  displayName: Example Operator
+  install:
+    spec:
+      deployments:
+        - name: example-operator-controller-manager
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                control-plane: controller-manager
+            strategy: {}
+            template:
+              metadata:
+                labels:
+                  control-plane: controller-manager
+              spec:
+                containers:
+                  - image: quay.io/openshift-scale/nginx:latest
+                    name: manager
+    strategy: deployment
+  installModes:
+    - supported: false
+      type: OwnNamespace
+    - supported: false
+      type: SingleNamespace
+    - supported: false
+      type: MultiNamespace
+    - supported: true
+      type: AllNamespaces
+  maturity: alpha
+  version: 0.1.0

test/e2e/testdata/fail-forward/multiple-bad-versions/example-operator.v0.2.1-non-existent-tag.yaml

@@ -0,0 +1,5 @@
+packageName: packageA
+channels:
+  - name: stable
+    currentCSV: example-operator.v0.1.0
+defaultChannel: stable

test/e2e/testdata/fail-forward/multiple-bad-versions/example-operator.v0.3.0.yaml

@@ -0,0 +1,38 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: example-operator.v0.2.0&invalid
+  namespace: placeholder
+spec:
+  replaces: example-operator.v0.1.0
+  displayName: Example Operator
+  install:
+    spec:
+      deployments:
+        - name: example-operator-controller-manager
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                control-plane: controller-manager
+            strategy: {}
+            template:
+              metadata:
+                labels:
+                  control-plane: controller-manager
+              spec:
+                containers:
+                  - image: quay.io/openshift-scale/nginx:latest
+                    name: manager
+    strategy: deployment
+  installModes:
+    - supported: false
+      type: OwnNamespace
+    - supported: false
+      type: SingleNamespace
+    - supported: false
+      type: MultiNamespace
+    - supported: true
+      type: AllNamespaces
+  maturity: alpha
+  version: 0.2.0

test/e2e/testdata/fail-forward/replaces/example-operator.v0.3.0.yaml

@@ -0,0 +1,5 @@
+packageName: packageA
+channels:
+  - name: stable
+    currentCSV: example-operator.v0.2.0&invalid
+defaultChannel: stable

test/e2e/testdata/fail-forward/skip-range/example-operator.v0.3.0.yaml

@@ -0,0 +1,38 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: example-operator.v0.2.0
+  namespace: placeholder
+spec:
+  replaces: example-operator.v0.1.0
+  displayName: Example Operator
+  install:
+    spec:
+      deployments:
+        - name: example-operator-controller-manager&invalid
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                control-plane: controller-manager
+            strategy: {}
+            template:
+              metadata:
+                labels:
+                  control-plane: controller-manager
+              spec:
+                containers:
+                  - image: quay.io/openshift-scale/nginx:latest
+                    name: manager
+    strategy: deployment
+  installModes:
+    - supported: false
+      type: OwnNamespace
+    - supported: false
+      type: SingleNamespace
+    - supported: false
+      type: MultiNamespace
+    - supported: true
+      type: AllNamespaces
+  maturity: alpha
+  version: 0.2.0

test/e2e/testdata/fail-forward/skips/example-operator.v0.3.0.yaml

@@ -0,0 +1,5 @@
+packageName: packageA
+channels:
+  - name: stable
+    currentCSV: example-operator.v0.2.0
+defaultChannel: stable

test/e2e/testdata/fail-forward/v0.1.0/clusterserviceversion.yaml

@@ -0,0 +1,40 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: example-operator.v0.2.1&invalid
+  namespace: placeholder
+spec:
+  skips:
+    - example-operator.v0.1.0
+    - example-operator.v0.2.0
+  displayName: Example Operator
+  install:
+    spec:
+      deployments:
+        - name: example-operator-controller-manager
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                control-plane: controller-manager
+            strategy: {}
+            template:
+              metadata:
+                labels:
+                  control-plane: controller-manager
+              spec:
+                containers:
+                  - image: quay.io/openshift-scale/nginx:latest
+                    name: manager
+    strategy: deployment
+  installModes:
+    - supported: false
+      type: OwnNamespace
+    - supported: false
+      type: SingleNamespace
+    - supported: false
+      type: MultiNamespace
+    - supported: true
+      type: AllNamespaces
+  maturity: alpha
+  version: 0.2.1

test/e2e/testdata/fail-forward/v0.1.0/packagemanifest.yaml

@@ -0,0 +1,5 @@
+packageName: packageA
+channels:
+  - name: stable
+    currentCSV: example-operator.v0.2.1&invalid
+defaultChannel: stable