Updates E2E tests

Changes required to account for a new flow where we
prevent `InstallPlan` from being created when unpack
job fails

Signed-off-by: Mikalai Radchuk <[email protected]>

Author: Mikalai Radchuk

test/e2e/catalog_e2e_test.go

@@ -1010,7 +1010,7 @@ var _ = Describe("Starting CatalogSource e2e tests", func() {
 
 		catSrcImage := "quay.io/olmtest/busybox-dependencies-index"
 
-		// Create gRPC CatalogSource
+		By("creating gRPC CatalogSource")
 		source := &v1alpha1.CatalogSource{
 			TypeMeta: metav1.TypeMeta{
 				Kind:       v1alpha1.CatalogSourceKind,
@@ -1028,30 +1028,29 @@ var _ = Describe("Starting CatalogSource e2e tests", func() {
 				},
 			},
 		}
-
 		source, err := crc.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.Background(), source, metav1.CreateOptions{})
 		Expect(err).ShouldNot(HaveOccurred())
 		defer func() {
 			err := crc.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Delete(context.Background(), source.GetName(), metav1.DeleteOptions{})
 			Expect(err).ShouldNot(HaveOccurred())
 		}()
 
-		// Wait for the CatalogSource to be ready
+		By("waiting for the CatalogSource to be ready")
 		_, err = fetchCatalogSourceOnStatus(crc, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
 		Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
 
-		// Create a Subscription for busybox
+		By("creating a Subscription for busybox")
 		subscriptionName := genName("sub-")
 		cleanupSubscription := createSubscriptionForCatalog(crc, source.GetNamespace(), subscriptionName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 		defer cleanupSubscription()
 
-		// Wait for the Subscription to succeed
+		By("waiting for the Subscription to succeed")
 		subscription, err := fetchSubscription(crc, ns.GetName(), subscriptionName, subscriptionStateAtLatestChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(subscription).ShouldNot(BeNil())
 		Expect(subscription.Status.InstalledCSV).To(Equal("busybox.v1.0.0"))
 
-		// Confirm that a subscription was created for busybox-dependency
+		By("confirming that a subscription was created for busybox-dependency")
 		subscriptionList, err := crc.OperatorsV1alpha1().Subscriptions(source.GetNamespace()).List(context.Background(), metav1.ListOptions{})
 		Expect(err).ShouldNot(HaveOccurred())
 		dependencySubscriptionName := ""
@@ -1062,13 +1061,13 @@ var _ = Describe("Starting CatalogSource e2e tests", func() {
 		}
 		Expect(dependencySubscriptionName).ToNot(BeEmpty())
 
-		// Wait for the Subscription to succeed
+		By("waiting for the Subscription to succeed")
 		subscription, err = fetchSubscription(crc, ns.GetName(), dependencySubscriptionName, subscriptionStateAtLatestChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(subscription).ShouldNot(BeNil())
 		Expect(subscription.Status.InstalledCSV).To(Equal("busybox-dependency.v1.0.0"))
 
-		// Update the catalog image
+		By("updating the catalog image")
 		Eventually(func() error {
 			existingSource, err := crc.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Get(context.Background(), sourceName, metav1.GetOptions{})
 			if err != nil {
@@ -1080,32 +1079,32 @@ var _ = Describe("Starting CatalogSource e2e tests", func() {
 			return err
 		}).Should(Succeed())
 
-		// Wait for the CatalogSource to be ready
+		By("waiting for the CatalogSource to be ready")
 		_, err = fetchCatalogSourceOnStatus(crc, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
 		Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
 
-		// Wait for the busybox v2 Subscription to succeed
+		By("waiting for the busybox v2 Subscription to succeed")
 		subChecker := func(sub *v1alpha1.Subscription) bool {
 			return sub.Status.InstalledCSV == "busybox.v2.0.0"
 		}
 		subscription, err = fetchSubscription(crc, ns.GetName(), subscriptionName, subChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(subscription).ShouldNot(BeNil())
 
-		// Wait for busybox v2 csv to succeed and check the replaces field
+		By("waiting for busybox v2 csv to succeed and check the replaces field")
 		csv, err := fetchCSV(crc, subscription.Status.CurrentCSV, subscription.GetNamespace(), csvSucceededChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(csv.Spec.Replaces).To(Equal("busybox.v1.0.0"))
 
-		// Wait for the busybox-dependency v2 Subscription to succeed
+		By("waiting for the busybox-dependency v2 Subscription to succeed")
 		subChecker = func(sub *v1alpha1.Subscription) bool {
 			return sub.Status.InstalledCSV == "busybox-dependency.v2.0.0"
 		}
 		subscription, err = fetchSubscription(crc, ns.GetName(), dependencySubscriptionName, subChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(subscription).ShouldNot(BeNil())
 
-		// Wait for busybox-dependency v2 csv to succeed and check the replaces field
+		By("waiting for busybox-dependency v2 csv to succeed and check the replaces field")
 		csv, err = fetchCSV(crc, subscription.Status.CurrentCSV, subscription.GetNamespace(), csvSucceededChecker)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(csv.Spec.Replaces).To(Equal("busybox-dependency.v1.0.0"))
@@ -1403,21 +1402,15 @@ var _ = Describe("Starting CatalogSource e2e tests", func() {
 				Expect(c.Create(context.Background(), subscription)).To(BeNil())
 			})
 
-			It("fails with a ResolutionFailed error condition, and a message that highlights the missing field in the CSV", func() {
-
-				subscription, err := fetchSubscription(crc, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanChecker)
-				Expect(err).Should(BeNil())
-				installPlanName := subscription.Status.Install.Name
-
-				// ensure we wait for the installPlan to fail before moving forward then fetch the subscription again
-				_, err = fetchInstallPlan(GinkgoT(), crc, installPlanName, subscription.GetNamespace(), buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseFailed))
-				Expect(err).To(BeNil())
-				subscription, err = fetchSubscription(crc, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanChecker)
-				Expect(err).To(BeNil())
+			It("fails with a BundleUnpackFailed error condition, and a message that highlights the missing field in the CSV", func() {
+				Eventually(func(g Gomega) string {
+					fetchedSubscription, err := crc.OperatorsV1alpha1().Subscriptions(ns.GetName()).Get(context.Background(), subscription.GetName(), metav1.GetOptions{})
+					g.Expect(err).NotTo(HaveOccurred())
 
-				// expect the message that API missing
-				failingCondition := subscription.Status.GetCondition(operatorsv1alpha1.SubscriptionInstallPlanFailed)
-				Expect(failingCondition.Message).To(ContainSubstring("missing APIVersion"))
+					// expect the message that API missing
+					failingCondition := fetchedSubscription.Status.GetCondition(v1alpha1.SubscriptionBundleUnpackFailed)
+					return failingCondition.Message
+				}).Should(ContainSubstring("missing APIVersion"))
 			})
 		})
 	})

test/e2e/fail_forward_e2e_test.go

@@ -55,12 +55,13 @@ var _ = Describe("Fail Forward Upgrades", func() {
 		TeardownNamespace(ns.GetName())
 	})
 
-	When("an InstallPlan is reporting a failed state", func() {
+	When("InstallPlan is not being created due to failed upgrade", func() {
 
 		var (
-			magicCatalog      *MagicCatalog
-			catalogSourceName string
-			subscription      *operatorsv1alpha1.Subscription
+			magicCatalog           *MagicCatalog
+			catalogSourceName      string
+			subscription           *operatorsv1alpha1.Subscription
+			originalInstallPlanRef *corev1.ObjectReference
 		)
 
 		BeforeEach(func() {
@@ -90,7 +91,7 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			subscription, err := fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanChecker)
 			Expect(err).Should(BeNil())
 
-			originalInstallPlanRef := subscription.Status.InstallPlanRef
+			originalInstallPlanRef = subscription.Status.InstallPlanRef
 
 			By("waiting for the v0.1.0 CSV to report a succeeded phase")
 			_, err = fetchCSV(crclient, subscription.Status.CurrentCSV, ns.GetName(), buildCSVConditionChecker(operatorsv1alpha1.CSVPhaseSucceeded))
@@ -102,40 +103,30 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			By("updating the catalog with a broken v0.2.0 bundle image")
 			brokenProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, failForwardTestDataBaseDir, "example-operator.v0.2.0.yaml"))
 			Expect(err).To(BeNil())
-
 			err = magicCatalog.UpdateCatalog(context.Background(), brokenProvider)
 			Expect(err).To(BeNil())
 
-			By("verifying the subscription is referencing a new installplan")
-			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanDifferentChecker(originalInstallPlanRef.Name))
-			Expect(err).Should(BeNil())
-
-			By("waiting for the bad InstallPlan to report a failed installation state")
-			ref := subscription.Status.InstallPlanRef
-			_, err = fetchInstallPlan(GinkgoT(), crclient, ref.Name, ref.Namespace, buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseFailed))
-			Expect(err).To(BeNil())
+			By("waiting for the subscription to maintain the example-operator.v0.1.0 status.currentCSV")
+			Consistently(subscriptionCurrentCSVGetter(crclient, subscription.GetNamespace(), subscription.GetName())).Should(Equal("example-operator.v0.1.0"))
 
+			By("verifying the subscription is referencing the same InstallPlan")
+			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanChecker)
+			Expect(err).Should(BeNil())
+			Expect(subscription.Status.InstallPlanRef.Name).To(Equal(originalInstallPlanRef.Name))
 		})
 		AfterEach(func() {
 			By("removing the testing catalog resources")
 			Expect(magicCatalog.UndeployCatalog(context.Background())).To(BeNil())
 		})
-		It("eventually reports a successful state when multiple bad versions are rolled forward", func() {
+		It("eventually gets created and reports a successful state when multiple bad versions are rolled forward", func() {
 			By("patching the catalog with another bad bundle version")
 			badProvider, err := NewFileBasedFiledBasedCatalogProvider(filepath.Join(testdataDir, "fail-forward/multiple-bad-versions", "example-operator.v0.2.1.yaml"))
 			Expect(err).To(BeNil())
-
 			err = magicCatalog.UpdateCatalog(context.Background(), badProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.2.1 status.updatedCSV")
-			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.2.1"))
-			Expect(err).Should(BeNil())
-
-			By("waiting for the bad v0.2.1 InstallPlan to report a failed installation state")
-			ref := subscription.Status.InstallPlanRef
-			_, err = fetchInstallPlan(GinkgoT(), crclient, ref.Name, ref.Namespace, buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseFailed))
-			Expect(err).To(BeNil())
+			By("waiting for the subscription to maintain the example-operator.v0.1.0 status.currentCSV")
+			Consistently(subscriptionCurrentCSVGetter(crclient, subscription.GetNamespace(), subscription.GetName())).Should(Equal("example-operator.v0.1.0"))
 
 			By("patching the OperatorGroup to increase the bundle unpacking timeout")
 			addBundleUnpackTimeoutOGAnnotation(context.Background(), c, types.NamespacedName{Name: ogName, Namespace: ns.GetName()}, "5m")
@@ -146,12 +137,21 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
-			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
+			_, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
 			Expect(err).Should(BeNil())
+
+			By("verifying the subscription is referencing a new InstallPlan")
+			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanDifferentChecker(originalInstallPlanRef.Name))
+			Expect(err).Should(BeNil())
+
+			By("waiting for the fixed v0.3.0 InstallPlan to report a successful state")
+			ref := subscription.Status.InstallPlanRef
+			_, err = fetchInstallPlan(GinkgoT(), crclient, ref.Name, ref.Namespace, buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseComplete))
+			Expect(err).To(BeNil())
 		})
 
-		It("eventually reports a successful state when using skip ranges", func() {
+		It("eventually gets created and reports a successful state when using skip ranges", func() {
 			By("patching the OperatorGroup to increase the bundle unpacking timeout")
 			addBundleUnpackTimeoutOGAnnotation(context.Background(), c, types.NamespacedName{Name: ogName, Namespace: ns.GetName()}, "5m")
 
@@ -161,11 +161,20 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
-			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
+			_, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
+			Expect(err).Should(BeNil())
+
+			By("verifying the subscription is referencing a new InstallPlan")
+			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanDifferentChecker(originalInstallPlanRef.Name))
 			Expect(err).Should(BeNil())
+
+			By("waiting for the fixed v0.3.0 InstallPlan to report a successful state")
+			ref := subscription.Status.InstallPlanRef
+			_, err = fetchInstallPlan(GinkgoT(), crclient, ref.Name, ref.Namespace, buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseComplete))
+			Expect(err).To(BeNil())
 		})
-		It("eventually reports a successful state when using skips", func() {
+		It("eventually gets created and reports a successful state when using skips", func() {
 			By("patching the OperatorGroup to increase the bundle unpacking timeout")
 			addBundleUnpackTimeoutOGAnnotation(context.Background(), c, types.NamespacedName{Name: ogName, Namespace: ns.GetName()}, "5m")
 
@@ -175,11 +184,20 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
-			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
+			_, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
+			Expect(err).Should(BeNil())
+
+			By("verifying the subscription is referencing a new InstallPlan")
+			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanDifferentChecker(originalInstallPlanRef.Name))
 			Expect(err).Should(BeNil())
+
+			By("waiting for the fixed v0.3.0 InstallPlan to report a successful state")
+			ref := subscription.Status.InstallPlanRef
+			_, err = fetchInstallPlan(GinkgoT(), crclient, ref.Name, ref.Namespace, buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseComplete))
+			Expect(err).To(BeNil())
 		})
-		It("eventually reports a failed state when using replaces", func() {
+		It("never gets created when using replaces", func() {
 			By("patching the OperatorGroup to increase the bundle unpacking timeout")
 			addBundleUnpackTimeoutOGAnnotation(context.Background(), c, types.NamespacedName{Name: ogName, Namespace: ns.GetName()}, "5m")
 
@@ -189,14 +207,13 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to maintain the example-operator.v0.2.0 status.updatedCSV")
-			Consistently(func() string {
-				subscription, err := crclient.OperatorsV1alpha1().Subscriptions(subscription.GetNamespace()).Get(context.Background(), subscription.GetName(), metav1.GetOptions{})
-				if err != nil || subscription == nil {
-					return ""
-				}
-				return subscription.Status.CurrentCSV
-			}).Should(Equal("example-operator.v0.2.0"))
+			By("waiting for the subscription to maintain the example-operator.v0.2.0 status.currentCSV")
+			Consistently(subscriptionCurrentCSVGetter(crclient, subscription.GetNamespace(), subscription.GetName())).Should(Equal("example-operator.v0.1.0"))
+
+			By("verifying the subscription is referencing the same InstallPlan")
+			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasInstallPlanChecker)
+			Expect(err).Should(BeNil())
+			Expect(subscription.Status.InstallPlanRef.Name).To(Equal(originalInstallPlanRef.Name))
 		})
 	})
 	When("a CSV resource is in a failed state", func() {
@@ -269,7 +286,7 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
 			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
 			Expect(err).Should(BeNil())
 		})
@@ -282,7 +299,7 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
 			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
 			Expect(err).Should(BeNil())
 		})
@@ -295,7 +312,7 @@ var _ = Describe("Fail Forward Upgrades", func() {
 			err = magicCatalog.UpdateCatalog(context.Background(), fixedProvider)
 			Expect(err).To(BeNil())
 
-			By("waiting for the subscription to have the example-operator.v0.3.0 status.updatedCSV")
+			By("waiting for the subscription to have the example-operator.v0.3.0 status.currentCSV")
 			subscription, err = fetchSubscription(crclient, subscription.GetNamespace(), subscription.GetName(), subscriptionHasCurrentCSV("example-operator.v0.3.0"))
 			Expect(err).Should(BeNil())
 		})