Improve reliability of e2e tests that include GRPC CatalogSources (#2419)

This commit introduces a change to the e2e tests that rely on
a grpc catalogSource that is stored in an off cluster registry.
These tests originally required that the following happen within a
minute:

1. The catalogSource is installed and ready to serve.
2. The contents of the bundle are unpacked and placed in an
installPlan.
3. The CSV is installed and in the ready state.

The tests will now wait for the catalogSource to be ready to serve
before moving onto steps 2 and 3.

Signed-off-by: Alexander Greene <[email protected]>

Author: awgreene

test/e2e/bundle_e2e_test.go

@@ -106,6 +106,10 @@ var _ = Describe("Installing bundles with new object types", func() {
 				return err
 			}).Should(Succeed())
 
+			// Wait for the CatalogSource to be ready
+			_, err = fetchCatalogSourceOnStatus(operatorClient, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 			// Create a Subscription for package
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 

test/e2e/catalog_e2e_test.go

@@ -482,6 +482,10 @@ var _ = Describe("Catalog represents a store of bundles which OLM can use to ins
 			Expect(err).ShouldNot(HaveOccurred())
 		}()
 
+		// Wait for the CatalogSource to be ready
+		_, err = fetchCatalogSourceOnStatus(crc, addressSource.GetName(), addressSource.GetNamespace(), catalogSourceRegistryPodSynced)
+		Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 		// Delete CatalogSources
 		err = crc.OperatorsV1alpha1().CatalogSources(testNamespace).Delete(context.TODO(), mainSourceName, metav1.DeleteOptions{})
 		Expect(err).ShouldNot(HaveOccurred())
@@ -892,6 +896,10 @@ var _ = Describe("Catalog represents a store of bundles which OLM can use to ins
 			Expect(err).ShouldNot(HaveOccurred())
 		}()
 
+		// Wait for the CatalogSource to be ready
+		_, err = fetchCatalogSourceOnStatus(crc, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+		Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 		// Create a Subscription for busybox
 		subscriptionName := genName("sub-")
 		cleanupSubscription := createSubscriptionForCatalog(crc, source.GetNamespace(), subscriptionName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
@@ -935,6 +943,10 @@ var _ = Describe("Catalog represents a store of bundles which OLM can use to ins
 			return false, nil
 		}).Should(BeTrue())
 
+		// Wait for the CatalogSource to be ready
+		_, err = fetchCatalogSourceOnStatus(crc, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+		Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 		// Wait for the busybox v2 Subscription to succeed
 		subChecker := func(sub *v1alpha1.Subscription) bool {
 			return sub.Status.InstalledCSV == "busybox.v2.0.0"
@@ -1065,7 +1077,6 @@ var _ = Describe("Catalog represents a store of bundles which OLM can use to ins
 		Expect(err).ShouldNot(HaveOccurred(), "error getting catalog source")
 
 		// create an annotation using the kube templates
-
 		source.SetAnnotations(map[string]string{catalogsource.CatalogImageTemplateAnnotation: fmt.Sprintf("quay.io/olmtest/catsrc-update-test:%s.%s.%s", catalogsource.TemplKubeMajorV, catalogsource.TemplKubeMinorV, catalogsource.TemplKubePatchV)})
 
 		// Update the catalog image

test/e2e/gc_e2e_test.go

@@ -358,6 +358,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
 				return nil
 			}).Should(Succeed(), "could not create catalog source")
 
+			// Wait for the CatalogSource to be ready
+			_, err := fetchCatalogSourceOnStatus(operatorClient, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 			// Create a Subscription for package
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 
@@ -468,6 +472,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
 				return err
 			}).Should(Succeed(), "could not create catalog source")
 
+			// Wait for the CatalogSource to be ready
+			_, err = fetchCatalogSourceOnStatus(operatorClient, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 			// Create a Subscription for package
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 
@@ -577,6 +585,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
 				return err
 			}).Should(Succeed())
 
+			// Wait for the CatalogSource to be ready
+			_, err = fetchCatalogSourceOnStatus(operatorClient, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 			// Create a Subscription for package
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 

test/e2e/operator_test.go

@@ -274,6 +274,10 @@ var _ = Describe("Operator API", func() {
 				return client.Create(clientCtx, cs)
 			}).Should(Succeed())
 
+			// Wait for the CatalogSource to be ready
+			_, err := fetchCatalogSourceOnStatus(newCRClient(), cs.GetName(), cs.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred())
+
 			sub = &operatorsv1alpha1.Subscription{
 				Spec: &operatorsv1alpha1.SubscriptionSpec{
 					CatalogSource:          cs.GetName(),

test/e2e/packagemanifest_e2e_test.go

@@ -217,6 +217,10 @@ var _ = Describe("Package Manifest API lists available Operators from Catalog So
 			var err error
 			catalogSource, err = crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Create(context.TODO(), catalogSource, metav1.CreateOptions{})
 			Expect(err).NotTo(HaveOccurred())
+
+			// Wait for the CatalogSource to be ready
+			_, err = fetchCatalogSourceOnStatus(crc, catalogSource.GetName(), catalogSource.GetNamespace(), catalogSourceRegistryPodSynced)
+			Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
 		})
 
 		AfterEach(func() {

test/e2e/subscription_e2e_test.go

@@ -847,6 +847,10 @@ var _ = Describe("Subscription", func() {
 						Expect(err).ToNot(HaveOccurred())
 					}()
 
+					// Wait for the CatalogSource status to be updated to reflect its invalid spec
+					_, err = fetchCatalogSourceOnStatus(crc, cs.GetName(), cs.GetNamespace(), catalogSourceInvalidSpec)
+					Expect(err).ToNot(HaveOccurred(), "catalog source did not become ready")
+
 					subName := genName("sub-")
 					cleanup := createSubscriptionForCatalog(
 						crc,

test/e2e/util_test.go

@@ -325,6 +325,10 @@ func catalogSourceRegistryPodSynced(catalog *v1alpha1.CatalogSource) bool {
 	return false
 }
 
+func catalogSourceInvalidSpec(catalog *v1alpha1.CatalogSource) bool {
+	return catalog.Status.Reason == v1alpha1.CatalogSourceSpecInvalidError
+}
+
 func fetchCatalogSourceOnStatus(crc versioned.Interface, name, namespace string, check catalogSourceCheckFunc) (*v1alpha1.CatalogSource, error) {
 	var fetched *v1alpha1.CatalogSource
 	var err error

test/e2e/webhook_e2e_test.go

@@ -673,6 +673,10 @@ var _ = Describe("CSVs with a Webhook", func() {
 				require.NoError(GinkgoT(), crc.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Delete(context.TODO(), source.GetName(), metav1.DeleteOptions{}))
 			}
 
+			// Wait for the CatalogSource to be ready
+			_, err = fetchCatalogSourceOnStatus(crc, source.GetName(), source.GetNamespace(), catalogSourceRegistryPodSynced)
+			require.NoError(GinkgoT(), err)
+
 			// Create a Subscription for the webhook-operator
 			subscriptionName := genName("sub-")
 			cleanupSubscription := createSubscriptionForCatalog(crc, testNamespace, subscriptionName, source.GetName(), packageName, channelName, "", operatorsv1alpha1.ApprovalAutomatic)