🐛 mitigate upgrade-e2e test flakiness (operator-framework#1627)

* add artifact collection to upgrade-e2e-tests

Signed-off-by: Per Goncalves da Silva <[email protected]>

* mitigate upgrade-e2e flakiness

Signed-off-by: Per Goncalves da Silva <[email protected]>

---------

Signed-off-by: Per Goncalves da Silva <[email protected]>
Co-authored-by: Per Goncalves da Silva <[email protected]>

Author: perdasilva

.github/workflows/e2e.yaml

@@ -58,4 +58,10 @@ jobs:
         go-version-file: go.mod
 
     - name: Run the upgrade e2e test
-      run: make test-upgrade-e2e
+      run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e
+
+    - uses: cytopia/[email protected]
+      if: failure()
+      with:
+        name: upgrade-e2e-artifacts
+        path: /tmp/artifacts/

test/e2e/cluster_extension_install_test.go

@@ -3,18 +3,13 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
-	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -24,12 +19,11 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/rand"
-	kubeclient "k8s.io/client-go/kubernetes"
-	"k8s.io/utils/env"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	ocv1 "github.com/operator-framework/operator-controller/api/v1"
 	catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1"
+	"github.com/operator-framework/operator-controller/test/utils"
 )
 
 const (
@@ -306,7 +300,7 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {
 
 			clusterExtension, extensionCatalog, sa, ns := testInit(t)
 			defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-			defer getArtifactsOutput(t)
+			defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 			clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 				Source: ocv1.SourceConfig{
@@ -366,7 +360,7 @@ func TestClusterExtensionInstallRegistryDynamic(t *testing.T) {
 
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 		Source: ocv1.SourceConfig{
@@ -441,7 +435,7 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
 	require.NoError(t, err)
 
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 	defer func(cat *catalogd.ClusterCatalog) {
 		require.NoError(t, c.Delete(context.Background(), cat))
 		require.Eventually(t, func() bool {
@@ -489,7 +483,7 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {
 
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	t.Log("By creating an ClusterExtension at a specified version")
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
@@ -552,7 +546,7 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {
 
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	t.Log("By creating an ClusterExtension at a specified version")
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
@@ -601,7 +595,7 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
 	t.Log("When resolving upgrade edges")
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	t.Log("By creating an ClusterExtension at a specified version")
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
@@ -649,7 +643,7 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
 	t.Log("It resolves again when a catalog is patched with new ImageRef")
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 		Source: ocv1.SourceConfig{
@@ -736,7 +730,7 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
 	sa, err := createServiceAccount(context.Background(), types.NamespacedName{Name: clusterExtensionName, Namespace: ns.Name}, clusterExtensionName)
 	require.NoError(t, err)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 		Source: ocv1.SourceConfig{
@@ -797,7 +791,7 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T
 	t.Log("It resolves again when managed content is changed")
 	clusterExtension, extensionCatalog, sa, ns := testInit(t)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 		Source: ocv1.SourceConfig{
@@ -860,7 +854,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
 	err := c.Create(context.Background(), sa)
 	require.NoError(t, err)
 	defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
-	defer getArtifactsOutput(t)
+	defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
 
 	clusterExtension.Spec = ocv1.ClusterExtensionSpec{
 		Source: ocv1.SourceConfig{
@@ -936,131 +930,3 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
 		}
 	}, pollDuration, pollInterval)
 }
-
-// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path.
-// Currently it saves:
-// - clusterextensions
-// - pods logs
-// - deployments
-// - catalogsources
-func getArtifactsOutput(t *testing.T) {
-	basePath := env.GetString("ARTIFACT_PATH", "")
-	if basePath == "" {
-		return
-	}
-
-	kubeClient, err := kubeclient.NewForConfig(cfg)
-	require.NoError(t, err)
-
-	// sanitize the artifact name for use as a directory name
-	testName := strings.ReplaceAll(strings.ToLower(t.Name()), " ", "-")
-	// Get the test description and sanitize it for use as a directory name
-	artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName)
-
-	// Create the full artifact path
-	err = os.MkdirAll(artifactPath, 0755)
-	require.NoError(t, err)
-
-	// Get all namespaces
-	namespaces := corev1.NamespaceList{}
-	if err := c.List(context.Background(), &namespaces); err != nil {
-		fmt.Printf("Failed to list namespaces: %v", err)
-	}
-
-	// get all cluster extensions save them to the artifact path.
-	clusterExtensions := ocv1.ClusterExtensionList{}
-	if err := c.List(context.Background(), &clusterExtensions, client.InNamespace("")); err != nil {
-		fmt.Printf("Failed to list cluster extensions: %v", err)
-	}
-	for _, clusterExtension := range clusterExtensions.Items {
-		// Save cluster extension to artifact path
-		clusterExtensionYaml, err := yaml.Marshal(clusterExtension)
-		if err != nil {
-			fmt.Printf("Failed to marshal cluster extension: %v", err)
-			continue
-		}
-		if err := os.WriteFile(filepath.Join(artifactPath, clusterExtension.Name+"-clusterextension.yaml"), clusterExtensionYaml, 0600); err != nil {
-			fmt.Printf("Failed to write cluster extension to file: %v", err)
-		}
-	}
-
-	// get all catalogsources save them to the artifact path.
-	catalogsources := catalogd.ClusterCatalogList{}
-	if err := c.List(context.Background(), &catalogsources, client.InNamespace("")); err != nil {
-		fmt.Printf("Failed to list catalogsources: %v", err)
-	}
-	for _, catalogsource := range catalogsources.Items {
-		// Save catalogsource to artifact path
-		catalogsourceYaml, err := yaml.Marshal(catalogsource)
-		if err != nil {
-			fmt.Printf("Failed to marshal catalogsource: %v", err)
-			continue
-		}
-		if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil {
-			fmt.Printf("Failed to write catalogsource to file: %v", err)
-		}
-	}
-
-	for _, namespace := range namespaces.Items {
-		// let's ignore kube-* namespaces.
-		if strings.Contains(namespace.Name, "kube-") {
-			continue
-		}
-
-		namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name)
-		if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil {
-			fmt.Printf("Failed to create namespaced artifact path: %v", err)
-			continue
-		}
-
-		// get all deployments in the namespace and save them to the artifact path.
-		deployments := appsv1.DeploymentList{}
-		if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil {
-			fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name)
-			continue
-		}
-
-		for _, deployment := range deployments.Items {
-			// Save deployment to artifact path
-			deploymentYaml, err := yaml.Marshal(deployment)
-			if err != nil {
-				fmt.Printf("Failed to marshal deployment: %v", err)
-				continue
-			}
-			if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil {
-				fmt.Printf("Failed to write deployment to file: %v", err)
-			}
-		}
-
-		// Get logs from all pods in all namespaces
-		pods := corev1.PodList{}
-		if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil {
-			fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name)
-		}
-		for _, pod := range pods.Items {
-			if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
-				continue
-			}
-			for _, container := range pod.Spec.Containers {
-				logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background())
-				if err != nil {
-					fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
-					continue
-				}
-				defer logs.Close()
-
-				outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt"))
-				if err != nil {
-					fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
-					continue
-				}
-				defer outFile.Close()
-
-				if _, err := io.Copy(outFile, logs); err != nil {
-					fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
-					continue
-				}
-			}
-		}
-	}
-}