Introduce a mechanism for gathering test artifacts during individual test failures

test/e2e/bundle_e2e_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 
 	"github.com/ghodss/yaml"
+	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -27,19 +28,23 @@ var vpaCRDRaw []byte
 
 var _ = Describe("Installing bundles with new object types", func() {
 	var (
-		kubeClient     operatorclient.ClientInterface
-		operatorClient versioned.Interface
-		dynamicClient  dynamic.Interface
+		kubeClient         operatorclient.ClientInterface
+		operatorClient     versioned.Interface
+		dynamicClient      dynamic.Interface
+		generatedNamespace corev1.Namespace
 	)
 
 	BeforeEach(func() {
 		kubeClient = ctx.Ctx().KubeClient()
 		operatorClient = ctx.Ctx().OperatorClient()
 		dynamicClient = ctx.Ctx().DynamicClient()
+
+		By("creating a test namespace")
+		generatedNamespace = SetupGeneratedTestNamespace(genName("bundle-e2e-"))
 	})
 
 	AfterEach(func() {
-		TearDown(testNamespace)
+		TeardownNamespace(generatedNamespace.GetName())
 	})
 
 	When("a bundle with a pdb, priorityclass, and VPA object is installed", func() {
@@ -66,7 +71,7 @@ var _ = Describe("Installing bundles with new object types", func() {
 			Expect(err).ToNot(HaveOccurred(), "could not convert vpa crd to unstructured")
 
 			Eventually(func() error {
-				err := ctx.Ctx().Client().Create(context.TODO(), &vpaCRD)
+				err := ctx.Ctx().Client().Create(context.Background(), &vpaCRD)
 				if err != nil {
 					if !k8serrors.IsAlreadyExists(err) {
 						return err
@@ -77,22 +82,21 @@ var _ = Describe("Installing bundles with new object types", func() {
 
 			// ensure vpa crd is established and accepted on the cluster before continuing
 			Eventually(func() (bool, error) {
-				crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), vpaCRD.GetName(), metav1.GetOptions{})
+				crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), vpaCRD.GetName(), metav1.GetOptions{})
 				if err != nil {
 					return false, err
 				}
 				return crdReady(&crd.Status), nil
 			}).Should(BeTrue())
 
-			var installPlanRef string
 			source := &v1alpha1.CatalogSource{
 				TypeMeta: metav1.TypeMeta{
 					Kind:       v1alpha1.CatalogSourceKind,
 					APIVersion: v1alpha1.CatalogSourceCRDAPIVersion,
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      sourceName,
-					Namespace: testNamespace,
+					Namespace: generatedNamespace.GetName(),
 					Labels:    map[string]string{"olm.catalogSource": sourceName},
 				},
 				Spec: v1alpha1.CatalogSourceSpec{
@@ -102,7 +106,7 @@ var _ = Describe("Installing bundles with new object types", func() {
 			}
 
 			Eventually(func() error {
-				source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
+				source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.Background(), source, metav1.CreateOptions{})
 				return err
 			}).Should(Succeed())
 
@@ -114,13 +118,13 @@ var _ = Describe("Installing bundles with new object types", func() {
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 
 			// Wait for the Subscription to succeed
-			sub, err := fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
+			sub, err := fetchSubscription(operatorClient, generatedNamespace.GetName(), subName, subscriptionStateAtLatestChecker)
 			Expect(err).ToNot(HaveOccurred(), "could not get subscription at latest status")
 
-			installPlanRef = sub.Status.InstallPlanRef.Name
+			installPlanRef := sub.Status.InstallPlanRef
 
 			// Wait for the installplan to complete (5 minute timeout)
-			_, err = fetchInstallPlan(GinkgoT(), operatorClient, installPlanRef, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete))
+			_, err = fetchInstallPlanWithNamespace(GinkgoT(), operatorClient, installPlanRef.Name, installPlanRef.Namespace, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete))
 			Expect(err).ToNot(HaveOccurred(), "could not get installplan at complete phase")
 
 			ctx.Ctx().Logf("install plan %s completed", installPlanRef)
@@ -144,25 +148,25 @@ var _ = Describe("Installing bundles with new object types", func() {
 
 			// confirm extra bundle objects are installed
 			Eventually(func() error {
-				_, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.TODO(), priorityClassName, metav1.GetOptions{})
+				_, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.Background(), priorityClassName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error getting priorityclass object associated with CSV")
 
 			Eventually(func() error {
-				_, err := dynamicClient.Resource(resource).Namespace(testNamespace).Get(context.TODO(), vpaName, metav1.GetOptions{})
+				_, err := dynamicClient.Resource(resource).Namespace(generatedNamespace.GetName()).Get(context.Background(), vpaName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error finding vpa object associated with csv")
 
 			Eventually(func() error {
-				_, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(testNamespace).Get(context.TODO(), pdbName, metav1.GetOptions{})
+				_, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(generatedNamespace.GetName()).Get(context.Background(), pdbName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error getting pdb object associated with CSV")
 		})
 
 		AfterEach(func() {
 			By("Deleting the VPA CRD")
 			Eventually(func() error {
-				err := ctx.Ctx().Client().Delete(context.TODO(), &vpaCRD)
+				err := ctx.Ctx().Client().Delete(context.Background(), &vpaCRD)
 				if k8serrors.IsNotFound(err) {
 					return nil
 				}

test/e2e/collect-ci-artifacts.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+set -o pipefail
+set -o nounset
+set -o errexit
+
+: "${KUBECONFIG:?}"
+: "${TEST_NAMESPACE:?}"
+: "${TEST_ARTIFACTS_DIR:?}"
+
+mkdir -p "${TEST_ARTIFACTS_DIR}"
+
+commands=()
+commands+=("get subscriptions -o yaml")
+commands+=("get operatorgroups -o yaml")
+commands+=("get clusterserviceversions -o yaml")
+commands+=("get installplans -o yaml")
+commands+=("get pods -o wide")
+commands+=("get events --sort-by .lastTimestamp")
+
+echo "Storing the test artifact output in the ${TEST_ARTIFACTS_DIR} directory"
+for command in "${commands[@]}"; do
+    echo "Collecting ${command} output..."
+    COMMAND_OUTPUT_FILE=${TEST_ARTIFACTS_DIR}/${command// /_}
+    kubectl -n ${TEST_NAMESPACE} ${command} >> "${COMMAND_OUTPUT_FILE}"
+done

test/e2e/ctx/ctx.go

@@ -2,6 +2,9 @@ package ctx
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
@@ -32,6 +35,7 @@ type TestContext struct {
 	dynamicClient  dynamic.Interface
 	packageClient  pversioned.Interface
 	ssaClient      *controllerclient.ServerSideApplier
+	artifactsDir   string
 
 	scheme *runtime.Scheme
 
@@ -86,6 +90,35 @@ func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
 	return ctx.ssaClient
 }
 
+func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
+	if ctx.artifactsDir == "" {
+		ctx.Logf("$ARTIFACTS_DIR is unset -- not collecting failed test case logs")
+		return nil
+	}
+	ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir)
+
+	logDir := filepath.Join(ctx.artifactsDir, namespace)
+	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
+		return err
+	}
+	envvars := []string{
+		"TEST_NAMESPACE=" + namespace,
+		"TEST_ARTIFACTS_DIR=" + logDir,
+		"KUBECONFIG=" + os.Getenv("KUBECONFIG"),
+	}
+
+	// compiled test binary running e2e tests is run from the root ./bin directory
+	cmd := exec.Command("../test/e2e/collect-ci-artifacts.sh")
+	cmd.Env = append(cmd.Env, envvars...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func setDerivedFields(ctx *TestContext) error {
 	if ctx == nil {
 		return fmt.Errorf("nil test context")

test/e2e/ctx/provisioner_kind.go

@@ -134,15 +134,18 @@ func Provision(ctx *TestContext) (func(), error) {
 	if err != nil {
 		return nil, fmt.Errorf("error loading kubeconfig: %s", err.Error())
 	}
-
 	ctx.restConfig = restConfig
 
+	if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+		ctx.artifactsDir = artifactsDir
+	}
+
 	var once sync.Once
 	deprovision := func() {
 		once.Do(func() {
-			if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+			if ctx.artifactsDir != "" {
 				ctx.Logf("collecting container logs for the %s cluster", name)
-				if err := provider.CollectLogs(name, filepath.Join(artifactsDir, logDir)); err != nil {
+				if err := provider.CollectLogs(name, filepath.Join(ctx.artifactsDir, logDir)); err != nil {
 					ctx.Logf("failed to collect logs: %v", err)
 				}
 			}

test/e2e/subscription_e2e_test.go

@@ -59,9 +59,7 @@ var _ = Describe("Subscription", func() {
 	})
 
 	AfterEach(func() {
-		Eventually(func() error {
-			return ctx.Ctx().Client().Delete(context.Background(), &generatedNamespace)
-		}).Should(Succeed())
+		TeardownNamespace(generatedNamespace.GetName())
 	})
 
 	When("an entry in the middle of a channel does not provide a required GVK", func() {

test/e2e/util_test.go

@@ -946,3 +946,20 @@ func SetupGeneratedTestNamespace(name string) corev1.Namespace {
 
 	return ns
 }
+
+func TeardownNamespace(ns string) {
+	log := ctx.Ctx().Logf
+
+	currentTest := CurrentGinkgoTestDescription()
+	if currentTest.Failed {
+		log("collecting the %s namespace artifacts as the '%s' test case failed", ns, currentTest.TestText)
+		if err := ctx.Ctx().DumpNamespaceArtifacts(ns); err != nil {
+			log("failed to collect namespace artifacts: %v", err)
+		}
+	}
+
+	log("tearing down the %s namespace", ns)
+	Eventually(func() error {
+		return ctx.Ctx().KubeClient().KubernetesInterface().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{})
+	}).Should(Succeed())
+}