Introduce mechanism for gathering test artifacts during individual test failures

Update the testing e2e suite and add a mechanism for gathering test
artifacts during individual test failures. Currently, container logs are
gathered when deprovisioning upstream kind clusters, yet we lack
fine-grain ability to diagnose test failures further.

Note: test failures use the `CurrentGinkgoTestDescription.Failed` field
to determine failures. Testing artifacts are only gathered when the base
$ARTIFACTS_DIR environment variable has been specified.

Add a collect-ci-artifacts.sh bash script, responsible for gathering OLM
native resources for an individual testing namespace. This bash script
will be called when tearing down the generated testing namespace for
relevant e2e packages. Currently, the artifact gathering process is
restricted to only a single namespace - longer term, it might be
possible to instead migrate towards collecting resources that all share
a similar label selector, and utilizing that label selector to handle
multi-namespace testing scenarios.

Introduce another helper function in test/e2e/util_test.go that's
responsible for gathering test artifacts (i.e. calling this newly
introduced script) when the test case had failed, and in either case,
delete the namespace.

Signed-off-by: timflannagan <[email protected]>

Author: timflannagan

Diff for: test/e2e/collect-ci-artifacts.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+set -o pipefail
+set -o nounset
+set -o errexit
+
+: "${KUBECONFIG:?}"
+: "${TEST_NAMESPACE:?}"
+: "${TEST_ARTIFACTS_DIR:?}"
+
+mkdir -p "${TEST_ARTIFACTS_DIR}"
+
+commands=()
+commands+=("get subscriptions -o yaml")
+commands+=("get operatorgroups -o yaml")
+commands+=("get clusterserviceversions -o yaml")
+commands+=("get installplans -o yaml")
+commands+=("get pods -o wide")
+commands+=("get events --sort-by .lastTimestamp")
+
+echo "Storing the test artifact output in the ${TEST_ARTIFACTS_DIR} directory"
+for command in "${commands[@]}"; do
+    echo "Collecting ${command} output..."
+    COMMAND_OUTPUT_FILE=${TEST_ARTIFACTS_DIR}/${command// /_}
+    kubectl -n ${TEST_NAMESPACE} ${command} >> "${COMMAND_OUTPUT_FILE}"
+done

Diff for: test/e2e/ctx/ctx.go

@@ -2,6 +2,9 @@ package ctx
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
@@ -32,6 +35,7 @@ type TestContext struct {
 	dynamicClient  dynamic.Interface
 	packageClient  pversioned.Interface
 	ssaClient      *controllerclient.ServerSideApplier
+	artifactsDir   string
 
 	scheme *runtime.Scheme
 
@@ -86,6 +90,35 @@ func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
 	return ctx.ssaClient
 }
 
+func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
+	if ctx.artifactsDir == "" {
+		ctx.Logf("$ARTIFACTS_DIR is unset -- not collecting failed test case logs")
+		return nil
+	}
+	ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir)
+
+	logDir := filepath.Join(ctx.artifactsDir, namespace)
+	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
+		return err
+	}
+	envvars := []string{
+		"TEST_NAMESPACE=" + namespace,
+		"TEST_ARTIFACTS_DIR=" + logDir,
+		"KUBECONFIG=" + os.Getenv("KUBECONFIG"),
+	}
+
+	// compiled test binary running e2e tests is run from the root ./bin directory
+	cmd := exec.Command("../test/e2e/collect-ci-artifacts.sh")
+	cmd.Env = append(cmd.Env, envvars...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func setDerivedFields(ctx *TestContext) error {
 	if ctx == nil {
 		return fmt.Errorf("nil test context")

Diff for: test/e2e/ctx/provisioner_kind.go

@@ -134,15 +134,18 @@ func Provision(ctx *TestContext) (func(), error) {
 	if err != nil {
 		return nil, fmt.Errorf("error loading kubeconfig: %s", err.Error())
 	}
-
 	ctx.restConfig = restConfig
 
+	if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+		ctx.artifactsDir = artifactsDir
+	}
+
 	var once sync.Once
 	deprovision := func() {
 		once.Do(func() {
-			if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+			if ctx.artifactsDir != "" {
 				ctx.Logf("collecting container logs for the %s cluster", name)
-				if err := provider.CollectLogs(name, filepath.Join(artifactsDir, logDir)); err != nil {
+				if err := provider.CollectLogs(name, filepath.Join(ctx.artifactsDir, logDir)); err != nil {
 					ctx.Logf("failed to collect logs: %v", err)
 				}
 			}

Diff for: test/e2e/subscription_e2e_test.go

@@ -59,9 +59,7 @@ var _ = Describe("Subscription", func() {
 	})
 
 	AfterEach(func() {
-		Eventually(func() error {
-			return ctx.Ctx().Client().Delete(context.Background(), &generatedNamespace)
-		}).Should(Succeed())
+		TeardownNamespace(generatedNamespace.GetName())
 	})
 
 	When("an entry in the middle of a channel does not provide a required GVK", func() {

Diff for: test/e2e/util_test.go

@@ -946,3 +946,20 @@ func SetupGeneratedTestNamespace(name string) corev1.Namespace {
 
 	return ns
 }
+
+func TeardownNamespace(ns string) {
+	log := ctx.Ctx().Logf
+
+	currentTest := CurrentGinkgoTestDescription()
+	if currentTest.Failed {
+		log("collecting the %s namespace artifacts as the '%s' test case failed", ns, currentTest.TestText)
+		if err := ctx.Ctx().DumpNamespaceArtifacts(ns); err != nil {
+			log("failed to collect namespace artifacts: %v", err)
+		}
+	}
+
+	log("tearing down the %s namespace", ns)
+	Eventually(func() error {
+		return ctx.Ctx().KubeClient().KubernetesInterface().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{})
+	}).Should(Succeed())
+}