From dccc832d9e43d302833cdb3c0806cfb0bbd0d6a1 Mon Sep 17 00:00:00 2001 From: timflannagan Date: Tue, 9 Nov 2021 18:56:49 -0500 Subject: [PATCH 1/2] Introduce mechanism for gathering test artifacts during individual test failures Update the testing e2e suite and add a mechanism for gathering test artifacts during individual test failures. Currently, container logs are gathered when deprovisioning upstream kind clusters, yet we lack fine-grain ability to diagnose test failures further. Note: test failures use the `CurrentGinkgoTestDescription.Failed` field to determine failures. Testing artifacts are only gathered when the base $ARTIFACTS_DIR environment variable has been specified. Add a collect-ci-artifacts.sh bash script, responsible for gathering OLM native resources for an individual testing namespace. This bash script will be called when tearing down the generated testing namespace for relevant e2e packages. Currently, the artifact gathering process is restricted to only a single namespace - longer term, it might be possible to instead migrate towards collecting resources that all share a similar label selector, and utilizing that label selector to handle multi-namespace testing scenarios. Introduce another helper function in test/e2e/util_test.go that's responsible for gathering test artifacts (i.e. calling this newly introduced script) when the test case had failed, and in either case, delete the namespace. Signed-off-by: timflannagan --- test/e2e/collect-ci-artifacts.sh | 26 ++++++++++++++++++++++++ test/e2e/ctx/ctx.go | 33 +++++++++++++++++++++++++++++++ test/e2e/ctx/provisioner_kind.go | 9 ++++++--- test/e2e/subscription_e2e_test.go | 4 +--- test/e2e/util_test.go | 17 ++++++++++++++++ 5 files changed, 83 insertions(+), 6 deletions(-) create mode 100755 test/e2e/collect-ci-artifacts.sh diff --git a/test/e2e/collect-ci-artifacts.sh b/test/e2e/collect-ci-artifacts.sh new file mode 100755 index 0000000000..8cd138b977 --- /dev/null +++ b/test/e2e/collect-ci-artifacts.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +set -o pipefail +set -o nounset +set -o errexit + +: "${KUBECONFIG:?}" +: "${TEST_NAMESPACE:?}" +: "${TEST_ARTIFACTS_DIR:?}" + +mkdir -p "${TEST_ARTIFACTS_DIR}" + +commands=() +commands+=("get subscriptions -o yaml") +commands+=("get operatorgroups -o yaml") +commands+=("get clusterserviceversions -o yaml") +commands+=("get installplans -o yaml") +commands+=("get pods -o wide") +commands+=("get events --sort-by .lastTimestamp") + +echo "Storing the test artifact output in the ${TEST_ARTIFACTS_DIR} directory" +for command in "${commands[@]}"; do + echo "Collecting ${command} output..." + COMMAND_OUTPUT_FILE=${TEST_ARTIFACTS_DIR}/${command// /_} + kubectl -n ${TEST_NAMESPACE} ${command} >> "${COMMAND_OUTPUT_FILE}" +done diff --git a/test/e2e/ctx/ctx.go b/test/e2e/ctx/ctx.go index 2a38a0c41e..43173808f6 100644 --- a/test/e2e/ctx/ctx.go +++ b/test/e2e/ctx/ctx.go @@ -2,6 +2,9 @@ package ctx import ( "fmt" + "os" + "os/exec" + "path/filepath" "strings" . "github.com/onsi/ginkgo" @@ -32,6 +35,7 @@ type TestContext struct { dynamicClient dynamic.Interface packageClient pversioned.Interface ssaClient *controllerclient.ServerSideApplier + artifactsDir string scheme *runtime.Scheme @@ -86,6 +90,35 @@ func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier { return ctx.ssaClient } +func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error { + if ctx.artifactsDir == "" { + ctx.Logf("$ARTIFACTS_DIR is unset -- not collecting failed test case logs") + return nil + } + ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir) + + logDir := filepath.Join(ctx.artifactsDir, namespace) + if err := os.MkdirAll(logDir, os.ModePerm); err != nil { + return err + } + envvars := []string{ + "TEST_NAMESPACE=" + namespace, + "TEST_ARTIFACTS_DIR=" + logDir, + "KUBECONFIG=" + os.Getenv("KUBECONFIG"), + } + + // compiled test binary running e2e tests is run from the root ./bin directory + cmd := exec.Command("../test/e2e/collect-ci-artifacts.sh") + cmd.Env = append(cmd.Env, envvars...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + func setDerivedFields(ctx *TestContext) error { if ctx == nil { return fmt.Errorf("nil test context") diff --git a/test/e2e/ctx/provisioner_kind.go b/test/e2e/ctx/provisioner_kind.go index 35f194fc19..1f16ccd3ac 100644 --- a/test/e2e/ctx/provisioner_kind.go +++ b/test/e2e/ctx/provisioner_kind.go @@ -134,15 +134,18 @@ func Provision(ctx *TestContext) (func(), error) { if err != nil { return nil, fmt.Errorf("error loading kubeconfig: %s", err.Error()) } - ctx.restConfig = restConfig + if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" { + ctx.artifactsDir = artifactsDir + } + var once sync.Once deprovision := func() { once.Do(func() { - if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" { + if ctx.artifactsDir != "" { ctx.Logf("collecting container logs for the %s cluster", name) - if err := provider.CollectLogs(name, filepath.Join(artifactsDir, logDir)); err != nil { + if err := provider.CollectLogs(name, filepath.Join(ctx.artifactsDir, logDir)); err != nil { ctx.Logf("failed to collect logs: %v", err) } } diff --git a/test/e2e/subscription_e2e_test.go b/test/e2e/subscription_e2e_test.go index 0af926a23d..978fe2b578 100644 --- a/test/e2e/subscription_e2e_test.go +++ b/test/e2e/subscription_e2e_test.go @@ -59,9 +59,7 @@ var _ = Describe("Subscription", func() { }) AfterEach(func() { - Eventually(func() error { - return ctx.Ctx().Client().Delete(context.Background(), &generatedNamespace) - }).Should(Succeed()) + TeardownNamespace(generatedNamespace.GetName()) }) When("an entry in the middle of a channel does not provide a required GVK", func() { diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index 30a43857a4..f6c055d6fe 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -946,3 +946,20 @@ func SetupGeneratedTestNamespace(name string) corev1.Namespace { return ns } + +func TeardownNamespace(ns string) { + log := ctx.Ctx().Logf + + currentTest := CurrentGinkgoTestDescription() + if currentTest.Failed { + log("collecting the %s namespace artifacts as the '%s' test case failed", ns, currentTest.TestText) + if err := ctx.Ctx().DumpNamespaceArtifacts(ns); err != nil { + log("failed to collect namespace artifacts: %v", err) + } + } + + log("tearing down the %s namespace", ns) + Eventually(func() error { + return ctx.Ctx().KubeClient().KubernetesInterface().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) + }).Should(Succeed()) +} From 95ba03730dacf386bf656bb35317c5b23688366c Mon Sep 17 00:00:00 2001 From: timflannagan Date: Tue, 9 Nov 2021 22:08:49 -0500 Subject: [PATCH 2/2] test/e2e: Refactor the bundle e2e tests to avoid using global test namespace Signed-off-by: timflannagan --- test/e2e/bundle_e2e_test.go | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/test/e2e/bundle_e2e_test.go b/test/e2e/bundle_e2e_test.go index febd897de2..6c3791d482 100644 --- a/test/e2e/bundle_e2e_test.go +++ b/test/e2e/bundle_e2e_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "github.com/ghodss/yaml" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -27,19 +28,23 @@ var vpaCRDRaw []byte var _ = Describe("Installing bundles with new object types", func() { var ( - kubeClient operatorclient.ClientInterface - operatorClient versioned.Interface - dynamicClient dynamic.Interface + kubeClient operatorclient.ClientInterface + operatorClient versioned.Interface + dynamicClient dynamic.Interface + generatedNamespace corev1.Namespace ) BeforeEach(func() { kubeClient = ctx.Ctx().KubeClient() operatorClient = ctx.Ctx().OperatorClient() dynamicClient = ctx.Ctx().DynamicClient() + + By("creating a test namespace") + generatedNamespace = SetupGeneratedTestNamespace(genName("bundle-e2e-")) }) AfterEach(func() { - TearDown(testNamespace) + TeardownNamespace(generatedNamespace.GetName()) }) When("a bundle with a pdb, priorityclass, and VPA object is installed", func() { @@ -66,7 +71,7 @@ var _ = Describe("Installing bundles with new object types", func() { Expect(err).ToNot(HaveOccurred(), "could not convert vpa crd to unstructured") Eventually(func() error { - err := ctx.Ctx().Client().Create(context.TODO(), &vpaCRD) + err := ctx.Ctx().Client().Create(context.Background(), &vpaCRD) if err != nil { if !k8serrors.IsAlreadyExists(err) { return err @@ -77,14 +82,13 @@ var _ = Describe("Installing bundles with new object types", func() { // ensure vpa crd is established and accepted on the cluster before continuing Eventually(func() (bool, error) { - crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), vpaCRD.GetName(), metav1.GetOptions{}) + crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), vpaCRD.GetName(), metav1.GetOptions{}) if err != nil { return false, err } return crdReady(&crd.Status), nil }).Should(BeTrue()) - var installPlanRef string source := &v1alpha1.CatalogSource{ TypeMeta: metav1.TypeMeta{ Kind: v1alpha1.CatalogSourceKind, @@ -92,7 +96,7 @@ var _ = Describe("Installing bundles with new object types", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: sourceName, - Namespace: testNamespace, + Namespace: generatedNamespace.GetName(), Labels: map[string]string{"olm.catalogSource": sourceName}, }, Spec: v1alpha1.CatalogSourceSpec{ @@ -102,7 +106,7 @@ var _ = Describe("Installing bundles with new object types", func() { } Eventually(func() error { - source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{}) + source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.Background(), source, metav1.CreateOptions{}) return err }).Should(Succeed()) @@ -114,13 +118,13 @@ var _ = Describe("Installing bundles with new object types", func() { _ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic) // Wait for the Subscription to succeed - sub, err := fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker) + sub, err := fetchSubscription(operatorClient, generatedNamespace.GetName(), subName, subscriptionStateAtLatestChecker) Expect(err).ToNot(HaveOccurred(), "could not get subscription at latest status") - installPlanRef = sub.Status.InstallPlanRef.Name + installPlanRef := sub.Status.InstallPlanRef // Wait for the installplan to complete (5 minute timeout) - _, err = fetchInstallPlan(GinkgoT(), operatorClient, installPlanRef, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) + _, err = fetchInstallPlanWithNamespace(GinkgoT(), operatorClient, installPlanRef.Name, installPlanRef.Namespace, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) Expect(err).ToNot(HaveOccurred(), "could not get installplan at complete phase") ctx.Ctx().Logf("install plan %s completed", installPlanRef) @@ -144,17 +148,17 @@ var _ = Describe("Installing bundles with new object types", func() { // confirm extra bundle objects are installed Eventually(func() error { - _, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.TODO(), priorityClassName, metav1.GetOptions{}) + _, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.Background(), priorityClassName, metav1.GetOptions{}) return err }).Should(Succeed(), "expected no error getting priorityclass object associated with CSV") Eventually(func() error { - _, err := dynamicClient.Resource(resource).Namespace(testNamespace).Get(context.TODO(), vpaName, metav1.GetOptions{}) + _, err := dynamicClient.Resource(resource).Namespace(generatedNamespace.GetName()).Get(context.Background(), vpaName, metav1.GetOptions{}) return err }).Should(Succeed(), "expected no error finding vpa object associated with csv") Eventually(func() error { - _, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(testNamespace).Get(context.TODO(), pdbName, metav1.GetOptions{}) + _, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(generatedNamespace.GetName()).Get(context.Background(), pdbName, metav1.GetOptions{}) return err }).Should(Succeed(), "expected no error getting pdb object associated with CSV") }) @@ -162,7 +166,7 @@ var _ = Describe("Installing bundles with new object types", func() { AfterEach(func() { By("Deleting the VPA CRD") Eventually(func() error { - err := ctx.Ctx().Client().Delete(context.TODO(), &vpaCRD) + err := ctx.Ctx().Client().Delete(context.Background(), &vpaCRD) if k8serrors.IsNotFound(err) { return nil }