Add e2e kube client (#2712)

Signed-off-by: perdasilva <[email protected]>

Author: perdasilva

test/e2e/ctx/ctx.go

@@ -7,8 +7,12 @@ import (
 	"path/filepath"
 	"strings"
 
-	g "github.com/onsi/ginkgo"
+	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/util"
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+
+	g "github.com/onsi/ginkgo"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/dynamic"
@@ -35,6 +39,7 @@ type TestContext struct {
 	operatorClient versioned.Interface
 	dynamicClient  dynamic.Interface
 	packageClient  pversioned.Interface
+	e2eClient      *util.E2EKubeClient
 	ssaClient      *controllerclient.ServerSideApplier
 
 	kubeconfigPath string
@@ -93,6 +98,17 @@ func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
 	return ctx.ssaClient
 }
 
+func (ctx TestContext) E2EClient() *util.E2EKubeClient {
+	return ctx.e2eClient
+}
+
+func (ctx TestContext) NewE2EClientSession() {
+	if ctx.e2eClient != nil {
+		_ = ctx.e2eClient.Reset()
+	}
+	ctx.e2eClient = util.NewK8sResourceManager(ctx.Client())
+}
+
 func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
 	if ctx.artifactsDir == "" {
 		ctx.Logf("$ARTIFACTS_DIR is unset -- not collecting failed test case logs")
@@ -163,12 +179,14 @@ func setDerivedFields(ctx *TestContext) error {
 
 	ctx.scheme = runtime.NewScheme()
 	localSchemeBuilder := runtime.NewSchemeBuilder(
+		apiextensions.AddToScheme,
 		kscheme.AddToScheme,
 		operatorsv1alpha1.AddToScheme,
 		operatorsv1.AddToScheme,
 		operatorsv2.AddToScheme,
 		apiextensionsv1.AddToScheme,
-		apiextensions.AddToScheme,
+		appsv1.AddToScheme,
+		apiregistrationv1.AddToScheme,
 	)
 	if err := localSchemeBuilder.AddToScheme(ctx.scheme); err != nil {
 		return err
@@ -181,6 +199,7 @@ func setDerivedFields(ctx *TestContext) error {
 		return err
 	}
 	ctx.client = client
+	ctx.e2eClient = util.NewK8sResourceManager(client)
 
 	ctx.ssaClient, err = controllerclient.NewForConfig(ctx.restConfig, ctx.scheme, "test.olm.registry")
 	if err != nil {

test/e2e/resource_manager_test.go

@@ -0,0 +1,78 @@
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/util"
+	corev1 "k8s.io/api/core/v1"
+	k8serror "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
+)
+
+var _ = Describe("ResourceManager", func() {
+
+	var generatedNamespace corev1.Namespace
+
+	BeforeEach(func() {
+		ctx.Ctx().NewE2EClientSession()
+		generatedNamespace = SetupGeneratedTestNamespace(genName("resource-manager-e2e-"))
+	})
+
+	AfterEach(func() {
+		TeardownNamespace(generatedNamespace.GetName())
+		Expect(ctx.Ctx().E2EClient().Reset()).To(Succeed())
+	})
+
+	It("should tag resources created with it", func() {
+		// Create a namespace
+		ns := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: genName("test-"),
+			},
+		}
+		Expect(ctx.Ctx().E2EClient().Create(context.TODO(), ns)).To(Succeed())
+
+		// Get namespace
+		Expect(ctx.Ctx().E2EClient().Get(context.TODO(), client.ObjectKeyFromObject(ns), ns)).To(Succeed())
+		Expect(ns.GetAnnotations()).NotTo(BeEmpty())
+		Expect(ns.GetAnnotations()[util.E2ETestNameTag]).To(Equal("ResourceManager should tag resources created with it"))
+	})
+
+	It("should delete resources on reset", func() {
+		// Create a namespace
+		ns := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: genName("test-"),
+			},
+		}
+		Expect(ctx.Ctx().E2EClient().Create(context.TODO(), ns)).To(Succeed())
+
+		// Add a config map
+		configMap := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: genName("configmap-"),
+
+				// creating the configmap in the generated (spec) namespace
+				// so if the namespace (ns, above) gets deleted on reset it won't take the config map with it
+				Namespace: generatedNamespace.GetName(),
+			},
+		}
+		Expect(ctx.Ctx().E2EClient().Create(context.TODO(), configMap))
+
+		// Reset the client
+		Expect(ctx.Ctx().E2EClient().Reset()).To(Succeed())
+
+		// And just like that resources should be gone
+		Eventually(func() error {
+			return ctx.Ctx().E2EClient().Get(context.TODO(), client.ObjectKeyFromObject(configMap), configMap)
+		}).Should(WithTransform(k8serror.IsNotFound, BeTrue()))
+		Eventually(func() error {
+			return ctx.Ctx().E2EClient().Get(context.TODO(), client.ObjectKeyFromObject(ns), ns)
+		}).Should(WithTransform(k8serror.IsNotFound, BeTrue()))
+	})
+})

test/e2e/util/e2e_client.go

@@ -0,0 +1,75 @@
+package util
+
+import (
+	"context"
+
+	"github.com/onsi/ginkgo"
+	k8serror "k8s.io/apimachinery/pkg/api/errors"
+	k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	E2ETestNameTag = "e2e.testName"
+)
+
+type E2EKubeClient struct {
+	k8scontrollerclient.Client
+	createdResources *ResourceQueue
+}
+
+func NewK8sResourceManager(client k8scontrollerclient.Client) *E2EKubeClient {
+	return &E2EKubeClient{
+		Client:           client,
+		createdResources: NewResourceQueue(),
+	}
+}
+
+func (m *E2EKubeClient) Create(context context.Context, obj k8scontrollerclient.Object, options ...k8scontrollerclient.CreateOption) error {
+	m.annotateTestResource(obj)
+	if err := m.Client.Create(context, obj, options...); err != nil {
+		return err
+	}
+	m.createdResources.EnqueueIgnoreExisting(obj)
+	return nil
+}
+
+func (m *E2EKubeClient) Update(context context.Context, obj k8scontrollerclient.Object, options ...k8scontrollerclient.UpdateOption) error {
+	m.annotateTestResource(obj)
+	if err := m.Client.Update(context, obj, options...); err != nil {
+		return err
+	}
+	m.createdResources.EnqueueIgnoreExisting(obj)
+	return nil
+}
+
+func (m *E2EKubeClient) Delete(context context.Context, obj k8scontrollerclient.Object, options ...k8scontrollerclient.DeleteOption) error {
+	if err := m.Client.Delete(context, obj, options...); err != nil {
+		return err
+	}
+	m.createdResources.RemoveIgnoreNotFound(obj)
+	return nil
+}
+
+func (m *E2EKubeClient) Reset() error {
+	for {
+		obj, ok := m.createdResources.DequeueTail()
+
+		if !ok {
+			break
+		}
+
+		if err := m.Delete(context.TODO(), obj); err != nil && !k8serror.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *E2EKubeClient) annotateTestResource(obj k8scontrollerclient.Object) {
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = make(map[string]string)
+	}
+	annotations[E2ETestNameTag] = ginkgo.CurrentGinkgoTestDescription().FullTestText
+	obj.SetAnnotations(annotations)
+}

test/e2e/util/resource_queue.go

@@ -0,0 +1,100 @@
+package util
+
+import (
+	"fmt"
+	k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sync"
+)
+
+type ResourceQueue struct {
+	queue       []k8scontrollerclient.Object
+	lookupTable map[k8scontrollerclient.Object]bool
+	lock        *sync.Mutex
+}
+
+func NewResourceQueue() *ResourceQueue {
+	return &ResourceQueue{
+		queue:       []k8scontrollerclient.Object{},
+		lookupTable: map[k8scontrollerclient.Object]bool{},
+		lock:        &sync.Mutex{},
+	}
+}
+
+func (q *ResourceQueue) Enqueue(obj k8scontrollerclient.Object) error {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	if _, ok := q.lookupTable[obj]; ok {
+		return fmt.Errorf("error inserting duplicate object: %s", obj)
+	}
+	q.queue = append(q.queue, obj)
+	q.lookupTable[obj] = true
+	return nil
+}
+
+func (q *ResourceQueue) EnqueueIgnoreExisting(obj k8scontrollerclient.Object) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	if _, ok := q.lookupTable[obj]; ok {
+		return
+	}
+	q.queue = append(q.queue, obj)
+	q.lookupTable[obj] = true
+}
+
+func (q *ResourceQueue) Length() int {
+	return len(q.queue)
+}
+
+func (q *ResourceQueue) RemoveIgnoreNotFound(obj k8scontrollerclient.Object) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	if _, ok := q.lookupTable[obj]; ok {
+		for index, existingObj := range q.queue {
+			if q.equals(existingObj, obj) {
+				_ = q.removeItem(index)
+				delete(q.lookupTable, obj)
+				return
+			}
+		}
+	}
+}
+
+func (q *ResourceQueue) equals(objOne k8scontrollerclient.Object, objTwo k8scontrollerclient.Object) bool {
+	return objOne.GetName() == objTwo.GetName() && objOne.GetNamespace() == objTwo.GetNamespace()
+}
+
+func (q *ResourceQueue) DequeueHead() (k8scontrollerclient.Object, bool) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	if q.Length() == 0 {
+		return nil, false
+	}
+	return q.removeItem(0), true
+}
+
+func (q *ResourceQueue) DequeueTail() (k8scontrollerclient.Object, bool) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	
+	if len(q.queue) == 0 {
+		return nil, false
+	}
+	return q.removeItem(q.Length() - 1), true
+}
+
+func (q *ResourceQueue) removeItem(index int) k8scontrollerclient.Object {
+	if index < 0 || index >= q.Length() {
+		panic("index out of bounds")
+	}
+	item := q.queue[index]
+	copy(q.queue[index:], q.queue[index+1:])
+	q.queue[len(q.queue)-1] = nil
+	q.queue = q.queue[:len(q.queue)-1]
+	delete(q.lookupTable, item)
+
+	return item
+}