From e8451f193bd6f060853c6de604b36248cbb75899 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 20 Mar 2025 11:11:49 -0400 Subject: [PATCH] create e2e testing queues via unstructured This eliminates the go-level dependency on Kueue APIs, which allows us for a looser coupling between AppWrapper and Kueue. --- go.mod | 1 - go.sum | 2 -- test/e2e/util_test.go | 83 +++++++++++++++++++++++++------------------ 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index bcc6522..6616bec 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( sigs.k8s.io/controller-runtime v0.20.3 sigs.k8s.io/controller-tools v0.16.5 sigs.k8s.io/jobset v0.8.0 - sigs.k8s.io/kueue v0.11.0 sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 511976a..3cd8485 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,6 @@ sigs.k8s.io/jobset v0.8.0 h1:80cJcPld+IMdKFOqzEW4et3Y6lGAPcP8YmBZ+aiKGYA= sigs.k8s.io/jobset v0.8.0/go.mod h1:yitjuGOExl2p964nhyevQGIkfiPSRHcdC3zNBneKCT8= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/kueue v0.11.0 h1:8MjV7xPucsD+XyNnSN5RE5jrW4qzM/c9SP3uNqAi24E= -sigs.k8s.io/kueue v0.11.0/go.mod h1:v26xR+cDxA53k9MNxfLD2t42FLIYJUu6V55IpAEQ8aY= sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 h1:o1mtt6vpxsxDYaZKrw3BnEtc+pAjLz7UffnIvHNbvW0= diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index d14d733..1b680b2 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -26,7 +26,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -34,8 +34,7 @@ import ( "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - kc "sigs.k8s.io/kueue/pkg/controller/constants" + "sigs.k8s.io/yaml" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,9 +44,8 @@ import ( ) const ( - testNamespace = "e2e-test" - testFlavorName = "e2e-test-flavor" - testQueueName = "e2e-test-queue" + testNamespace = "e2e-test" + testQueueName = "e2e-test-queue" ) type myKey struct { @@ -69,7 +67,6 @@ func extendContextWithClient(ctx context.Context) context.Context { scheme := runtime.NewScheme() Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) Expect(awv1beta2.AddToScheme(scheme)).To(Succeed()) - Expect(kueue.AddToScheme(scheme)).To(Succeed()) // Create a client with full permissions k8sClient, err := client.New(baseConfig, client.Options{Scheme: scheme}) @@ -115,34 +112,52 @@ func extendContextWithLimitedClient(ctx context.Context) context.Context { return context.WithValue(ctx, myKey{key: "kubelimitedclient"}, limitedClient) } -func ensureTestQueuesExist(ctx context.Context) { - rf := &kueue.ResourceFlavor{ObjectMeta: metav1.ObjectMeta{Name: testFlavorName}} - err := getClient(ctx).Create(ctx, rf) - Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) - cq := &kueue.ClusterQueue{ - ObjectMeta: metav1.ObjectMeta{Name: testQueueName}, - Spec: kueue.ClusterQueueSpec{ - NamespaceSelector: &metav1.LabelSelector{}, - ResourceGroups: []kueue.ResourceGroup{{ - CoveredResources: []v1.ResourceName{v1.ResourceCPU, "nvidia.com/gpu"}, - Flavors: []kueue.FlavorQuotas{{ - Name: testFlavorName, - Resources: []kueue.ResourceQuota{{Name: v1.ResourceCPU, NominalQuota: *resource.NewMilliQuantity(2000, resource.DecimalSI)}, - {Name: "nvidia.com/gpu", NominalQuota: *resource.NewQuantity(2, resource.DecimalSI)}}, - }}, - }, - }, - }, - } - err = getClient(ctx).Create(ctx, cq) +const flavorYAML = ` +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ResourceFlavor +metadata: + name: e2e-test-flavor +` +const clusterQueueYAML = ` +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ClusterQueue +metadata: + name: ` + testQueueName + ` +spec: + namespaceSelector: {} + resourceGroups: + - coveredResources: [cpu, nvidia.com/gpu] + flavors: + - name: e2e-test-flavor + resources: + - name: cpu + nominalQuota: 2000m + - name: nvidia.com/gpu + nominalQuota: 2 +` +const localQueueYAML = ` +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + namespace: ` + testNamespace + ` + name: ` + testQueueName + ` +spec: + clusterQueue: ` + testQueueName + +func createFromYaml(ctx context.Context, yamlString string) { + jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString)) + Expect(err).NotTo(HaveOccurred()) + obj := &unstructured.Unstructured{} + _, _, err = unstructured.UnstructuredJSONScheme.Decode(jsonBytes, nil, obj) + Expect(err).NotTo(HaveOccurred()) + err = getClient(ctx).Create(ctx, obj) Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) +} - lq := &kueue.LocalQueue{ - ObjectMeta: metav1.ObjectMeta{Name: testQueueName, Namespace: testNamespace}, - Spec: kueue.LocalQueueSpec{ClusterQueue: kueue.ClusterQueueReference(testQueueName)}, - } - err = getClient(ctx).Create(ctx, lq) - Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) +func ensureTestQueuesExist(ctx context.Context) { + createFromYaml(ctx, flavorYAML) + createFromYaml(ctx, clusterQueueYAML) + createFromYaml(ctx, localQueueYAML) } func cleanupTestObjects(ctx context.Context, appwrappers []*awv1beta2.AppWrapper) { @@ -181,7 +196,7 @@ func toAppWrapper(components ...awv1beta2.AppWrapperComponent) *awv1beta2.AppWra ObjectMeta: metav1.ObjectMeta{ Name: randName("aw"), Namespace: testNamespace, - Labels: map[string]string{kc.QueueLabel: testQueueName}, + Labels: map[string]string{"kueue.x-k8s.io/queue-name": testQueueName}, }, Spec: awv1beta2.AppWrapperSpec{Components: components}, }