Test runtimextension integration

Signed-off-by: Danil-Grigorev <[email protected]>

Author: Danil-Grigorev

Makefile

@@ -618,6 +618,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml
 	$(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml
 
 .PHONY: generate-metrics-config

test/e2e/cluster_upgrade_runtimesdk.go

@@ -95,6 +95,9 @@ type ClusterUpgradeWithRuntimeSDKSpecInput struct {
 	ExtensionServiceNamespace string
 	// ExtensionServiceName is the name of the service to configure in the test-namespace scoped ExtensionConfig.
 	ExtensionServiceName string
+
+	// ClassNamespace is an optional class namespace reference, configuring cross-namespace cluster class reference
+	ClassNamespace bool
 }
 
 // ClusterUpgradeWithRuntimeSDKSpec implements a spec that upgrades a cluster and runs the Kubernetes conformance suite.
@@ -109,9 +112,9 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 	)
 
 	var (
-		input         ClusterUpgradeWithRuntimeSDKSpecInput
-		namespace     *corev1.Namespace
-		cancelWatches context.CancelFunc
+		input                             ClusterUpgradeWithRuntimeSDKSpecInput
+		namespace, infraNamespace         *corev1.Namespace
+		cancelWatches, cancelInfraWatches context.CancelFunc
 
 		controlPlaneMachineCount int64
 		workerMachineCount       int64
@@ -148,6 +151,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 
 		// Set up a Namespace where to host objects for this spec and create a watcher for the Namespace events.
 		namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
+		infraNamespace, cancelInfraWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
 		clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
 		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
 	})
@@ -162,7 +166,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 		By("Deploy Test Extension ExtensionConfig")
 
 		Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
-			extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))).
+			extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName, namespace.Name, infraNamespace.Name))).
 			To(Succeed(), "Failed to create the extension config")
 
 		By("Creating a workload cluster; creation waits for BeforeClusterCreateHook to gate the operation")
@@ -177,6 +181,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 			infrastructureProvider = *input.InfrastructureProvider
 		}
 
+		variables := map[string]string{}
+		if input.ClassNamespace {
+			variables["CLUSTER_CLASS_NAMESPACE"] = infraNamespace.Name
+		}
+
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: input.BootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
@@ -190,6 +199,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
 				ControlPlaneMachineCount: ptr.To[int64](controlPlaneMachineCount),
 				WorkerMachineCount:       ptr.To[int64](workerMachineCount),
+				ClusterctlVariables:      variables,
 			},
 			PreWaitForCluster: func() {
 				beforeClusterCreateTestHandler(ctx,
@@ -304,7 +314,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 		if !input.SkipCleanup {
 			// Delete the extensionConfig first to ensure the BeforeDeleteCluster hook doesn't block deletion.
 			Eventually(func() error {
-				return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))
+				return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName))
 			}, 10*time.Second, 1*time.Second).Should(Succeed(), "delete extensionConfig failed")
 
 			Byf("Deleting cluster %s", klog.KObj(clusterResources.Cluster))
@@ -322,8 +332,17 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
 				Deleter: input.BootstrapClusterProxy.GetClient(),
 				Name:    namespace.Name,
 			})
+
+			if input.ClassNamespace {
+				Byf("Deleting namespace used for optionally hosting the %q infrastructure spec", specName)
+				framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
+					Deleter: input.BootstrapClusterProxy.GetClient(),
+					Name:    infraNamespace.Name,
+				})
+			}
 		}
 		cancelWatches()
+		cancelInfraWatches()
 	})
 }
 
@@ -429,7 +448,7 @@ func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client,
 // We make sure this cluster-wide object does not conflict with others by using a random generated
 // name and a NamespaceSelector selecting on the namespace of the current test.
 // Thus, this object is "namespaced" to the current test even though it's a cluster-wide object.
-func extensionConfig(name, namespace, extensionServiceNamespace, extensionServiceName string) *runtimev1.ExtensionConfig {
+func extensionConfig(name, extensionServiceNamespace, extensionServiceName string, namespaces ...string) *runtimev1.ExtensionConfig {
 	return &runtimev1.ExtensionConfig{
 		ObjectMeta: metav1.ObjectMeta{
 			// Note: We have to use a constant name here as we have to be able to reference it in the ClusterClass
@@ -454,7 +473,7 @@ func extensionConfig(name, namespace, extensionServiceNamespace, extensionServic
 					{
 						Key:      "kubernetes.io/metadata.name",
 						Operator: metav1.LabelSelectorOpIn,
-						Values:   []string{namespace},
+						Values:   namespaces,
 					},
 				},
 			},

test/e2e/cluster_upgrade_runtimesdk_test.go

@@ -60,3 +60,36 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass with Runt
 		}
 	})
 })
+
+var _ = Describe("When upgrading a workload cluster using ClusterClass in a different NS with RuntimeSDK [PR-Blocking] [ClusterClass]", func() {
+	ClusterUpgradeWithRuntimeSDKSpec(ctx, func() ClusterUpgradeWithRuntimeSDKSpecInput {
+		version, err := semver.ParseTolerant(e2eConfig.GetVariable(KubernetesVersionUpgradeFrom))
+		Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version")
+		if version.LT(semver.MustParse("1.24.0")) {
+			Fail("This test only supports upgrades from Kubernetes >= v1.24.0")
+		}
+
+		return ClusterUpgradeWithRuntimeSDKSpecInput{
+			E2EConfig:              e2eConfig,
+			ClusterctlConfigPath:   clusterctlConfigPath,
+			BootstrapClusterProxy:  bootstrapClusterProxy,
+			ArtifactFolder:         artifactFolder,
+			SkipCleanup:            skipCleanup,
+			InfrastructureProvider: ptr.To("docker"),
+			PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
+				// continuous reconciles when everything should be stable.
+				framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+			},
+			// "upgrades" is the same as the "topology" flavor but with an additional MachinePool.
+			Flavor:         ptr.To("cross-ns-upgrades-runtimesdk"),
+			ClassNamespace: true,
+			// The runtime extension gets deployed to the test-extension-system namespace and is exposed
+			// by the test-extension-webhook-service.
+			// The below values are used when creating the cluster-wide ExtensionConfig to refer
+			// the actual service.
+			ExtensionServiceNamespace: "test-extension-system",
+			ExtensionServiceName:      "test-extension-webhook-service",
+		}
+	})
+})

test/e2e/config/docker.yaml

@@ -348,6 +348,7 @@ providers:
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-ignition.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-topology.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml"
@@ -377,10 +378,10 @@ variables:
   # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation.
   # The following Kubernetes versions should be the latest versions with already published kindest/node images.
   # This avoids building node images in the default case which improves the test duration significantly.
-  KUBERNETES_VERSION_MANAGEMENT: "v1.32.0-beta.0"
-  KUBERNETES_VERSION: "v1.32.0-beta.0"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.31.0"
+  KUBERNETES_VERSION: "v1.31.0"
   KUBERNETES_VERSION_UPGRADE_FROM: "v1.31.0"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.32.0-beta.0"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.31.0"
   KUBERNETES_VERSION_LATEST_CI: "ci/latest-1.32"
   ETCD_VERSION_UPGRADE_TO: "3.5.16-0"
   COREDNS_VERSION_UPGRADE_TO: "v1.11.3"

test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/cluster.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+spec:
+  topology:
+    classNamespace: '${CLUSTER_CLASS_NAMESPACE}'

test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../cluster-template-upgrades-runtimesdk
+
+patches:
+- path: cluster.yaml