diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go index 4f7e7a3dc6e4..f65449c29a7d 100644 --- a/test/e2e/cluster_upgrade_runtimesdk.go +++ b/test/e2e/cluster_upgrade_runtimesdk.go @@ -29,15 +29,22 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1" + "sigs.k8s.io/cluster-api/test/e2e/internal/log" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" ) +var hookFailedMessage = "hook failed" + // clusterUpgradeWithRuntimeSDKSpecInput is the input for clusterUpgradeWithRuntimeSDKSpec. type clusterUpgradeWithRuntimeSDKSpecInput struct { E2EConfig *clusterctl.E2EConfig @@ -113,7 +120,7 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl workerMachineCount = *input.WorkerMachineCount } - // Setup a Namespace where to host objects for this spec and create a watcher for the Namespace events. + // Set up a Namespace where to host objects for this spec and create a watcher for the Namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) }) @@ -156,6 +163,12 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl ControlPlaneMachineCount: pointer.Int64Ptr(controlPlaneMachineCount), WorkerMachineCount: pointer.Int64Ptr(workerMachineCount), }, + PreWaitForCluster: func() { + beforeClusterCreateTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + namespace.Name, clusterName, + input.E2EConfig.GetIntervals(specName, "wait-cluster")) + }, WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), @@ -176,6 +189,21 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), WaitForDNSUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), WaitForEtcdUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + PreWaitForControlPlaneToBeUpgraded: func() { + beforeClusterUpgradeTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + namespace.Name, + clusterName, + input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) + }, + PreWaitForMachineDeploymentToBeUpgraded: func() { + afterControlPlaneUpgradeTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + namespace.Name, + clusterName, + input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) + }, }) // Only attempt to upgrade MachinePools if they were provided in the template. @@ -201,13 +229,13 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl }) By("Checking all lifecycle hooks have been called") - // Assert that each hook passed to this function is marked as "true" in the response configmap - err = checkLifecycleHooks(ctx, input.BootstrapClusterProxy.GetClient(), namespace.Name, clusterName, map[string]string{ - "BeforeClusterCreate": "", - "BeforeClusterUpgrade": "", - "AfterControlPlaneInitialized": "", - "AfterControlPlaneUpgrade": "", - "AfterClusterUpgrade": "", + // Assert that each hook has been called and returned "Success" during the test. + err = checkLifecycleHookResponses(ctx, input.BootstrapClusterProxy.GetClient(), namespace.Name, clusterName, map[string]string{ + "BeforeClusterCreate": "Success", + "BeforeClusterUpgrade": "Success", + "AfterControlPlaneInitialized": "Success", + "AfterControlPlaneUpgrade": "Success", + "AfterClusterUpgrade": "Success", }) Expect(err).ToNot(HaveOccurred(), "Lifecycle hook calls were not as expected") @@ -266,26 +294,193 @@ func responsesConfigMap(name string, namespace *corev1.Namespace) *corev1.Config Name: fmt.Sprintf("%s-hookresponses", name), Namespace: namespace.Name, }, - // Every response contain only Status:Success. The test checks whether each handler has been called at least once. + // Set the initial preloadedResponses for each of the tested hooks. Data: map[string]string{ - "BeforeClusterCreate-response": `{"Status": "Success"}`, - "BeforeClusterUpgrade-response": `{"Status": "Success"}`, - "AfterControlPlaneInitialized-response": `{"Status": "Success"}`, - "AfterControlPlaneUpgrade-response": `{"Status": "Success"}`, - "AfterClusterUpgrade-response": `{"Status": "Success"}`, + // Blocking hooks are set to Status:Failure initially. These will be changed during the test. + "BeforeClusterCreate-preloadedResponse": fmt.Sprintf(`{"Status": "Failure", "Message": %q}`, hookFailedMessage), + "BeforeClusterUpgrade-preloadedResponse": fmt.Sprintf(`{"Status": "Failure", "Message": %q}`, hookFailedMessage), + "AfterControlPlaneUpgrade-preloadedResponse": fmt.Sprintf(`{"Status": "Failure", "Message": %q}`, hookFailedMessage), + + // Non-blocking hooks are set to Status:Success. + "AfterControlPlaneInitialized-preloadedResponse": `{"Status": "Success"}`, + "AfterClusterUpgrade-preloadedResponse": `{"Status": "Success"}`, }, } } -func checkLifecycleHooks(ctx context.Context, c client.Client, namespace string, clusterName string, hooks map[string]string) error { - configMap := &corev1.ConfigMap{} - configMapName := clusterName + "-hookresponses" - err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap) - Expect(err).ToNot(HaveOccurred(), "Failed to get the hook response configmap") - for hook := range hooks { - if _, ok := configMap.Data[hook+"-called"]; !ok { - return errors.Errorf("hook %s call not recorded in configMap %s/%s", hook, namespace, configMapName) +// Check that each hook in hooks has been called at least once by checking if its actualResponseStatus is in the hook response configmap. +// If the provided hooks have both keys and values check that the values match those in the hook response configmap. +func checkLifecycleHookResponses(ctx context.Context, c client.Client, namespace string, clusterName string, expectedHookResponses map[string]string) error { + responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, namespace, clusterName) + for hookName, expectedResponse := range expectedHookResponses { + actualResponse, ok := responseData[hookName+"-actualResponseStatus"] + if !ok { + return errors.Errorf("hook %s call not recorded in configMap %s/%s", hookName, namespace, clusterName+"-hookresponses") + } + if expectedResponse != "" && expectedResponse != actualResponse { + return errors.Errorf("hook %s was expected to be %s in configMap got %s", hookName, expectedResponse, actualResponse) + } + } + return nil +} + +// Check that each hook in expectedHooks has been called at least once by checking if its actualResponseStatus is in the hook response configmap. +func checkLifecycleHooksCalledAtLeastOnce(ctx context.Context, c client.Client, namespace string, clusterName string, expectedHooks []string) error { + responseData := getLifecycleHookResponsesFromConfigMap(ctx, c, namespace, clusterName) + for _, hookName := range expectedHooks { + if _, ok := responseData[hookName+"-actualResponseStatus"]; !ok { + return errors.Errorf("hook %s call not recorded in configMap %s/%s", hookName, namespace, clusterName+"-hookresponses") } } return nil } + +func getLifecycleHookResponsesFromConfigMap(ctx context.Context, c client.Client, namespace string, clusterName string) map[string]string { + configMap := &corev1.ConfigMap{} + configMapName := clusterName + "-hookresponses" + Eventually(func() error { + return c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap) + }).Should(Succeed(), "Failed to get the hook response configmap") + return configMap.Data +} + +// beforeClusterCreateTestHandler calls runtimeHookTestHandler with a blockedCondition function which returns false if +// the Cluster has entered ClusterPhaseProvisioned. +func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, namespace, clusterName string, intervals []interface{}) { + log.Logf("Blocking with BeforeClusterCreate hook") + hookName := "BeforeClusterCreate" + runtimeHookTestHandler(ctx, c, namespace, clusterName, hookName, func() bool { + blocked := true + // This hook should block the Cluster from entering the "Provisioned" state. + cluster := &clusterv1.Cluster{} + Eventually(func() error { + return c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, cluster) + }).Should(Succeed()) + + // Check if the TopologyReconciled condition message contains both the hook name and hookFailedMessage. + if !clusterConditionShowsHookFailed(cluster, hookName) { + blocked = false + } + if cluster.Status.Phase == string(clusterv1.ClusterPhaseProvisioned) { + blocked = false + } + return blocked + }, intervals) +} + +// beforeClusterUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if the +// Cluster has controlplanev1.RollingUpdateInProgressReason in its ReadyCondition. +func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, namespace, clusterName string, intervals []interface{}) { + log.Logf("Blocking with BeforeClusterUpgrade hook") + hookName := "BeforeClusterUpgrade" + runtimeHookTestHandler(ctx, c, namespace, clusterName, hookName, func() bool { + var blocked = true + + cluster := &clusterv1.Cluster{} + Eventually(func() error { + return c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, cluster) + }).Should(Succeed()) + + // Check if the TopologyReconciled condition message contains both the hook name and hookFailedMessage. + if !clusterConditionShowsHookFailed(cluster, hookName) { + blocked = false + } + // Check if the Cluster is showing the RollingUpdateInProgress condition reason. If it has the update process is unblocked. + if conditions.IsFalse(cluster, clusterv1.ReadyCondition) && + conditions.GetReason(cluster, clusterv1.ReadyCondition) == controlplanev1.RollingUpdateInProgressReason { + blocked = false + } + return blocked + }, intervals) +} + +// afterControlPlaneUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if any +// MachineDeployment in the Cluster has upgraded to the target Kubernetes version. +func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, namespace, clusterName, version string, intervals []interface{}) { + log.Logf("Blocking with AfterControlPlaneUpgrade hook") + hookName := "AfterControlPlaneUpgrade" + runtimeHookTestHandler(ctx, c, namespace, clusterName, hookName, func() bool { + var blocked = true + cluster := &clusterv1.Cluster{} + Eventually(func() error { + return c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, cluster) + }).Should(Succeed()) + + // Check if the TopologyReconciled condition message contains both the hook name and hookFailedMessage. + if !clusterConditionShowsHookFailed(cluster, hookName) { + blocked = false + } + + mds := &clusterv1.MachineDeploymentList{} + Eventually(func() error { + return c.List(ctx, mds, client.MatchingLabels{ + clusterv1.ClusterLabelName: clusterName, + clusterv1.ClusterTopologyOwnedLabel: "", + }) + }).Should(Succeed()) + + // If any of the MachineDeployments have the target Kubernetes Version, the hook is unblocked. + for _, md := range mds.Items { + if *md.Spec.Template.Spec.Version == version { + blocked = false + } + } + return blocked + }, intervals) +} + +// runtimeHookTestHandler runs a series of tests in sequence to check if the runtimeHook passed to it succeeds. +// 1) Checks that the hook has been called at least once the TopologyReconciled condition is a Failure. +// 2) Check that the hook's blockingCondition is consistently true. +// - At this point the function sets the hook's response to be non-blocking. +// 3) Check that the hook's blocking condition becomes false. +// Note: runtimeHookTestHandler assumes that the hook passed to it is currently returning a blocking response. +// Updating the response to be non-blocking happens inline in the function. +func runtimeHookTestHandler(ctx context.Context, c client.Client, namespace, clusterName, hookName string, blockingCondition func() bool, intervals []interface{}) { + // Check that the LifecycleHook has been called at least once and the TopologyReconciled condition is a Failure. + Eventually(func() error { + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, namespace, clusterName, []string{hookName}); err != nil { + return err + } + cluster := &clusterv1.Cluster{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, cluster); err != nil { + return err + } + if !(conditions.GetReason(cluster, clusterv1.TopologyReconciledCondition) == clusterv1.TopologyReconcileFailedReason) { + return errors.New("Condition not found on Cluster object") + } + return nil + }, 60*time.Second).Should(Succeed(), "%s has not been called", hookName) + + // blockingCondition should consistently be true as the Runtime hook is returning "Failure". + Consistently(func() bool { + return blockingCondition() + }, 30*time.Second).Should(BeTrue(), + fmt.Sprintf("Cluster Topology reconciliation continued unexpectedly: hook %s not blocking", hookName)) + + // Patch the ConfigMap to set the hook response to "Success". + Byf("Setting %s response to Status:Success to unblock the reconciliation", hookName) + + configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: clusterName + "-hookresponses", Namespace: namespace}} + Eventually(func() error { + return c.Get(ctx, util.ObjectKey(configMap), configMap) + }).Should(Succeed()) + patch := client.RawPatch(types.MergePatchType, + []byte(fmt.Sprintf(`{"data":{"%s-preloadedResponse":%s}}`, hookName, "\"{\\\"Status\\\": \\\"Success\\\"}\""))) + Eventually(func() error { + return c.Patch(ctx, configMap, patch) + }).Should(Succeed()) + + // Expect the Hook to pass, setting the blockingCondition to false before the timeout ends. + Eventually(func() bool { + return blockingCondition() + }, intervals...).Should(BeFalse(), + fmt.Sprintf("ClusterTopology reconcile did not unblock after updating hook response: hook %s still blocking", hookName)) +} + +// clusterConditionShowsHookFailed checks if the TopologyReconciled condition message contains both the hook name and hookFailedMessage. +func clusterConditionShowsHookFailed(cluster *clusterv1.Cluster, hookName string) bool { + return conditions.GetReason(cluster, clusterv1.TopologyReconciledCondition) == clusterv1.TopologyReconcileFailedReason && + strings.Contains(conditions.GetMessage(cluster, clusterv1.TopologyReconciledCondition), hookFailedMessage) && + strings.Contains(conditions.GetMessage(cluster, clusterv1.TopologyReconciledCondition), hookName) +} diff --git a/test/extension/handlers/lifecycle/handlers.go b/test/extension/handlers/lifecycle/handlers.go index 95f17318cdda..2a1f759bc6e1 100644 --- a/test/extension/handlers/lifecycle/handlers.go +++ b/test/extension/handlers/lifecycle/handlers.go @@ -42,18 +42,15 @@ func (h *Handler) DoBeforeClusterCreate(ctx context.Context, request *runtimehoo log := ctrl.LoggerFrom(ctx) log.Info("BeforeClusterCreate is called") cluster := request.Cluster - if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate); err != nil { + + if err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - log.Info("BeforeClusterCreate has been recorded in configmap", "cm", cluster.Name+"-hookresponses") - - err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate, response) - if err != nil { + if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterCreate, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() - return } } @@ -62,16 +59,16 @@ func (h *Handler) DoBeforeClusterUpgrade(ctx context.Context, request *runtimeho log := ctrl.LoggerFrom(ctx) log.Info("BeforeClusterUpgrade is called") cluster := request.Cluster - if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade); err != nil { + + if err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade, response) - if err != nil { + + if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.BeforeClusterUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() - return } } @@ -80,16 +77,16 @@ func (h *Handler) DoAfterControlPlaneInitialized(ctx context.Context, request *r log := ctrl.LoggerFrom(ctx) log.Info("AfterControlPlaneInitialized is called") cluster := request.Cluster - if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized); err != nil { + + if err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized, response) - if err != nil { + + if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneInitialized, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() - return } } @@ -98,16 +95,16 @@ func (h *Handler) DoAfterControlPlaneUpgrade(ctx context.Context, request *runti log := ctrl.LoggerFrom(ctx) log.Info("AfterControlPlaneUpgrade is called") cluster := request.Cluster - if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade); err != nil { + + if err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade, response) - if err != nil { + + if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterControlPlaneUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() - return } } @@ -116,16 +113,16 @@ func (h *Handler) DoAfterClusterUpgrade(ctx context.Context, request *runtimehoo log := ctrl.LoggerFrom(ctx) log.Info("AfterClusterUpgrade is called") cluster := request.Cluster - if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade); err != nil { + + if err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - err := h.readResponseFromConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade, response) - if err != nil { + + if err := h.recordCallInConfigMap(ctx, cluster.Name, cluster.Namespace, runtimehooksv1.AfterClusterUpgrade, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() - return } } @@ -136,13 +133,13 @@ func (h *Handler) readResponseFromConfigMap(ctx context.Context, name, namespace if err := h.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap); err != nil { return errors.Wrapf(err, "failed to read the ConfigMap %s/%s", namespace, configMapName) } - if err := yaml.Unmarshal([]byte(configMap.Data[hookName+"-response"]), response); err != nil { + if err := yaml.Unmarshal([]byte(configMap.Data[hookName+"-preloadedResponse"]), response); err != nil { return errors.Wrapf(err, "failed to read %q response information from ConfigMap", hook) } return nil } -func (h *Handler) recordCallInConfigMap(ctx context.Context, name, namespace string, hook runtimecatalog.Hook) error { +func (h *Handler) recordCallInConfigMap(ctx context.Context, name, namespace string, hook runtimecatalog.Hook, response runtimehooksv1.ResponseObject) error { hookName := runtimecatalog.HookName(hook) configMap := &corev1.ConfigMap{} configMapName := name + "-hookresponses" @@ -150,8 +147,9 @@ func (h *Handler) recordCallInConfigMap(ctx context.Context, name, namespace str return errors.Wrapf(err, "failed to read the ConfigMap %s/%s", namespace, configMapName) } + // Patch the actualResponseStatus with the returned value patch := client.RawPatch(types.MergePatchType, - []byte(fmt.Sprintf(`{"data":{"%s-called":"true"}}`, hookName))) + []byte(fmt.Sprintf(`{"data":{"%s-actualResponseStatus":"%s"}}`, hookName, response.GetStatus()))) //nolint:gocritic if err := h.Client.Patch(ctx, configMap, patch); err != nil { return errors.Wrapf(err, "failed to update the ConfigMap %s/%s", namespace, configMapName) } diff --git a/test/framework/cluster_topology_helpers.go b/test/framework/cluster_topology_helpers.go index 0ff92aaa0a17..f1528fc418d1 100644 --- a/test/framework/cluster_topology_helpers.go +++ b/test/framework/cluster_topology_helpers.go @@ -58,17 +58,19 @@ func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput // UpgradeClusterTopologyAndWaitForUpgradeInput is the input type for UpgradeClusterTopologyAndWaitForUpgrade. type UpgradeClusterTopologyAndWaitForUpgradeInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - ControlPlane *controlplanev1.KubeadmControlPlane - EtcdImageTag string - DNSImageTag string - MachineDeployments []*clusterv1.MachineDeployment - KubernetesUpgradeVersion string - WaitForMachinesToBeUpgraded []interface{} - WaitForKubeProxyUpgrade []interface{} - WaitForDNSUpgrade []interface{} - WaitForEtcdUpgrade []interface{} + ClusterProxy ClusterProxy + Cluster *clusterv1.Cluster + ControlPlane *controlplanev1.KubeadmControlPlane + EtcdImageTag string + DNSImageTag string + MachineDeployments []*clusterv1.MachineDeployment + KubernetesUpgradeVersion string + WaitForMachinesToBeUpgraded []interface{} + WaitForKubeProxyUpgrade []interface{} + WaitForDNSUpgrade []interface{} + WaitForEtcdUpgrade []interface{} + PreWaitForControlPlaneToBeUpgraded func() + PreWaitForMachineDeploymentToBeUpgraded func() } // UpgradeClusterTopologyAndWaitForUpgrade upgrades a Cluster topology and waits for it to be upgraded. @@ -102,6 +104,14 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC return patchHelper.Patch(ctx, input.Cluster) }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed()) + // Once we have patched the Kubernetes Cluster we can run PreWaitForControlPlaneToBeUpgraded. + // Note: This can e.g. be used to verify the BeforeClusterUpgrade lifecycle hook is executed + // and blocking correctly. + if input.PreWaitForControlPlaneToBeUpgraded != nil { + log.Logf("Calling PreWaitForControlPlaneToBeUpgraded") + input.PreWaitForControlPlaneToBeUpgraded() + } + log.Logf("Waiting for control-plane machines to have the upgraded Kubernetes version") WaitForControlPlaneMachinesToBeUpgraded(ctx, WaitForControlPlaneMachinesToBeUpgradedInput{ Lister: mgmtClient, @@ -133,6 +143,14 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC Condition: EtcdImageTagCondition(input.EtcdImageTag, int(*input.ControlPlane.Spec.Replicas)), }, input.WaitForEtcdUpgrade...) + // Once the ControlPlane is upgraded we can run PreWaitForMachineDeploymentToBeUpgraded. + // Note: This can e.g. be used to verify the AfterControlPlaneUpgrade lifecycle hook is executed + // and blocking correctly. + if input.PreWaitForMachineDeploymentToBeUpgraded != nil { + log.Logf("Calling PreWaitForMachineDeploymentToBeUpgraded") + input.PreWaitForMachineDeploymentToBeUpgraded() + } + for _, deployment := range input.MachineDeployments { if *deployment.Spec.Replicas > 0 { log.Logf("Waiting for Kubernetes versions of machines in MachineDeployment %s/%s to be upgraded to %s", diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index fb32dfe03875..38d298d5e174 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -184,10 +184,11 @@ type ApplyClusterTemplateAndWaitInput struct { WaitForMachineDeployments []interface{} WaitForMachinePools []interface{} Args []string // extra args to be used during `kubectl apply` + PreWaitForCluster func() ControlPlaneWaiters } -// Waiter is a function that runs and waits for a long running operation to finish and updates the result. +// Waiter is a function that runs and waits for a long-running operation to finish and updates the result. type Waiter func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) // ControlPlaneWaiters are Waiter functions for the control plane. @@ -272,6 +273,14 @@ func ApplyClusterTemplateAndWait(ctx context.Context, input ApplyClusterTemplate log.Logf("Applying the cluster template yaml to the cluster") Expect(input.ClusterProxy.Apply(ctx, workloadClusterTemplate, input.Args...)).To(Succeed()) + // Once we applied the cluster template we can run PreWaitForCluster. + // Note: This can e.g. be used to verify the BeforeClusterCreate lifecycle hook is executed + // and blocking correctly. + if input.PreWaitForCluster != nil { + log.Logf("Calling PreWaitForCluster") + input.PreWaitForCluster() + } + log.Logf("Waiting for the cluster infrastructure to be provisioned") result.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.ClusterProxy.GetClient(),