add more conditions

Author: fabriziopandini

controlplane/kubeadm/api/v1alpha3/condition_consts.go

@@ -28,7 +28,6 @@ const (
 const (
 	// CertificatesAvailableCondition documents that cluster certificates were generated as part of the
 	// processing of a a KubeadmControlPlane object.
-	// IMPORTANT: This condition won't be re-created after clusterctl move.
 	CertificatesAvailableCondition clusterv1.ConditionType = "CertificatesAvailable"
 
 	// CertificatesGenerationFailedReason (Severity=Warning) documents a KubeadmControlPlane controller detecting
@@ -42,7 +41,28 @@ const (
 	// and so the control plane is available and an API server instance is ready for processing requests.
 	AvailableCondition clusterv1.ConditionType = "Available"
 
-	// WaitingForKubeadmInitReason (Severity=Info) a KubeadmControlPlane object waiting for the first
+	// WaitingForKubeadmInitReason (Severity=Info) documents a KubeadmControlPlane object waiting for the first
 	// control plane instance to complete the kubeadm init operation.
 	WaitingForKubeadmInitReason = "WaitingForKubeadmInit"
 )
+
+const (
+	// MachinesSpecUpToDateCondition documents that the spec of the machines controlled by the KubeadmControlPlane
+	// is up to date. Whe this condition is false, the KubeadmControlPlane is executing a rolling upgrade.
+	MachinesSpecUpToDateCondition clusterv1.ConditionType = "MachinesSpecUpToDate"
+
+	// RollingUpdateInProgressReason (Severity=Warning) documents a KubeadmControlPlane object executing a
+	// rolling upgrade for aligning the machines spec to the desired state.
+	RollingUpdateInProgressReason = "RollingUpdateInProgress"
+)
+
+const (
+	// ResizedCondition documents a KubeadmControlPlane that is resizing the set of controlled machines.
+	ResizedCondition clusterv1.ConditionType = "Resized"
+
+	// ScalingUpReason (Severity=Info) documents a KubeadmControlPlane that is increasing the number of replicas.
+	ScalingUpReason = "ScalingUp"
+
+	// ScalingDownReason (Severity=Info) documents a KubeadmControlPlane that is decreasing the number of replicas.
+	ScalingDownReason = "ScalingDown"
+)

controlplane/kubeadm/controllers/controller.go

@@ -162,15 +162,6 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(req ctrl.Request) (res ctrl.Re
 			}
 		}
 
-		// Always update the readyCondition.
-		conditions.SetSummary(kcp,
-			conditions.WithConditions(
-				controlplanev1.MachinesReadyCondition,
-				controlplanev1.AvailableCondition,
-				controlplanev1.CertificatesAvailableCondition,
-			),
-		)
-
 		// Always attempt to update status.
 		if err := r.updateStatus(ctx, kcp, cluster); err != nil {
 			var connFailure *internal.RemoteClusterConnectionError
@@ -182,6 +173,17 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(req ctrl.Request) (res ctrl.Re
 			}
 		}
 
+		// Always update the readyCondition.
+		conditions.SetSummary(kcp,
+			conditions.WithConditions(
+				controlplanev1.MachinesSpecUpToDateCondition,
+				controlplanev1.ResizedCondition,
+				controlplanev1.MachinesReadyCondition,
+				controlplanev1.AvailableCondition,
+				controlplanev1.CertificatesAvailableCondition,
+			),
+		)
+
 		// Always attempt to Patch the KubeadmControlPlane object and status after each reconciliation.
 		if err := patchHelper.Patch(ctx, kcp); err != nil {
 			logger.Error(err, "Failed to patch KubeadmControlPlane")
@@ -269,14 +271,25 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	controlPlane := internal.NewControlPlane(cluster, kcp, ownedMachines)
 
 	// Aggregate the operational state of all the machines; while aggregating we are adding the
-	// source ref to the aggregate reason (reason@machine/name) so the problem can be easily tracked down to its source.
+	// source ref (reason@machine/name) so the problem can be easily tracked down to its source machine.
 	conditions.SetAggregate(controlPlane.KCP, controlplanev1.MachinesReadyCondition, ownedMachines.ConditionGetters(), conditions.AddSourceRef())
 
-	requireUpgrade := controlPlane.MachinesNeedingUpgrade()
-	// Upgrade takes precedence over other operations
-	if len(requireUpgrade) > 0 {
-		logger.Info("Upgrading Control Plane")
+	// Control plane machines rollout due to configuration changes (e.g. upgrades) takes precedence over other operations.
+	needRollout := controlPlane.MachinesNeedingRollout()
+	switch {
+	case len(needRollout) > 0:
+		logger.Info("Rolling out Control Plane machines")
+		// NOTE: we are using Status.UpdatedReplicas from the previous reconciliation only to provide a meaningful message
+		// and this does not influence any reconciliation logic.
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.MachinesSpecUpToDateCondition, controlplanev1.RollingUpdateInProgressReason, clusterv1.ConditionSeverityWarning, "Rolling %d replicas with outdated spec (%d replicas up to date)", len(needRollout), kcp.Status.UpdatedReplicas)
 		return r.upgradeControlPlane(ctx, cluster, kcp, controlPlane)
+	default:
+		// make sure last upgrade operation is marked as completed.
+		// NOTE: we are checking the condition already exists in order to avoid to set this condition at the first
+		// reconciliation/before a rolling upgrade actually starts.
+		if conditions.Has(controlPlane.KCP, controlplanev1.MachinesSpecUpToDateCondition) {
+			conditions.MarkTrue(controlPlane.KCP, controlplanev1.MachinesSpecUpToDateCondition)
+		}
 	}
 
 	// If we've made it this far, we can assume that all ownedMachines are up to date

controlplane/kubeadm/controllers/scale.go

@@ -146,7 +146,7 @@ func (r *KubeadmControlPlaneReconciler) scaleDownControlPlane(
 
 func selectMachineForScaleDown(controlPlane *internal.ControlPlane) (*clusterv1.Machine, error) {
 	machines := controlPlane.Machines
-	if needingUpgrade := controlPlane.MachinesNeedingUpgrade(); needingUpgrade.Len() > 0 {
+	if needingUpgrade := controlPlane.MachinesNeedingRollout(); needingUpgrade.Len() > 0 {
 		machines = needingUpgrade
 	}
 	return controlPlane.MachineInFailureDomainWithMostMachines(machines)

controlplane/kubeadm/controllers/status.go

@@ -45,12 +45,30 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c
 	kcp.Status.UpdatedReplicas = int32(len(currentMachines))
 
 	replicas := int32(len(ownedMachines))
+	desiredReplicas := *kcp.Spec.Replicas
 
 	// set basic data that does not require interacting with the workload cluster
 	kcp.Status.Replicas = replicas
 	kcp.Status.ReadyReplicas = 0
 	kcp.Status.UnavailableReplicas = replicas
 
+	switch {
+	// We are scaling up
+	case replicas < desiredReplicas:
+		conditions.MarkFalse(kcp, controlplanev1.ResizedCondition, controlplanev1.ScalingUpReason, clusterv1.ConditionSeverityWarning, "Scaling up to %d replicas (actual %d)", desiredReplicas, replicas)
+	// We are scaling down
+	case replicas > desiredReplicas:
+		conditions.MarkFalse(kcp, controlplanev1.ResizedCondition, controlplanev1.ScalingDownReason, clusterv1.ConditionSeverityWarning, "Scaling down to %d replicas (actual %d)", desiredReplicas, replicas)
+	default:
+		// make sure last resize operation is marked as completed.
+		// NOTE: we are checking the number of machines ready so we report resize completed only when the machines
+		// are actually provisioned (vs reporting completed immediately after the last machine object is created).
+		readyMachines := ownedMachines.Filter(machinefilters.IsReady())
+		if int32(len(readyMachines)) == replicas {
+			conditions.MarkTrue(kcp, controlplanev1.ResizedCondition)
+		}
+	}
+
 	// Return early if the deletion timestamp is set, we don't want to try to connect to the workload cluster.
 	if !kcp.DeletionTimestamp.IsZero() {
 		return nil

controlplane/kubeadm/internal/control_plane.go

@@ -93,8 +93,11 @@ func (c *ControlPlane) EtcdImageData() (string, string) {
 	return "", ""
 }
 
-// MachinesNeedingUpgrade return a list of machines that need to be upgraded.
-func (c *ControlPlane) MachinesNeedingUpgrade() FilterableMachineCollection {
+// MachinesNeedingRollout return a list of machines that need to be rolled out due to configuration changes.
+//
+// NOTE: Expiration of the spec.UpgradeAfter value forces inclusion of all the machines in this set even if
+// no changes have been made to the KubeadmControlPlane.
+func (c *ControlPlane) MachinesNeedingRollout() FilterableMachineCollection {
 	now := metav1.Now()
 	if c.KCP.Spec.UpgradeAfter != nil && c.KCP.Spec.UpgradeAfter.Before(&now) {
 		return c.Machines.AnyFilter(

controlplane/kubeadm/internal/machinefilters/machine_filters.go

@@ -20,6 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
+	"sigs.k8s.io/cluster-api/util/conditions"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
@@ -148,6 +149,16 @@ func MatchesConfigurationHash(configHash string) Func {
 	}
 }
 
+// IsReady returns a filter to find all machines with the ReadyCondition equals to True.
+func IsReady() Func {
+	return func(machine *clusterv1.Machine) bool {
+		if machine == nil {
+			return false
+		}
+		return conditions.IsTrue(machine, clusterv1.ReadyCondition)
+	}
+}
+
 // OlderThan returns a filter to find all machines
 // that have a CreationTimestamp earlier than the given time.
 func OlderThan(t *metav1.Time) Func {