Add v1beta2 available condition to KCP

Author: fabriziopandini

controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go

@@ -20,9 +20,20 @@ import clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 // KubeadmControlPlane's Available condition and corresponding reasons that will be used in v1Beta2 API version.
 const (
-	// KubeadmControlPlaneAvailableV1Beta2Condition is True if the control plane can be reached, EtcdClusterHealthy is true,
-	// and CertificatesAvailable is true.
+	// KubeadmControlPlaneAvailableV1Beta2Condition is True if KubeadmControlPlane not delete, `CertificatesAvailable` is true,
+	// at least one Kubernetes API server, scheduler and controller manager control plane are healthy,
+	// and etcd has enough operational members to meet quorum requirements.
 	KubeadmControlPlaneAvailableV1Beta2Condition = clusterv1.AvailableV1Beta2Condition
+
+	// KubeadmControlPlaneAvailableInspectionFailedV1Beta2Reason documents a failure when inspecting the status of the
+	// etcd cluster hosted on KubeadmControlPlane controlled machines.
+	KubeadmControlPlaneAvailableInspectionFailedV1Beta2Reason = clusterv1.InspectionFailedV1Beta2Reason
+
+	// KubeadmControlPlaneAvailableV1Beta2Reason surfaces when a Deployment is available.
+	KubeadmControlPlaneAvailableV1Beta2Reason = clusterv1.AvailableV1Beta2Reason
+
+	// KubeadmControlPlaneNotAvailableV1Beta2Reason surfaces when a Deployment is not available.
+	KubeadmControlPlaneNotAvailableV1Beta2Reason = clusterv1.NotAvailableV1Beta2Reason
 )
 
 // KubeadmControlPlane's Initialized condition and corresponding reasons that will be used in v1Beta2 API version.
@@ -215,12 +226,6 @@ const (
 	KubeadmControlPlaneDeletingV1Beta2Condition = clusterv1.DeletingV1Beta2Condition
 )
 
-// KubeadmControlPlane's Paused condition and corresponding reasons that will be used in v1Beta2 API version.
-const (
-	// KubeadmControlPlanePausedV1Beta2Condition is true if this resource or the Cluster it belongs to are paused.
-	KubeadmControlPlanePausedV1Beta2Condition = clusterv1.PausedV1Beta2Condition
-)
-
 // APIServerPodHealthy, ControllerManagerPodHealthy, SchedulerPodHealthy and EtcdPodHealthy condition and corresponding
 // reasons that will be used for KubeadmControlPlane controlled machines in v1Beta2 API version.
 const (

controlplane/kubeadm/internal/control_plane.go

@@ -30,6 +30,7 @@ import (
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controllers/external"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/etcd"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/failuredomains"
 	"sigs.k8s.io/cluster-api/util/patch"
@@ -58,6 +59,14 @@ type ControlPlane struct {
 	KubeadmConfigs map[string]*bootstrapv1.KubeadmConfig
 	InfraResources map[string]*unstructured.Unstructured
 
+	// EtcdMembers is the list of members read while computing reconcileControlPlaneConditions; also additional info below
+	// comes from the same func.
+	// NOTE: Those info are computed on what we know, so we can reason about availability eve if with a certain degree of problems in the cluster
+	EtcdMembers                       []*etcd.Member
+	EtcdMembersAgreeOnMemberList      bool
+	EtcdMembersAgreeOnClusterID       bool
+	EtcdMembersAndMachinesAreMatching bool
+
 	managementCluster ManagementCluster
 	workloadCluster   WorkloadCluster
 }

controlplane/kubeadm/internal/controllers/status.go

@@ -31,6 +31,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
+	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/etcd"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	v1beta2conditions "sigs.k8s.io/cluster-api/util/conditions/v1beta2"
@@ -167,8 +168,8 @@ func (r *KubeadmControlPlaneReconciler) updateV1beta2Status(ctx context.Context,
 	setMachinesReadyCondition(ctx, controlPlane.KCP, controlPlane.Machines)
 	setMachinesUpToDateCondition(ctx, controlPlane.KCP, controlPlane.Machines)
 	setRemediatingCondition(ctx, controlPlane.KCP, controlPlane.MachinesToBeRemediatedByKCP(), controlPlane.UnhealthyMachines())
-
-	// TODO: Available, Deleting
+	// TODO: Deleting
+	setAvailableCondition(ctx, controlPlane.KCP, controlPlane.IsEtcdManaged(), controlPlane.EtcdMembers, controlPlane.EtcdMembersAgreeOnMemberList, controlPlane.EtcdMembersAgreeOnClusterID, controlPlane.EtcdMembersAndMachinesAreMatching, controlPlane.Machines)
 }
 
 func setReplicas(_ context.Context, kcp *controlplanev1.KubeadmControlPlane, machines collections.Machines) {
@@ -423,6 +424,139 @@ func setRemediatingCondition(ctx context.Context, kcp *controlplanev1.KubeadmCon
 	})
 }
 
+func setAvailableCondition(_ context.Context, kcp *controlplanev1.KubeadmControlPlane, etcdIsManaged bool, etcdMembers []*etcd.Member, etcdMembersAgreeOnMemberList bool, etcdMembersAgreeOnClusterID bool, etcdMembersAndMachinesAreMatching bool, machines collections.Machines) {
+	if !kcp.Status.Initialized {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status:  metav1.ConditionFalse,
+			Reason:  controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason,
+			Message: "Control plane not yet initialized",
+		})
+		return
+	}
+
+	if etcdIsManaged && etcdMembers == nil {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status:  metav1.ConditionUnknown,
+			Reason:  controlplanev1.KubeadmControlPlaneAvailableInspectionFailedV1Beta2Reason,
+			Message: "Failed to get etcd members",
+		})
+		return
+	}
+
+	if etcdIsManaged && !etcdMembersAgreeOnMemberList {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status:  metav1.ConditionFalse,
+			Reason:  controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason,
+			Message: "At least one etcd member reports a list of etcd members different than the list reported by other members",
+		})
+		return
+	}
+
+	if etcdIsManaged && !etcdMembersAgreeOnClusterID {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status:  metav1.ConditionFalse,
+			Reason:  controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason,
+			Message: "At least one etcd member reports a cluster ID different than the cluster ID reported by other members",
+		})
+		return
+	}
+
+	if etcdIsManaged && !etcdMembersAndMachinesAreMatching {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status:  metav1.ConditionFalse,
+			Reason:  controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason,
+			Message: "The list of etcd members does not match the list of Machines and Nodes",
+		})
+		return
+	}
+
+	// Determine control plane availability looking at machines conditions, which at this stage are
+	// already surfacing status from etcd member and all control plane pods hosted on every machine.
+	// Note: we intentionally use the number of etcd members for determine the etcd quorum because
+	// etcd members could not match with machines, e.g. while provisioning a new machine.
+	etcdQuorum := (len(etcdMembers) / 2.0) + 1
+	k8sControlPlaneHealthy := 0
+	etcdMembersHealthy := 0
+	for _, machine := range machines {
+		// if external etcd, only look at the status of the K8s control plane components on this machine.
+		if !etcdIsManaged {
+			if v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition) &&
+				v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition) &&
+				v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition) {
+				k8sControlPlaneHealthy++
+			}
+			continue
+		}
+
+		// Otherwise, etcd is managed.
+		// In this case, when looking at the k8s control plane we should consider how kubeadm layouts control plane components,
+		// and more specifically:
+		// - API server on one machine only connect to the local etcd member
+		// - ControllerManager and scheduler on a machine connect to the local API server (not to the control plane endpoint)
+		// As a consequence, we consider the K8s control plane on this machine healthy only if everything is healthy.
+
+		if v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) {
+			etcdMembersHealthy++
+		}
+
+		if v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyV1Beta2Condition) &&
+			v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyV1Beta2Condition) &&
+			v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyV1Beta2Condition) &&
+			v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyV1Beta2Condition) &&
+			v1beta2conditions.IsTrue(machine, controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyV1Beta2Condition) {
+			k8sControlPlaneHealthy++
+		}
+	}
+
+	if kcp.DeletionTimestamp.IsZero() &&
+		(!etcdIsManaged || etcdMembersHealthy >= etcdQuorum) &&
+		k8sControlPlaneHealthy >= 1 &&
+		v1beta2conditions.IsTrue(kcp, controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition) {
+		v1beta2conditions.Set(kcp, metav1.Condition{
+			Type:   controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+			Status: metav1.ConditionTrue,
+			Reason: controlplanev1.KubeadmControlPlaneAvailableV1Beta2Reason,
+		})
+		return
+	}
+
+	messages := []string{}
+	if !kcp.DeletionTimestamp.IsZero() {
+		messages = append(messages, "Control plane metadata.deletionTimestamp is set")
+	}
+
+	if !v1beta2conditions.IsTrue(kcp, controlplanev1.KubeadmControlPlaneCertificatesAvailableV1Beta2Condition) {
+		messages = append(messages, "Control plane certificates are not available")
+	}
+
+	if etcdIsManaged && etcdMembersHealthy < etcdQuorum {
+		switch etcdMembersHealthy {
+		case 0:
+			messages = append(messages, fmt.Sprintf("There are no healthy etcd member, at least %d required", etcdQuorum))
+		case 1:
+			messages = append(messages, fmt.Sprintf("There is 1 healthy etcd member, at least %d required", etcdQuorum))
+		default:
+			messages = append(messages, fmt.Sprintf("There are %d healthy etcd members, at least %d required", etcdMembersHealthy, etcdQuorum))
+		}
+	}
+
+	if k8sControlPlaneHealthy < 1 {
+		messages = append(messages, "There are no healthy control plane instances, at least 1 required")
+	}
+
+	v1beta2conditions.Set(kcp, metav1.Condition{
+		Type:    controlplanev1.KubeadmControlPlaneAvailableV1Beta2Condition,
+		Status:  metav1.ConditionFalse,
+		Reason:  controlplanev1.KubeadmControlPlaneNotAvailableV1Beta2Reason,
+		Message: strings.Join(messages, ";"),
+	})
+}
+
 func aggregateStaleMachines(machines collections.Machines) string {
 	if len(machines) == 0 {
 		return ""