Add conditions to the KubeadmControlPlane object

Author: fabriziopandini

controlplane/kubeadm/api/v1alpha3/condition_consts.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha3
+
+import clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+
+// Conditions and condition Reasons for the KubeadmControlPlane object
+
+// kubeadmControlPlaneConditionsCount defines the total number of conditions on the KubeadmControlPlane object.
+const KubeadmControlPlaneConditionsCount = 2
+
+const (
+	// MachinesReady reports an aggregate of current status of the machines controlled by the KubeadmControlPlane.
+	MachinesReadyCondition clusterv1.ConditionType = "MachinesReady"
+)
+
+const (
+	// CertificatesGeneratedCondition documents that cluster certificates were generated as part of the
+	// processing of a a KubeadmControlPlane object.
+	CertificatesGeneratedCondition clusterv1.ConditionType = "CertificatesGenerated"
+)
+
+const (
+	// InitializedCondition documents that the control plane was initialized and the first control plane instance
+	// has completed the kubeadm init operation, and as a consequence, there is an API server instance ready for
+	// processing requests.
+	InitializedCondition clusterv1.ConditionType = "Initialized"
+)
+
+const (
+	// ControlPlaneUpgradingCondition signals that the KubeadmControlPlane successfully completed the
+	// upgrade of the controlled machines.
+	ControlPlaneUpgradingCondition clusterv1.ConditionType = "UpgradeSucceeded"
+
+	// UpgradingReason documents a KubeadmControlPlane executing an upgrade.
+	UpgradingReason = "Upgrading"
+)
+
+const (
+	// ResizeSucceededCondition signals that the KubeadmControlPlane is currently resizing the controlled machines.
+	ResizeSucceededCondition clusterv1.ConditionType = "ResizeSucceeded"
+
+	// ScalingUpReason documents a KubeadmControlPlane scaling up the controlled machines.
+	ScalingUpReason = "ScalingUp"
+
+	// ScalingDownReason documents a KubeadmControlPlane scaling down the controlled machines.
+	ScalingDownReason = "ScalingDown"
+)

controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go

@@ -19,6 +19,7 @@ package v1alpha3
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 
 	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/errors"
@@ -116,6 +117,10 @@ type KubeadmControlPlaneStatus struct {
 	// ObservedGeneration is the latest generation observed by the controller.
 	// +optional
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Conditions defines current service state of the KubeadmControlPlane.
+	// +optional
+	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -139,6 +144,14 @@ type KubeadmControlPlane struct {
 	Status KubeadmControlPlaneStatus `json:"status,omitempty"`
 }
 
+func (c *KubeadmControlPlane) GetConditions() clusterv1.Conditions {
+	return c.Status.Conditions
+}
+
+func (c *KubeadmControlPlane) SetConditions(conditions clusterv1.Conditions) {
+	c.Status.Conditions = conditions
+}
+
 // +kubebuilder:object:root=true
 
 // KubeadmControlPlaneList contains a list of KubeadmControlPlane.

controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go

@@ -1047,6 +1047,50 @@ spec:
           status:
             description: KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane.
             properties:
+              conditions:
+                description: Conditions defines current service state of the KubeadmControlPlane.
+                items:
+                  description: Condition defines an observation of a Cluster API resource
+                    operational state.
+                  properties:
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another. This should be when the underlying condition changed.
+                        If that is not known, then using the time when the API field
+                        changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: A human readable message indicating details about
+                        the transition. This field may be empty.
+                      type: string
+                    reason:
+                      description: The reason for the condition's last transition
+                        in CamelCase. The specific API may choose whether or not this
+                        field is considered a guaranteed API. This field may not be
+                        empty.
+                      type: string
+                    severity:
+                      description: Severity provides an explicit classification of
+                        Reason code, so the users or machines can immediately understand
+                        the current situation and act accordingly. The Severity field
+                        MUST be set only when Status=False.
+                      type: string
+                    status:
+                      description: Status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type of condition in CamelCase or in foo.example.com/CamelCase.
+                        Many .condition.type values are consistent across resources
+                        like Available, but because arbitrary conditions can be useful
+                        (see .node.status.conditions), the ability to deconflict is
+                        important.
+                      type: string
+                  required:
+                  - status
+                  - type
+                  type: object
+                type: array
               failureMessage:
                 description: ErrorMessage indicates that there is a terminal problem
                   reconciling the state, and will be set to a descriptive error message.

controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml

@@ -49,6 +49,7 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
@@ -161,6 +162,13 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(req ctrl.Request) (res ctrl.Re
 			}
 		}
 
+		// Always update the readyCondition; the summary is represented using the "1 of x completed" notation.
+		conditions.SetSummary(kcp, conditions.WithConditionOrder(
+			controlplanev1.MachinesReadyCondition,
+			controlplanev1.ControlPlaneUpgradingCondition,
+			controlplanev1.ResizeSucceededCondition,
+		))
+
 		// Always attempt to update status.
 		if err := r.updateStatus(ctx, kcp, cluster); err != nil {
 			var connFailure *internal.RemoteClusterConnectionError
@@ -222,6 +230,9 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 		logger.Error(err, "unable to lookup or create cluster certificates")
 		return ctrl.Result{}, err
 	}
+	if certificates.HasGenerated() {
+		conditions.MarkTrue(kcp, controlplanev1.CertificatesGeneratedCondition)
+	}
 
 	// If ControlPlaneEndpoint is not set, return early
 	if cluster.Spec.ControlPlaneEndpoint.IsZero() {
@@ -255,11 +266,23 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	}
 
 	controlPlane := internal.NewControlPlane(cluster, kcp, ownedMachines)
-	requireUpgrade := controlPlane.MachinesNeedingUpgrade()
+
+	// Aggregate the operational state of all the machines; while aggregating we are adding the
+	// source ref to the aggregate reason (reason@machine/name) so the problem can be easily tracked down to its source.
+	conditions.SetAggregateCondition(controlPlane.KCP, controlplanev1.MachinesReadyCondition, ownedMachines.ConditionGetters(), conditions.AddSourceRef())
+
 	// Upgrade takes precedence over other operations
-	if len(requireUpgrade) > 0 {
+	requireUpgrade := controlPlane.MachinesNeedingUpgrade()
+	switch {
+	case len(requireUpgrade) > 0:
 		logger.Info("Upgrading Control Plane")
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.ControlPlaneUpgradingCondition, controlplanev1.UpgradingReason, clusterv1.ConditionSeverityWarning, "Upgrading to %s version (%d new version, %d old)", kcp.Spec.Version, kcp.Status.UpdatedReplicas, len(requireUpgrade))
 		return r.upgradeControlPlane(ctx, cluster, kcp, controlPlane)
+	default:
+		// make sure last upgrade operation is marked as completed (if any)
+		if conditions.Has(controlPlane.KCP, controlplanev1.ControlPlaneUpgradingCondition) {
+			conditions.MarkTrue(controlPlane.KCP, controlplanev1.ControlPlaneUpgradingCondition)
+		}
 	}
 
 	// If we've made it this far, we can assume that all ownedMachines are up to date
@@ -271,16 +294,24 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	case numMachines < desiredReplicas && numMachines == 0:
 		// Create new Machine w/ init
 		logger.Info("Initializing control plane", "Desired", desiredReplicas, "Existing", numMachines)
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.ResizeSucceededCondition, controlplanev1.ScalingUpReason, clusterv1.ConditionSeverityWarning, "Scaling up to %d replicas (actual %d)", desiredReplicas, numMachines)
 		return r.initializeControlPlane(ctx, cluster, kcp, controlPlane)
 	// We are scaling up
 	case numMachines < desiredReplicas && numMachines > 0:
 		// Create a new Machine w/ join
 		logger.Info("Scaling up control plane", "Desired", desiredReplicas, "Existing", numMachines)
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.ResizeSucceededCondition, controlplanev1.ScalingUpReason, clusterv1.ConditionSeverityWarning, "Scaling up to %d replicas (actual %d)", desiredReplicas, numMachines)
 		return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane)
 	// We are scaling down
 	case numMachines > desiredReplicas:
 		logger.Info("Scaling down control plane", "Desired", desiredReplicas, "Existing", numMachines)
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.ResizeSucceededCondition, controlplanev1.ScalingDownReason, clusterv1.ConditionSeverityWarning, "Scaling down to %d replicas (actual %d)", desiredReplicas, numMachines)
 		return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane)
+	default:
+		// make sure last resize operation is marked as completed (if any)
+		if conditions.Has(controlPlane.KCP, controlplanev1.ResizeSucceededCondition) {
+			conditions.MarkTrue(controlPlane.KCP, controlplanev1.ResizeSucceededCondition)
+		}
 	}
 
 	// Get the workload cluster client.
@@ -328,6 +359,9 @@ func (r *KubeadmControlPlaneReconciler) reconcileDelete(ctx context.Context, clu
 		return ctrl.Result{}, nil
 	}
 
+	// TODO: consider if this makes sense, given that it seems going down from 3 to 0 immediately
+	conditions.MarkFalse(kcp, "Ready", "Deleting", clusterv1.ConditionSeverityInfo, "Deleting all replicas (actual %d)", len(ownedMachines))
+
 	// Verify that only control plane machines remain
 	if len(allMachines) != len(ownedMachines) {
 		logger.V(2).Info("Waiting for worker nodes to be deleted first")

controlplane/kubeadm/controllers/controller.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/scheme"

controlplane/kubeadm/controllers/scale_test.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/hash"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/conditions"
 )
 
 // updateStatus is called after every reconcilitation loop in a defer statement to always make sure we have the
@@ -69,6 +70,7 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c
 	// This only gets initialized once and does not change if the kubeadm config map goes away.
 	if status.HasKubeadmConfig {
 		kcp.Status.Initialized = true
+		conditions.MarkTrue(kcp, controlplanev1.InitializedCondition)
 	}
 
 	if kcp.Status.ReadyReplicas > 0 {

controlplane/kubeadm/controllers/status.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/cluster-api/util/conditions"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -197,6 +198,7 @@ func TestKubeadmControlPlaneReconciler_updateStatusAllMachinesReady(t *testing.T
 	g.Expect(kcp.Status.FailureMessage).To(BeNil())
 	g.Expect(kcp.Status.FailureReason).To(BeEquivalentTo(""))
 	g.Expect(kcp.Status.Initialized).To(BeTrue())
+	g.Expect(conditions.Has(kcp, controlplanev1.InitializedCondition)).To(BeTrue())
 	g.Expect(kcp.Status.Ready).To(BeTrue())
 }
 

controlplane/kubeadm/controllers/status_test.go

@@ -148,7 +148,7 @@ func (s UInt64Set) IsSuperset(s2 UInt64Set) bool {
 	return true
 }
 
-// Equal returns true if and only if s1 is equal (as a set) to s2.
+// Equals returns true if and only if s1 is equal (as a set) to s2.
 // Two sets are equal if their membership is identical.
 // (In practice, this means same elements, order doesn't matter)
 func (s UInt64Set) Equal(s2 UInt64Set) bool {

controlplane/kubeadm/internal/etcd/util/set.go

@@ -33,6 +33,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/conditions"
 )
 
 // FilterableMachineCollection is a set of Machines
@@ -134,3 +135,13 @@ func (s FilterableMachineCollection) DeepCopy() FilterableMachineCollection {
 	}
 	return result
 }
+
+// ConditionGetters returns the slice with machines converted into conditions.Getter.
+func (s FilterableMachineCollection) ConditionGetters() []conditions.Getter {
+	res := make([]conditions.Getter, 0, len(s))
+	for _, v := range s {
+		value := *v
+		res = append(res, &value)
+	}
+	return res
+}