add conditions to kcp

Author: fabriziopandini

controlplane/kubeadm/api/v1alpha3/condition_consts.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha3
+
+import clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+
+// Conditions and condition Reasons for the KubeadmControlPlane object
+
+const (
+	// MachinesReady reports an aggregate of current status of the machines controlled by the KubeadmControlPlane.
+	MachinesReadyCondition clusterv1.ConditionType = "MachinesReady"
+)
+
+const (
+	// CertificatesAvailableCondition documents that cluster certificates were generated as part of the
+	// processing of a a KubeadmControlPlane object.
+	// IMPORTANT: This condition won't be re-created after clusterctl move.
+	CertificatesAvailableCondition clusterv1.ConditionType = "CertificatesAvailable"
+
+	// CertificatesGenerationFailedReason (Severity=Warning) documents a KubeadmControlPlane controller detecting
+	// an error while generating certificates; those kind of errors are usually temporary and the controller
+	// automatically recover from them.
+	CertificatesGenerationFailedReason = "CertificatesGenerationFailed"
+)
+
+const (
+	// AvailableCondition documents that the first control plane instance has completed the kubeadm init operation
+	// and so the control plane is available and an API server instance is ready for processing requests.
+	AvailableCondition clusterv1.ConditionType = "Available"
+
+	// WaitingForKubeadmInitReason (Severity=Info) a KubeadmControlPlane object waiting for the first
+	// control plane instance to complete the kubeadm init operation.
+	WaitingForKubeadmInitReason = "WaitingForKubeadmInit"
+)

controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go

@@ -19,6 +19,7 @@ package v1alpha3
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 
 	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/errors"
@@ -114,6 +115,10 @@ type KubeadmControlPlaneStatus struct {
 	// ObservedGeneration is the latest generation observed by the controller.
 	// +optional
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Conditions defines current service state of the KubeadmControlPlane.
+	// +optional
+	Conditions clusterv1.Conditions `json:"conditions,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -137,6 +142,14 @@ type KubeadmControlPlane struct {
 	Status KubeadmControlPlaneStatus `json:"status,omitempty"`
 }
 
+func (in *KubeadmControlPlane) GetConditions() clusterv1.Conditions {
+	return in.Status.Conditions
+}
+
+func (in *KubeadmControlPlane) SetConditions(conditions clusterv1.Conditions) {
+	in.Status.Conditions = conditions
+}
+
 // +kubebuilder:object:root=true
 
 // KubeadmControlPlaneList contains a list of KubeadmControlPlane.

controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go

@@ -1045,6 +1045,50 @@ spec:
           status:
             description: KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane.
             properties:
+              conditions:
+                description: Conditions defines current service state of the KubeadmControlPlane.
+                items:
+                  description: Condition defines an observation of a Cluster API resource
+                    operational state.
+                  properties:
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status
+                        to another. This should be when the underlying condition changed.
+                        If that is not known, then using the time when the API field
+                        changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: A human readable message indicating details about
+                        the transition. This field may be empty.
+                      type: string
+                    reason:
+                      description: The reason for the condition's last transition
+                        in CamelCase. The specific API may choose whether or not this
+                        field is considered a guaranteed API. This field may not be
+                        empty.
+                      type: string
+                    severity:
+                      description: Severity provides an explicit classification of
+                        Reason code, so the users or machines can immediately understand
+                        the current situation and act accordingly. The Severity field
+                        MUST be set only when Status=False.
+                      type: string
+                    status:
+                      description: Status of the condition, one of True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type of condition in CamelCase or in foo.example.com/CamelCase.
+                        Many .condition.type values are consistent across resources
+                        like Available, but because arbitrary conditions can be useful
+                        (see .node.status.conditions), the ability to deconflict is
+                        important.
+                      type: string
+                  required:
+                  - status
+                  - type
+                  type: object
+                type: array
               failureMessage:
                 description: ErrorMessage indicates that there is a terminal problem
                   reconciling the state, and will be set to a descriptive error message.

controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml

@@ -49,6 +49,7 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
@@ -161,6 +162,15 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(req ctrl.Request) (res ctrl.Re
 			}
 		}
 
+		// Always update the readyCondition.
+		conditions.SetSummary(kcp,
+			conditions.WithConditions(
+				controlplanev1.MachinesReadyCondition,
+				controlplanev1.AvailableCondition,
+				controlplanev1.CertificatesAvailableCondition,
+			),
+		)
+
 		// Always attempt to update status.
 		if err := r.updateStatus(ctx, kcp, cluster); err != nil {
 			var connFailure *internal.RemoteClusterConnectionError
@@ -220,8 +230,10 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane"))
 	if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil {
 		logger.Error(err, "unable to lookup or create cluster certificates")
+		conditions.MarkFalse(kcp, controlplanev1.CertificatesAvailableCondition, controlplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		return ctrl.Result{}, err
 	}
+	conditions.MarkTrue(kcp, controlplanev1.CertificatesAvailableCondition)
 
 	// If ControlPlaneEndpoint is not set, return early
 	if cluster.Spec.ControlPlaneEndpoint.IsZero() {
@@ -255,6 +267,11 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	}
 
 	controlPlane := internal.NewControlPlane(cluster, kcp, ownedMachines)
+
+	// Aggregate the operational state of all the machines; while aggregating we are adding the
+	// source ref to the aggregate reason (reason@machine/name) so the problem can be easily tracked down to its source.
+	conditions.SetAggregate(controlPlane.KCP, controlplanev1.MachinesReadyCondition, ownedMachines.ConditionGetters(), conditions.AddSourceRef())
+
 	requireUpgrade := controlPlane.MachinesNeedingUpgrade()
 	// Upgrade takes precedence over other operations
 	if len(requireUpgrade) > 0 {
@@ -271,6 +288,7 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 	case numMachines < desiredReplicas && numMachines == 0:
 		// Create new Machine w/ init
 		logger.Info("Initializing control plane", "Desired", desiredReplicas, "Existing", numMachines)
+		conditions.MarkFalse(controlPlane.KCP, controlplanev1.AvailableCondition, controlplanev1.WaitingForKubeadmInitReason, clusterv1.ConditionSeverityInfo, "")
 		return r.initializeControlPlane(ctx, cluster, kcp, controlPlane)
 	// We are scaling up
 	case numMachines < desiredReplicas && numMachines > 0:

controlplane/kubeadm/controllers/controller.go

@@ -43,12 +43,13 @@ import (
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/hash"
 	capierrors "sigs.k8s.io/cluster-api/errors"
+	"sigs.k8s.io/cluster-api/test/helpers"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -756,6 +757,7 @@ kubernetesVersion: metav1.16.1`,
 
 	g.Expect(kcp.Status.Selector).NotTo(BeEmpty())
 	g.Expect(kcp.Status.Replicas).To(BeEquivalentTo(1))
+	g.Expect(conditions.IsFalse(kcp, controlplanev1.AvailableCondition)).To(BeTrue())
 
 	s, err := secret.GetFromNamespacedName(context.Background(), fakeClient, client.ObjectKey{Namespace: "test", Name: "foo"}, secret.ClusterCA)
 	g.Expect(err).NotTo(HaveOccurred())
@@ -1180,7 +1182,7 @@ func newFakeClient(g *WithT, initObjs ...runtime.Object) client.Client {
 	g.Expect(controlplanev1.AddToScheme(scheme.Scheme)).To(Succeed())
 	return &fakeClient{
 		startTime: time.Now(),
-		Client:    fake.NewFakeClientWithScheme(scheme.Scheme, initObjs...),
+		Client:    helpers.NewFakeClientWithScheme(scheme.Scheme, initObjs...),
 	}
 }
 

controlplane/kubeadm/controllers/controller_test.go

@@ -184,6 +184,7 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
 			endMachines := internal.NewFilterableMachineCollectionFromMachineList(controlPlaneMachines)
 			for _, m := range endMachines {
 				bm, ok := beforeMachines[m.Name]
+				bm.SetResourceVersion("1")
 				g.Expect(ok).To(BeTrue())
 				g.Expect(m).To(Equal(bm))
 			}

controlplane/kubeadm/controllers/scale_test.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/hash"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/conditions"
 )
 
 // updateStatus is called after every reconcilitation loop in a defer statement to always make sure we have the
@@ -69,6 +70,7 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c
 	// This only gets initialized once and does not change if the kubeadm config map goes away.
 	if status.HasKubeadmConfig {
 		kcp.Status.Initialized = true
+		conditions.MarkTrue(kcp, controlplanev1.AvailableCondition)
 	}
 
 	if kcp.Status.ReadyReplicas > 0 {

controlplane/kubeadm/controllers/status.go

@@ -32,6 +32,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
+	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
@@ -197,6 +198,7 @@ func TestKubeadmControlPlaneReconciler_updateStatusAllMachinesReady(t *testing.T
 	g.Expect(kcp.Status.FailureMessage).To(BeNil())
 	g.Expect(kcp.Status.FailureReason).To(BeEquivalentTo(""))
 	g.Expect(kcp.Status.Initialized).To(BeTrue())
+	g.Expect(conditions.IsTrue(kcp, controlplanev1.AvailableCondition)).To(BeTrue())
 	g.Expect(kcp.Status.Ready).To(BeTrue())
 }
 

controlplane/kubeadm/controllers/status_test.go

@@ -33,6 +33,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/machinefilters"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/conditions"
 )
 
 // FilterableMachineCollection is a set of Machines
@@ -134,3 +135,13 @@ func (s FilterableMachineCollection) DeepCopy() FilterableMachineCollection {
 	}
 	return result
 }
+
+// ConditionGetters returns the slice with machines converted into conditions.Getter.
+func (s FilterableMachineCollection) ConditionGetters() []conditions.Getter {
+	res := make([]conditions.Getter, 0, len(s))
+	for _, v := range s {
+		value := *v
+		res = append(res, &value)
+	}
+	return res
+}