add KCP rollout support based on machine certificate expiry

A new annotation is added to KubeadmConfig that captures the
certificate expiry information. This information is propagated to
machines. This new information can be used by KCP to perform a rollout
if the machine certificates are about to expire.

Note: The expiry time captured in KubeadmConfig is an approximate time
of when the certificates will expire but it is guaranteed to be before
the actual certificate expiry.

Author: Yuvaraj Kakaraparthi

api/v1alpha3/conversion.go

@@ -99,6 +99,7 @@ func (src *Machine) ConvertTo(dstRaw conversion.Hub) error {
 
 	dst.Spec.NodeDeletionTimeout = restored.Spec.NodeDeletionTimeout
 	dst.Status.NodeInfo = restored.Status.NodeInfo
+	dst.Status.CertificatesExpiryDate = restored.Status.CertificatesExpiryDate
 	return nil
 }
 

api/v1alpha3/zz_generated.conversion.go

@@ -159,6 +159,7 @@ func (src *Machine) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	dst.Spec.NodeDeletionTimeout = restored.Spec.NodeDeletionTimeout
+	dst.Status.CertificatesExpiryDate = restored.Status.CertificatesExpiryDate
 	return nil
 }
 
@@ -333,3 +334,8 @@ func Convert_v1beta1_ControlPlaneTopology_To_v1alpha4_ControlPlaneTopology(in *c
 	// controlPlaneTopology.nodeDrainTimeout has been added with v1beta1.
 	return autoConvert_v1beta1_ControlPlaneTopology_To_v1alpha4_ControlPlaneTopology(in, out, s)
 }
+
+func Convert_v1beta1_MachineStatus_To_v1alpha4_MachineStatus(in *clusterv1.MachineStatus, out *MachineStatus, s apiconversion.Scope) error {
+	// MachineStatus.CertificatesExpiryDate has been added in v1beta1.
+	return autoConvert_v1beta1_MachineStatus_To_v1alpha4_MachineStatus(in, out, s)
+}

api/v1alpha4/conversion.go

@@ -50,6 +50,12 @@ const (
 	// to pause reconciliation of deletion. These hooks will prevent removal of
 	// an instance from an infrastructure provider until all are removed.
 	PreTerminateDeleteHookAnnotationPrefix = "pre-terminate.delete.hook.machine.cluster.x-k8s.io"
+
+	// MachineCertificatesExpiryDateAnnotation annotation specifies the expiry date of the machine certificates in RFC3339 format.
+	// This annotation can be used on control plane machines to trigger rollout before certificates expire.
+	// This annotation can be set on BootstrapConfig or Machine objects. The value set on the Machine object takes precedence.
+	// This annotation can only be used on Control Plane Machines.
+	MachineCertificatesExpiryDateAnnotation = "machine.cluster.x-k8s.io/certificates-expiry"
 )
 
 // ANCHOR: MachineSpec
@@ -171,6 +177,11 @@ type MachineStatus struct {
 	// +optional
 	Phase string `json:"phase,omitempty"`
 
+	// CertificatesExpiryDate is the expiry date of the machine certificates.
+	// This value is only set for control plane machines.
+	// +optional
+	CertificatesExpiryDate *metav1.Time `json:"certificatesExpiryDate,omitempty"`
+
 	// BootstrapReady is the state of the bootstrap provider.
 	// +optional
 	BootstrapReady bool `json:"bootstrapReady"`

api/v1alpha4/zz_generated.conversion.go

@@ -65,8 +65,15 @@ const (
 const (
 	// DefaultTokenTTL is the default TTL used for tokens.
 	DefaultTokenTTL = 15 * time.Minute
+
+	// This hard-coded duration matches the hard-coded value used by kubeadm certificate generation.
+	certificateExpiryDuration = 365 * 24 * time.Hour
 )
 
+// now returns the current time.
+// This is defined as a variable so that it can be overridden in unit tests.
+var now = time.Now
+
 // InitLocker is a lock that is used around kubeadm init.
 type InitLocker interface {
 	Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool
@@ -439,6 +446,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		return ctrl.Result{}, err
 	}
+
 	conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition)
 
 	verbosityFlag := ""
@@ -495,6 +503,10 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 		return ctrl.Result{}, err
 	}
 
+	// Update the certificate expiration time in the config.
+	// This annotation will be used by KCP to trigger control plane machines rollout before the certificate generated on the machine are going to expire.
+	r.addCertificateExpiryAnnotation(scope.Config)
+
 	return ctrl.Result{}, nil
 }
 
@@ -620,6 +632,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
 		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesCorruptedReason, clusterv1.ConditionSeverityError, err.Error())
 		return ctrl.Result{}, err
 	}
+
 	conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition)
 
 	// Ensure that joinConfiguration.Discovery is properly set for joining node on the current cluster.
@@ -695,6 +708,9 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
 		return ctrl.Result{}, err
 	}
 
+	// Update the certificate expiration time in the config.
+	r.addCertificateExpiryAnnotation(scope.Config)
+
 	return ctrl.Result{}, nil
 }
 
@@ -1012,3 +1028,19 @@ func (r *KubeadmConfigReconciler) storeBootstrapData(ctx context.Context, scope
 	conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition)
 	return nil
 }
+
+// addCertificateExpiryAnnotation sets the certificate expiration time as an
+// annotation on KubeadmConfig, if it doesn't exist already.
+// NOTE: the certificate expiry date stored in the annotation will be slightly different from the one
+// actually used in the certificates - that depends on the exact time kubeadm runs on the machine-,
+// but this approximation is acceptable given that it happens before the actual expiration date.
+func (r *KubeadmConfigReconciler) addCertificateExpiryAnnotation(config *bootstrapv1.KubeadmConfig) {
+	annotations := config.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	if _, ok := annotations[clusterv1.MachineCertificatesExpiryDateAnnotation]; !ok {
+		annotations[clusterv1.MachineCertificatesExpiryDateAnnotation] = now().Add(certificateExpiryDuration).Format(time.RFC3339)
+		config.SetAnnotations(annotations)
+	}
+}

api/v1beta1/machine_types.go

@@ -2069,6 +2069,48 @@ func TestKubeadmConfigReconciler_ResolveUsers(t *testing.T) {
 	}
 }
 
+func TestKubeadmConfigReconciler_ReconcileCertificateExpiryTime(t *testing.T) {
+	fakeNow, _ := time.Parse(time.RFC3339, "2022-01-01T00:00:00Z")
+	now = func() time.Time {
+		return fakeNow
+	}
+	oneYearFromNow := "2023-01-01T00:00:00Z"
+	time2 := "2023-10-01T00:00:00Z"
+
+	tests := []struct {
+		name     string
+		cfg      *bootstrapv1.KubeadmConfig
+		wantTime string
+	}{
+		{
+			name:     "set the expiry time to one year from now if the expiry time is not set",
+			cfg:      &bootstrapv1.KubeadmConfig{},
+			wantTime: oneYearFromNow,
+		},
+		{
+			name: "do not change the expiry time if it is already set",
+			cfg: &bootstrapv1.KubeadmConfig{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						clusterv1.MachineCertificatesExpiryDateAnnotation: time2,
+					},
+				},
+			},
+			wantTime: time2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			k := &KubeadmConfigReconciler{}
+			k.addCertificateExpiryAnnotation(tt.cfg)
+			annotations := tt.cfg.GetAnnotations()
+			g.Expect(annotations[clusterv1.MachineCertificatesExpiryDateAnnotation]).To(Equal(tt.wantTime))
+		})
+	}
+}
+
 // test utils.
 
 // newWorkerMachineForCluster returns a Machine with the passed Cluster's information and a pre-configured name.

api/v1beta1/zz_generated.deepcopy.go

@@ -82,6 +82,8 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.KubeadmConfigSpec.JoinConfiguration.SkipPhases = restored.Spec.KubeadmConfigSpec.JoinConfiguration.SkipPhases
 	}
 
+	dst.Spec.RolloutBefore = restored.Spec.RolloutBefore
+
 	return nil
 }
 

api/v1beta1/zz_generated.openapi.go

@@ -67,6 +67,7 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout
+	dst.Spec.RolloutBefore = restored.Spec.RolloutBefore
 
 	return nil
 }
@@ -140,6 +141,8 @@ func (src *KubeadmControlPlaneTemplate) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.Template.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.Template.Spec.MachineTemplate.NodeDeletionTimeout
 	}
 
+	dst.Spec.Template.Spec.RolloutBefore = restored.Spec.Template.Spec.RolloutBefore
+
 	return nil
 }
 
@@ -226,3 +229,8 @@ func Convert_v1beta1_KubeadmControlPlaneMachineTemplate_To_v1alpha4_KubeadmContr
 	// .NodeDrainTimeout was added in v1beta1.
 	return autoConvert_v1beta1_KubeadmControlPlaneMachineTemplate_To_v1alpha4_KubeadmControlPlaneMachineTemplate(in, out, s)
 }
+
+func Convert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in *controlplanev1.KubeadmControlPlaneSpec, out *KubeadmControlPlaneSpec, scope apiconversion.Scope) error {
+	// .RolloutBefore was added in v1beta1.
+	return autoConvert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in, out, scope)
+}

bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go

@@ -70,10 +70,14 @@ type KubeadmControlPlaneSpec struct {
 	// to use for initializing and joining machines to the control plane.
 	KubeadmConfigSpec bootstrapv1.KubeadmConfigSpec `json:"kubeadmConfigSpec"`
 
+	// RolloutBefore is a field to indicate a rollout should be performed
+	// if the specified criteria is met.
+	// +optional
+	RolloutBefore *RolloutBefore `json:"rolloutBefore,omitempty"`
+
 	// RolloutAfter is a field to indicate a rollout should be performed
 	// after the specified time even if no changes have been made to the
 	// KubeadmControlPlane.
-	//
 	// +optional
 	RolloutAfter *metav1.Time `json:"rolloutAfter,omitempty"`
 
@@ -109,6 +113,14 @@ type KubeadmControlPlaneMachineTemplate struct {
 	NodeDeletionTimeout *metav1.Duration `json:"nodeDeletionTimeout,omitempty"`
 }
 
+// RolloutBefore describes when a rollout should be performed on the KCP machines.
+type RolloutBefore struct {
+	// CertificatesExpiryDays indicates a rollout needs to be performed if the
+	// certificates of the machine will expire within the specified days.
+	// +optional
+	CertificatesExpiryDays *int32 `json:"certificatesExpiryDays,omitempty"`
+}
+
 // RolloutStrategy describes how to replace existing machines
 // with new ones.
 type RolloutStrategy struct {