Skip to content

🌱 Handle finalizers early in Reconciles #11286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 14, 2024
Merged

🌱 Handle finalizers early in Reconciles #11286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 6 additions & 16 deletions controlplane/kubeadm/internal/controllers/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
"sigs.k8s.io/cluster-api/util/secret"
Expand Down Expand Up @@ -149,6 +150,11 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, kcp, controlplanev1.KubeadmControlPlaneFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Fetch the Cluster.
cluster, err := util.GetOwnerCluster(ctx, r.Client, kcp.ObjectMeta)
if err != nil {
Expand Down Expand Up @@ -176,22 +182,6 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.
return ctrl.Result{Requeue: true}, nil
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if kcp.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(kcp, controlplanev1.KubeadmControlPlaneFinalizer) {
controllerutil.AddFinalizer(kcp, controlplanev1.KubeadmControlPlaneFinalizer)

// patch and return right away instead of reusing the main defer,
// because the main defer may take too much time to get cluster status
// Patch ObservedGeneration only if the reconciliation completed successfully
patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}}
if err := patchHelper.Patch(ctx, kcp, patchOpts...); err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to add finalizer")
}

return ctrl.Result{}, nil
}

// Initialize the control plane scope; this includes also checking for orphan machines and
// adopt them if necessary.
controlPlane, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
Expand Down
3 changes: 3 additions & 0 deletions controlplane/kubeadm/internal/controllers/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,9 @@ func TestReconcileNoCluster(t *testing.T) {
Name: "foo",
},
},
Finalizers: []string{
controlplanev1.KubeadmControlPlaneFinalizer,
},
},
Spec: controlplanev1.KubeadmControlPlaneSpec{
Version: "v1.16.6",
Expand Down
13 changes: 6 additions & 7 deletions exp/addons/internal/controllers/clusterresourceset_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
resourcepredicates "sigs.k8s.io/cluster-api/exp/addons/internal/controllers/predicates"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -122,6 +123,11 @@ func (r *ClusterResourceSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, clusterResourceSet, addonsv1.ClusterResourceSetFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Initialize the patch helper.
patchHelper, err := patch.NewHelper(clusterResourceSet, r.Client)
if err != nil {
Expand Down Expand Up @@ -152,13 +158,6 @@ func (r *ClusterResourceSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, r.reconcileDelete(ctx, clusters, clusterResourceSet)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(clusterResourceSet, addonsv1.ClusterResourceSetFinalizer) {
controllerutil.AddFinalizer(clusterResourceSet, addonsv1.ClusterResourceSetFinalizer)
return ctrl.Result{}, nil
}

errs := []error{}
errClusterLockedOccurred := false
for _, cluster := range clusters {
Expand Down
15 changes: 7 additions & 8 deletions exp/internal/controllers/machinepool_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -151,9 +152,14 @@ func (r *MachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(mp.ObjectMeta.Namespace, mp.Spec.ClusterName))
log = log.WithValues("Cluster", klog.KRef(mp.Namespace, mp.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, mp, expv1.MachinePoolFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, mp.ObjectMeta.Namespace, mp.Spec.ClusterName)
if err != nil {
log.Error(err, "Failed to get Cluster for MachinePool.", "MachinePool", klog.KObj(mp), "Cluster", klog.KRef(mp.ObjectMeta.Namespace, mp.Spec.ClusterName))
Expand Down Expand Up @@ -223,13 +229,6 @@ func (r *MachinePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(mp, expv1.MachinePoolFinalizer) {
controllerutil.AddFinalizer(mp, expv1.MachinePoolFinalizer)
return ctrl.Result{}, nil
}

// Handle normal reconciliation loop.
scope := &scope{
cluster: cluster,
Expand Down
13 changes: 6 additions & 7 deletions internal/controllers/cluster/cluster_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
)
Expand Down Expand Up @@ -120,6 +121,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, cluster, clusterv1.ClusterFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// Return early if the object or Cluster is paused.
if annotations.IsPaused(cluster, cluster) {
log.Info("Reconciliation is paused for this object")
Expand Down Expand Up @@ -152,13 +158,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return r.reconcileDelete(ctx, cluster)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(cluster, clusterv1.ClusterFinalizer) {
controllerutil.AddFinalizer(cluster, clusterv1.ClusterFinalizer)
return ctrl.Result{}, nil
}

// Handle normal reconciliation loop.
return r.reconcile(ctx, cluster)
}
Expand Down
19 changes: 9 additions & 10 deletions internal/controllers/machine/machine_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/finalizers"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -173,16 +174,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KRef(m.Namespace, m.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, m, clusterv1.MachineFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// AddOwners adds the owners of Machine as k/v pairs to the logger.
// Specifically, it will add KubeadmControlPlane, MachineSet and MachineDeployment.
ctx, log, err := clog.AddOwners(ctx, r.Client, m)
if err != nil {
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(m.ObjectMeta.Namespace, m.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

cluster, err := util.GetClusterByName(ctx, r.Client, m.ObjectMeta.Namespace, m.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to get cluster %q for machine %q in namespace %q",
Expand Down Expand Up @@ -220,13 +226,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
}
}()

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(m, clusterv1.MachineFinalizer) && m.ObjectMeta.DeletionTimestamp.IsZero() {
controllerutil.AddFinalizer(m, clusterv1.MachineFinalizer)
return ctrl.Result{}, nil
}

alwaysReconcile := []machineReconcileFunc{
r.reconcileMachineOwnerAndLabels,
r.reconcileBootstrap,
Expand Down
13 changes: 6 additions & 7 deletions internal/controllers/machinedeployment/machinedeployment_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/conditions"
utilconversion "sigs.k8s.io/cluster-api/util/conversion"
"sigs.k8s.io/cluster-api/util/finalizers"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -131,6 +132,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
log = log.WithValues("Cluster", klog.KRef(deployment.Namespace, deployment.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, deployment, clusterv1.MachineDeploymentFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, deployment.Namespace, deployment.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -165,13 +171,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, deployment)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(deployment, clusterv1.MachineDeploymentFinalizer) {
controllerutil.AddFinalizer(deployment, clusterv1.MachineDeploymentFinalizer)
return ctrl.Result{}, nil
}

err = r.reconcile(ctx, cluster, deployment)
if err != nil {
r.recorder.Eventf(deployment, corev1.EventTypeWarning, "ReconcileError", "%v", err)
Expand Down
19 changes: 9 additions & 10 deletions internal/controllers/machineset/machineset_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import (
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/conditions"
utilconversion "sigs.k8s.io/cluster-api/util/conversion"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/labels/format"
clog "sigs.k8s.io/cluster-api/util/log"
"sigs.k8s.io/cluster-api/util/patch"
Expand Down Expand Up @@ -149,16 +150,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, err
}

log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KRef(machineSet.Namespace, machineSet.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, machineSet, clusterv1.MachineSetFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

// AddOwners adds the owners of MachineSet as k/v pairs to the logger.
// Specifically, it will add MachineDeployment.
ctx, log, err := clog.AddOwners(ctx, r.Client, machineSet)
if err != nil {
return ctrl.Result{}, err
}

log = log.WithValues("Cluster", klog.KRef(machineSet.ObjectMeta.Namespace, machineSet.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

cluster, err := util.GetClusterByName(ctx, r.Client, machineSet.ObjectMeta.Namespace, machineSet.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -188,13 +194,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, machineSet)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(machineSet, clusterv1.MachineSetFinalizer) {
controllerutil.AddFinalizer(machineSet, clusterv1.MachineSetFinalizer)
return ctrl.Result{}, nil
}

result, err := r.reconcile(ctx, cluster, machineSet)
if err != nil {
// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
Expand Down
25 changes: 12 additions & 13 deletions internal/controllers/topology/machinedeployment/machinedeployment_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"sigs.k8s.io/cluster-api/internal/controllers/topology/machineset"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/finalizers"
"sigs.k8s.io/cluster-api/util/labels"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/cluster-api/util/predicates"
Expand Down Expand Up @@ -124,6 +125,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
log = log.WithValues("Cluster", klog.KRef(md.Namespace, md.Spec.ClusterName))
ctx = ctrl.LoggerInto(ctx, log)

// Return early if the MachineDeployment is not topology owned.
if !labels.IsTopologyOwned(md) {
log.Info(fmt.Sprintf("Reconciliation is skipped because the MachineDeployment does not have the %q label", clusterv1.ClusterTopologyOwnedLabel))
return ctrl.Result{}, nil
}

// Add finalizer first if not set to avoid the race condition between init and delete.
if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, md, clusterv1.MachineDeploymentTopologyFinalizer); err != nil || finalizerAdded {
return ctrl.Result{}, err
}

cluster, err := util.GetClusterByName(ctx, r.Client, md.Namespace, md.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, err
Expand All @@ -135,12 +147,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, nil
}

// Return early if the MachineDeployment is not topology owned.
if !labels.IsTopologyOwned(md) {
log.Info(fmt.Sprintf("Reconciliation is skipped because the MachineDeployment does not have the %q label", clusterv1.ClusterTopologyOwnedLabel))
return ctrl.Result{}, nil
}

// Create a patch helper to add or remove the finalizer from the MachineDeployment.
patchHelper, err := patch.NewHelper(md, r.Client)
if err != nil {
Expand All @@ -157,13 +163,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
return ctrl.Result{}, r.reconcileDelete(ctx, md)
}

// Add finalizer first if not set to avoid the race condition between init and delete.
// Note: Finalizers in general can only be added when the deletionTimestamp is not set.
if !controllerutil.ContainsFinalizer(md, clusterv1.MachineDeploymentTopologyFinalizer) {
controllerutil.AddFinalizer(md, clusterv1.MachineDeploymentTopologyFinalizer)
return ctrl.Result{}, nil
}

return ctrl.Result{}, nil
}

Expand Down
Loading
Loading