add --wait-for-node-deletion flag

causes the machine controller to re-queue a machine if node deletion
fails. Without the flag reconciliation just continues and the error
is ignored.

Author: schrej

controllers/machine_controller.go

@@ -82,6 +82,8 @@ type MachineReconciler struct {
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 
+	WaitForNodeDeletion bool
+
 	controller      controller.Controller
 	recorder        record.EventRecorder
 	externalTracker external.ObjectTracker
@@ -386,9 +388,13 @@ func (r *MachineReconciler) reconcileDelete(ctx context.Context, cluster *cluste
 			return true, nil
 		})
 		if waitErr != nil {
-			log.Error(deleteNodeErr, "Timed out deleting node, moving on", "node", m.Status.NodeRef.Name)
+			log.Error(deleteNodeErr, "Timed out deleting node", "node", m.Status.NodeRef.Name)
 			conditions.MarkFalse(m, clusterv1.MachineNodeHealthyCondition, clusterv1.DeletionFailedReason, clusterv1.ConditionSeverityWarning, "")
 			r.recorder.Eventf(m, corev1.EventTypeWarning, "FailedDeleteNode", "error deleting Machine's node: %v", deleteNodeErr)
+			if r.WaitForNodeDeletion {
+				return ctrl.Result{}, deleteNodeErr
+			}
+			log.Info("Waiting for node deletion is disabled, moving on", "node", m.Status.NodeRef.Name)
 		}
 	}
 

main.go

@@ -80,6 +80,7 @@ var (
 	machinePoolConcurrency        int
 	clusterResourceSetConcurrency int
 	machineHealthCheckConcurrency int
+	waitForNodeDeletion           bool
 	syncPeriod                    time.Duration
 	webhookPort                   int
 	webhookCertDir                string
@@ -160,6 +161,9 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&machineHealthCheckConcurrency, "machinehealthcheck-concurrency", 10,
 		"Number of machine health checks to process simultaneously")
 
+	fs.BoolVar(&waitForNodeDeletion, "wait-for-node-deletion", false,
+		"If set, the Machine controller will postpone deletion of a Machine until the corresponding node is deleted in the Cluster's API.")
+
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
@@ -341,10 +345,11 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
 		os.Exit(1)
 	}
 	if err := (&controllers.MachineReconciler{
-		Client:           mgr.GetClient(),
-		APIReader:        mgr.GetAPIReader(),
-		Tracker:          tracker,
-		WatchFilterValue: watchFilterValue,
+		Client:              mgr.GetClient(),
+		APIReader:           mgr.GetAPIReader(),
+		Tracker:             tracker,
+		WatchFilterValue:    watchFilterValue,
+		WaitForNodeDeletion: waitForNodeDeletion,
 	}).SetupWithManager(ctx, mgr, concurrency(machineConcurrency)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Machine")
 		os.Exit(1)