Issue-10544 ignore unreachable cluster while deleting machinePool

Signed-off-by: melserngawy <[email protected]>

Author: serngawy

exp/internal/controllers/machinepool_controller.go

@@ -253,33 +253,52 @@ func (r *MachinePoolReconciler) reconcile(ctx context.Context, cluster *clusterv
 	return res, kerrors.NewAggregate(errs)
 }
 
-func (r *MachinePoolReconciler) reconcileDelete(ctx context.Context, cluster *clusterv1.Cluster, mp *expv1.MachinePool) error {
-	if ok, err := r.reconcileDeleteExternal(ctx, mp); !ok || err != nil {
-		// Return early and don't remove the finalizer if we got an error or
-		// the external reconciliation deletion isn't ready.
-		return err
-	}
+func (r *MachinePoolReconciler) reconcileDelete(ctx context.Context, cluster *clusterv1.Cluster, machinepool *expv1.MachinePool) error {
+	deleteAllowed, clusterClient, err := r.isDeleteMachinePoolAllowed(ctx, cluster)
+
+	// Check for cluster allowing delete or machinePool delete timeout.
+	if deleteAllowed || r.isMachinePoolDeleteTimeoutPassed(machinepool) {
+		if ok, err := r.reconcileDeleteExternal(ctx, machinepool); !ok || err != nil {
+			// Return early and don't remove the finalizer if we got an error or
+			// the external reconciliation deletion isn't ready.
+			return err
+		}
 
-	if err := r.reconcileDeleteNodes(ctx, cluster, mp); err != nil {
-		// Return early and don't remove the finalizer if we got an error.
-		return err
+		// Delete nodes when cluster accessor available & there are nodes to delete.
+		if len(machinepool.Status.NodeRefs) > 0 && clusterClient != nil {
+			if err = r.deleteRetiredNodes(ctx, clusterClient, machinepool.Status.NodeRefs, machinepool.Spec.ProviderIDList); err != nil {
+				return err
+			}
+		}
+		// Remove finalizer to delete the resource.
+		controllerutil.RemoveFinalizer(machinepool, expv1.MachinePoolFinalizer)
+
+		return nil
 	}
 
-	controllerutil.RemoveFinalizer(mp, expv1.MachinePoolFinalizer)
-	return nil
+	return err
 }
 
-func (r *MachinePoolReconciler) reconcileDeleteNodes(ctx context.Context, cluster *clusterv1.Cluster, machinepool *expv1.MachinePool) error {
-	if len(machinepool.Status.NodeRefs) == 0 {
-		return nil
+// isMachinePoolDeleteTimeoutPassed check the machinePool node delete time out.
+func (r *MachinePoolReconciler) isMachinePoolDeleteTimeoutPassed(machinepool *expv1.MachinePool) bool {
+	if !machinepool.DeletionTimestamp.IsZero() {
+		return machinepool.DeletionTimestamp.Add(machinepool.Spec.Template.Spec.NodeDeletionTimeout.Duration).After(time.Now())
 	}
+	return false
+}
 
-	clusterClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
-	if err != nil {
-		return err
+// isDeleteMachinePoolAllowed check the cluster status and target cluster client accessor state.
+func (r *MachinePoolReconciler) isDeleteMachinePoolAllowed(ctx context.Context, cluster *clusterv1.Cluster) (bool, client.Client, error) {
+	// Check if cluster is been deleted
+	if !cluster.DeletionTimestamp.IsZero() {
+		// return true when cluster is been deleted to delete external resources and skip delete nodes refs.
+		return true, nil, nil
 	}
 
-	return r.deleteRetiredNodes(ctx, clusterClient, machinepool.Status.NodeRefs, machinepool.Spec.ProviderIDList)
+	// Check if the target cluster client is reachable.
+	clusterClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
+
+	return err == nil, clusterClient, err
 }
 
 // reconcileDeleteExternal tries to delete external references, returning true if it cannot find any.