Issue-10544 ignore unreachable cluster while deleting machinePool

Signed-off-by: melserngawy <[email protected]>

Author: serngawy

controllers/remote/cluster_cache_tracker.go

@@ -59,6 +59,9 @@ const (
 	healthCheckUnhealthyThreshold = 10
 	initialCacheSyncTimeout       = 5 * time.Minute
 	clusterCacheControllerName    = "cluster-cache-tracker"
+
+	// ClusterAccessorErrorMessage error message for failing to create cluster accessor.
+	ClusterAccessorErrorMessage = "failed to create cluster accessor"
 )
 
 // ErrClusterLocked is returned in methods that require cluster-level locking
@@ -267,7 +270,7 @@ func (t *ClusterCacheTracker) getClusterAccessor(ctx context.Context, cluster cl
 	// for the cluster at the same time.
 	// Return an error if another go routine already tries to create a clusterAccessor.
 	if ok := t.clusterLock.TryLock(cluster); !ok {
-		return nil, errors.Wrapf(ErrClusterLocked, "failed to create cluster accessor: failed to get lock for cluster")
+		return nil, errors.Wrapf(ErrClusterLocked, ClusterAccessorErrorMessage)
 	}
 	defer t.clusterLock.Unlock(cluster)
 
@@ -281,7 +284,7 @@ func (t *ClusterCacheTracker) getClusterAccessor(ctx context.Context, cluster cl
 	log.V(4).Info("Creating new cluster accessor")
 	accessor, err := t.newClusterAccessor(ctx, cluster, indexes...)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to create cluster accessor")
+		return nil, errors.Wrap(err, ClusterAccessorErrorMessage)
 	}
 
 	log.V(4).Info("Storing new cluster accessor")

exp/internal/controllers/machinepool_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -260,7 +261,9 @@ func (r *MachinePoolReconciler) reconcileDelete(ctx context.Context, cluster *cl
 		return err
 	}
 
-	if err := r.reconcileDeleteNodes(ctx, cluster, mp); err != nil {
+	err := r.reconcileDeleteNodes(ctx, cluster, mp)
+	// Check if the return error has ClusterAccessorErrorMessage to ignore unreachable cluster.
+	if err != nil && !strings.Contains(err.Error(), remote.ClusterAccessorErrorMessage) {
 		// Return early and don't remove the finalizer if we got an error.
 		return err
 	}
@@ -274,7 +277,13 @@ func (r *MachinePoolReconciler) reconcileDeleteNodes(ctx context.Context, cluste
 		return nil
 	}
 
+	// return if cluster is deleting
+	if !cluster.DeletionTimestamp.IsZero() {
+		return nil
+	}
+
 	clusterClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
+
 	if err != nil {
 		return err
 	}