Add finalizer for CSV cleanup

Cleanup CRB/CR/MWC/VWC via finalizer.

Signed-off-by: Todd Short <[email protected]>

Author: tmshort

Diff for: pkg/controller/operators/operatorconditiongenerator_controller.go

@@ -2,15 +2,20 @@ package operators
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 
 	"github.com/go-logr/logr"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -95,6 +100,10 @@ func (r *OperatorConditionGeneratorReconciler) Reconcile(ctx context.Context, re
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	if err, ok := r.processFinalizer(ctx, in); !ok {
+		return ctrl.Result{}, err
+	}
+
 	operatorCondition := &operatorsv2.OperatorCondition{
 		ObjectMeta: metav1.ObjectMeta{
 			// For now, only generate an OperatorCondition with the same name as the csv.
@@ -170,3 +179,100 @@ func (r *OperatorConditionGeneratorReconciler) ensureOperatorCondition(operatorC
 	existingOperatorCondition.Spec.ServiceAccounts = operatorCondition.Spec.ServiceAccounts
 	return r.Client.Update(context.TODO(), existingOperatorCondition)
 }
+
+// Return values, err, ok; ok == true: continue Reconcile, ok == false: exit Reconcile
+func (r *OperatorConditionGeneratorReconciler) processFinalizer(ctx context.Context, csv *operatorsv1alpha1.ClusterServiceVersion) (error, bool) {
+	myFinalizerName := "operators.coreos.com/csv-cleanup"
+	log := r.log.WithValues("name", csv.GetName()).WithValues("namespace", csv.GetNamespace())
+
+	if csv.ObjectMeta.DeletionTimestamp.IsZero() {
+		// CSV is not being deleted, add finalizer if not present
+		if !controllerutil.ContainsFinalizer(csv, myFinalizerName) {
+			patch := client.MergeFrom(csv.DeepCopy())
+			controllerutil.AddFinalizer(csv, myFinalizerName)
+			if err := r.Client.Patch(ctx, csv, patch); err != nil {
+				log.Error(err, "Adding finalizer")
+				return err, false
+			}
+		}
+		return nil, true
+	}
+
+	if !controllerutil.ContainsFinalizer(csv, myFinalizerName) {
+		// Finalizer has been removed; stop reconciliation as the CSV is being deleted
+		return nil, false
+	}
+
+	// CSV is being deleted and the finalizer still present; do any clean up
+	ownerSelector := ownerutil.CSVOwnerSelector(csv)
+	listOptions := client.ListOptions{
+		LabelSelector: ownerSelector,
+	}
+	deleteOptions := client.DeleteAllOfOptions{
+		ListOptions: listOptions,
+	}
+	// Look for resources owned by this CSV, and delete them.
+	log.WithValues("selector", ownerSelector).Info("Cleaning up resources after CSV deletion")
+	var errs []error
+
+	err := r.Client.DeleteAllOf(ctx, &rbacv1.ClusterRoleBinding{}, &deleteOptions)
+	if client.IgnoreNotFound(err) != nil {
+		log.Error(err, "Deleting ClusterRoleBindings on CSV delete")
+		errs = append(errs, err)
+	}
+
+	err = r.Client.DeleteAllOf(ctx, &rbacv1.ClusterRole{}, &deleteOptions)
+	if client.IgnoreNotFound(err) != nil {
+		log.Error(err, "Deleting ClusterRoles on CSV delete")
+		errs = append(errs, err)
+	}
+
+	err = r.Client.DeleteAllOf(ctx, &admissionregistrationv1.MutatingWebhookConfiguration{}, &deleteOptions)
+	if client.IgnoreNotFound(err) != nil {
+		log.Error(err, "Deleting MutatingWebhookConfigurations on CSV delete")
+		errs = append(errs, err)
+	}
+
+	err = r.Client.DeleteAllOf(ctx, &admissionregistrationv1.ValidatingWebhookConfiguration{}, &deleteOptions)
+	if client.IgnoreNotFound(err) != nil {
+		log.Error(err, "Deleting ValidatingWebhookConfigurations on CSV delete")
+		errs = append(errs, err)
+	}
+
+	// Make sure things are deleted
+	crbList := &rbacv1.ClusterRoleBindingList{}
+	if _ = r.Client.List(ctx, crbList, &listOptions); len(crbList.Items) != 0 {
+		errs = append(errs, fmt.Errorf("Waiting for ClusterRoleBindings to delete"))
+	}
+
+	crList := &rbacv1.ClusterRoleList{}
+	if _ = r.Client.List(ctx, crList, &listOptions); len(crList.Items) != 0 {
+		errs = append(errs, fmt.Errorf("Waiting for ClusterRoles to delete"))
+	}
+
+	mwcList := &admissionregistrationv1.MutatingWebhookConfigurationList{}
+	if _ = r.Client.List(ctx, mwcList, &listOptions); len(mwcList.Items) != 0 {
+		errs = append(errs, fmt.Errorf("Waiting for MutatingWebhookConfigurations to delete"))
+	}
+
+	vwcList := &admissionregistrationv1.ValidatingWebhookConfigurationList{}
+	if _ = r.Client.List(ctx, vwcList, &listOptions); len(vwcList.Items) != 0 {
+		errs = append(errs, fmt.Errorf("Waiting for ValidatingWebhookConfigurations to delete"))
+	}
+
+	// Return any errors
+	if err := utilerrors.NewAggregate(errs); err != nil {
+		return err, false
+	}
+
+	// If no errors, remove our finalizer from the CSV and update
+	patch := client.MergeFrom(csv.DeepCopy())
+	controllerutil.RemoveFinalizer(csv, myFinalizerName)
+	if err := r.Client.Patch(ctx, csv, patch); err != nil {
+		log.Error(err, "Removing finalizer")
+		return err, false
+	}
+
+	// Stop reconciliation as the csv is being deleted
+	return nil, false
+}