implement forceful deletion

Fixes #36

Author: dgrove-oss

api/v1beta2/appwrapper_types.go

@@ -95,13 +95,15 @@ const (
 	ResourcesDeployed AppWrapperCondition = "ResourcesDeployed"
 	PodsReady         AppWrapperCondition = "PodsReady"
 	Unhealthy         AppWrapperCondition = "Unhealthy"
+	DeletingResources AppWrapperCondition = "DeletingResources"
 )
 
 const (
 	WarmupGracePeriodDurationAnnotation  = "workload.codeflare.dev.appwrapper/warmupGracePeriodDuration"
 	FailureGracePeriodDurationAnnotation = "workload.codeflare.dev.appwrapper/failureGracePeriodDuration"
 	ResetPauseDurationAnnotation         = "workload.codeflare.dev.appwrapper/resetPauseDuration"
 	RetryLimitAnnotation                 = "workload.codeflare.dev.appwrapper/retryLimit"
+	DeletionGracePeriodAnnotation        = "workload.codeflare.dev.appwrapper/deletionGracePeriodDuration"
 )
 
 //+kubebuilder:object:root=true

internal/controller/appwrapper/appwrapper_controller.go

@@ -437,6 +437,17 @@ func (r *AppWrapperReconciler) resettingPauseDuration(ctx context.Context, aw *w
 	return r.Config.FaultTolerance.ResetPause
 }
 
+func (r *AppWrapperReconciler) deletionGraceDuration(ctx context.Context, aw *workloadv1beta2.AppWrapper) time.Duration {
+	if userPeriod, ok := aw.Annotations[workloadv1beta2.DeletionGracePeriodAnnotation]; ok {
+		if duration, err := time.ParseDuration(userPeriod); err == nil {
+			return duration
+		} else {
+			log.FromContext(ctx).Info("Malformed deletion period annotation", "annotation", userPeriod, "error", err)
+		}
+	}
+	return r.Config.FaultTolerance.DeletionGracePeriod
+}
+
 func clearCondition(aw *workloadv1beta2.AppWrapper, condition workloadv1beta2.AppWrapperCondition, reason string, message string) {
 	if meta.IsStatusConditionTrue(aw.Status.Conditions, string(condition)) {
 		meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{

internal/controller/appwrapper/resource_management.go

@@ -19,9 +19,11 @@ package appwrapper
 import (
 	"context"
 	"fmt"
+	"time"
 
 	workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
 	"github.com/project-codeflare/appwrapper/pkg/utils"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -164,7 +166,11 @@ func (r *AppWrapperReconciler) createComponents(ctx context.Context, aw *workloa
 }
 
 func (r *AppWrapperReconciler) deleteComponents(ctx context.Context, aw *workloadv1beta2.AppWrapper) bool {
-	// TODO forceful deletion: See https://github.com/project-codeflare/appwrapper/issues/36
+	meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
+		Type:   string(workloadv1beta2.DeletingResources),
+		Status: metav1.ConditionTrue,
+		Reason: "DeletionInitiated",
+	})
 	log := log.FromContext(ctx)
 	remaining := 0
 	for _, component := range aw.Spec.Components {
@@ -181,5 +187,59 @@ func (r *AppWrapperReconciler) deleteComponents(ctx context.Context, aw *workloa
 		}
 		remaining++ // no error deleting resource, resource therefore still exists
 	}
-	return remaining == 0
+
+	deletionGracePeriod := r.deletionGraceDuration(ctx, aw)
+	if deletionGracePeriod <= 0 {
+		// forced deletion is disabled; once remaining is 0 we are done
+		if remaining == 0 {
+			clearCondition(aw, workloadv1beta2.DeletingResources, "DeletionComplete", "")
+			return true
+		} else {
+			return false
+		}
+	}
+
+	pods := &v1.PodList{Items: []v1.Pod{}}
+	if err := r.List(ctx, pods,
+		client.UnsafeDisableDeepCopy,
+		client.InNamespace(aw.Namespace),
+		client.MatchingLabels{AppWrapperLabel: aw.Name}); err != nil {
+		log.Error(err, "Pod list error")
+	}
+
+	if remaining == 0 && len(pods.Items) == 0 {
+		// no resources, no pods, deletion is complete
+		clearCondition(aw, workloadv1beta2.DeletingResources, "DeletionComplete", "")
+		return true
+	}
+
+	whenInitiated := meta.FindStatusCondition(aw.Status.Conditions, string(workloadv1beta2.DeletingResources)).LastTransitionTime
+	if time.Now().Before(whenInitiated.Time.Add(deletionGracePeriod)) {
+		// Deadline hasn't expired, just requeue the deletion
+		return false
+	}
+
+	if len(pods.Items) > 0 {
+		// force deletion of pods first
+		for _, pod := range pods.Items {
+			if err := r.Delete(ctx, &pod, client.GracePeriodSeconds(0)); err != nil {
+				log.Error(err, "Forceful pod deletion error")
+			}
+		}
+	} else {
+		// force deletion of wrapped resources once pods are gone
+		for _, component := range aw.Spec.Components {
+			obj, err := parseComponent(aw, component.Template.Raw)
+			if err != nil {
+				log.Error(err, "Parsing error")
+				continue
+			}
+			if err := r.Delete(ctx, obj, client.GracePeriodSeconds(0)); err != nil && !apierrors.IsNotFound(err) {
+				log.Error(err, "Forceful deletion error")
+			}
+		}
+	}
+
+	// requeue deletion
+	return false
 }

pkg/config/config.go

@@ -26,10 +26,11 @@ type AppWrapperConfig struct {
 }
 
 type FaultToleranceConfig struct {
-	WarmupGracePeriod  time.Duration `json:"warmupGracePeriod,omitempty"`
-	FailureGracePeriod time.Duration `json:"failureGracePeriod,omitempty"`
-	ResetPause         time.Duration `json:"resetPause,omitempty"`
-	RetryLimit         int32         `json:"retryLimit,omitempty"`
+	WarmupGracePeriod   time.Duration `json:"warmupGracePeriod,omitempty"`
+	FailureGracePeriod  time.Duration `json:"failureGracePeriod,omitempty"`
+	ResetPause          time.Duration `json:"resetPause,omitempty"`
+	RetryLimit          int32         `json:"retryLimit,omitempty"`
+	DeletionGracePeriod time.Duration `json:"deletionGracePeriod,omitempty"`
 }
 
 type CertManagementConfig struct {
@@ -49,10 +50,11 @@ func NewConfig(namespace string) *AppWrapperConfig {
 		ManageJobsWithoutQueueName: true,
 		StandaloneMode:             false,
 		FaultTolerance: FaultToleranceConfig{
-			WarmupGracePeriod:  5 * time.Minute,
-			FailureGracePeriod: 1 * time.Minute,
-			ResetPause:         90 * time.Second,
-			RetryLimit:         3,
+			WarmupGracePeriod:   5 * time.Minute,
+			FailureGracePeriod:  1 * time.Minute,
+			ResetPause:          90 * time.Second,
+			RetryLimit:          3,
+			DeletionGracePeriod: 10 * time.Minute,
 		},
 		CertManagement: CertManagementConfig{
 			Namespace:                   namespace,