feat: garbage collection feature enable

This change uses the new garbage collection service enables this during
the reconciliation of `AWSCluster` and `AWSManagedControlPlane`. Its
enabled via a new feature flag `ExternalResourceGC` which is disabled by
default. If the feature flag is enabled then the the gc service is called
in `reconcileDelete` for the infra clusters. The actual gc service does the
work of cleanup.

New commands have been added to `clusterawsadm` to allow users to
opt-in/out an already existing cluster from garbage collection.

Additionally, with the new mocks folder introduced with the gc service
the existing mocks have been deleted and tests/controllers updated.

Signed-off-by: Richard Case <[email protected]>

Author: richardcase

cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go

@@ -132,6 +132,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"elasticloadbalancing:CreateLoadBalancer",
 				"elasticloadbalancing:ConfigureHealthCheck",
 				"elasticloadbalancing:DeleteLoadBalancer",
+				"elasticloadbalancing:DeleteTargetGroup",
 				"elasticloadbalancing:DescribeLoadBalancers",
 				"elasticloadbalancing:DescribeLoadBalancerAttributes",
 				"elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml

@@ -190,6 +190,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml

@@ -189,6 +189,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml

@@ -189,6 +189,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml

@@ -189,6 +189,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml

@@ -184,6 +184,7 @@ Resources:
           - elasticloadbalancing:CreateLoadBalancer
           - elasticloadbalancing:ConfigureHealthCheck
           - elasticloadbalancing:DeleteLoadBalancer
+          - elasticloadbalancing:DeleteTargetGroup
           - elasticloadbalancing:DescribeLoadBalancers
           - elasticloadbalancing:DescribeLoadBalancerAttributes
           - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer

cmd/clusterawsadm/cmd/gc/disable.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gc
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/util/homedir"
+
+	gcproc "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/gc"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/cmd"
+)
+
+func newDisableCmd() *cobra.Command {
+	clusterName := ""
+	namespace := ""
+	kubeConfig := ""
+	kubeConfigDefault := ""
+
+	if home := homedir.HomeDir(); home != "" {
+		kubeConfigDefault = filepath.Join(home, ".kube", "config")
+	}
+
+	newCmd := &cobra.Command{
+		Use:   "disable",
+		Short: "Mark a cluster as NOT requiring external resource gc",
+		Long: cmd.LongDesc(`
+			This command will mark the given cluster as not requiring external
+			resource garbage collection (i.e. deleting) when the cluster is
+			requested to be deleted.
+		`),
+		Example: cmd.Examples(`
+			# Disable GC for a cluster using existing k8s context
+			clusterawsadm gc disable --cluster-name=test-cluster
+
+			# Disable GC for a cluster using kubeconfig
+			clusterawsadm gc disable --cluster-name=test-cluster --kubeconfig=test.kubeconfig
+		`),
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			proc, err := gcproc.New(gcproc.GCInput{
+				ClusterName:    clusterName,
+				Namespace:      namespace,
+				KubeconfigPath: kubeConfig,
+			})
+			if err != nil {
+				return fmt.Errorf("creating command processor: %w", err)
+			}
+
+			err = proc.Disable(cmd.Context())
+			if err != nil {
+				return fmt.Errorf("disabling garbage collection: %w", err)
+			}
+			fmt.Printf("Disabled garbage collection for cluster %s/%s\n", namespace, clusterName)
+
+			return nil
+		},
+	}
+
+	newCmd.Flags().StringVar(&clusterName, "cluster-name", "", "The name of the CAPA cluster")
+	newCmd.Flags().StringVarP(&namespace, "namespace", "n", "default", "The namespace for the cluster definition")
+	newCmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfigDefault, "Path to the kubeconfig file to use")
+
+	newCmd.MarkFlagRequired("cluster-name") //nolint: errcheck
+
+	return newCmd
+}

cmd/clusterawsadm/cmd/gc/enable.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gc
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/util/homedir"
+
+	gcproc "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/gc"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/cmd"
+)
+
+func newEnableCmd() *cobra.Command {
+	clusterName := ""
+	namespace := ""
+	kubeConfig := ""
+	kubeConfigDefault := ""
+
+	if home := homedir.HomeDir(); home != "" {
+		kubeConfigDefault = filepath.Join(home, ".kube", "config")
+	}
+
+	newCmd := &cobra.Command{
+		Use:   "enable",
+		Short: "Mark a cluster as requiring external resource gc",
+		Long: cmd.LongDesc(`
+			This command will mark the given cluster as requiring external
+			resource garbage collection (i.e. deleting) when the cluster is
+			requested to be deleted. This works by adding an annotation to the
+			infra cluster.
+		`),
+		Example: cmd.Examples(`
+			# Enable GC for a cluster using existing k8s context
+			clusterawsadm gc enable --cluster-name=test-cluster
+
+			# Enable GC for a cluster using kubeconfig
+			clusterawsadm gc enable --cluster-name=test-cluster --kubeconfig=test.kubeconfig
+		`),
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			proc, err := gcproc.New(gcproc.GCInput{
+				ClusterName:    clusterName,
+				Namespace:      namespace,
+				KubeconfigPath: kubeConfig,
+			})
+			if err != nil {
+				return fmt.Errorf("creating command processor: %w", err)
+			}
+
+			err = proc.Enable(cmd.Context())
+			if err != nil {
+				return fmt.Errorf("enabling garbage collection: %w", err)
+			}
+			fmt.Printf("Enabled garbage collection for cluster %s/%s\n", namespace, clusterName)
+
+			return nil
+		},
+	}
+
+	newCmd.Flags().StringVar(&clusterName, "cluster-name", "", "The name of the CAPA cluster")
+	newCmd.Flags().StringVarP(&namespace, "namespace", "n", "default", "The namespace for the cluster definition")
+	newCmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfigDefault, "Path to the kubeconfig file to use")
+
+	newCmd.MarkFlagRequired("cluster-name") //nolint: errcheck
+
+	return newCmd
+}

cmd/clusterawsadm/cmd/gc/gc.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gc
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// RootCmd is the root of the `gc command`.
+func RootCmd() *cobra.Command {
+	newCmd := &cobra.Command{
+		Use:   "gc [command]",
+		Short: "Commands related to garbage collecting external resources of clusters",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := cmd.Help(); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	newCmd.AddCommand(newEnableCmd())
+	newCmd.AddCommand(newDisableCmd())
+
+	return newCmd
+}

cmd/clusterawsadm/cmd/root.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/bootstrap"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/controller"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/eks"
+	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/gc"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/resource"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/version"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/cmd"
@@ -71,6 +72,7 @@ func RootCmd() *cobra.Command {
 	newCmd.AddCommand(eks.RootCmd())
 	newCmd.AddCommand(controller.RootCmd())
 	newCmd.AddCommand(resource.RootCmd())
+	newCmd.AddCommand(gc.RootCmd())
 
 	return newCmd
 }

cmd/clusterawsadm/gc/disable.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gc
+
+import (
+	"context"
+	"fmt"
+
+	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/annotations"
+	"sigs.k8s.io/cluster-api/util/patch"
+)
+
+// Disable is used to disable external resource garbage collection for a cluster.
+func (c *CmdProcessor) Disable(ctx context.Context) error {
+	infraObj, err := c.getInfraCluster(ctx)
+	if err != nil {
+		return err
+	}
+
+	patchHelper, err := patch.NewHelper(infraObj, c.client)
+	if err != nil {
+		return fmt.Errorf("creating patch helper: %w", err)
+	}
+
+	annotations.Set(infraObj, expinfrav1.ExternalResourceGCAnnotation, "false")
+
+	if err := patchHelper.Patch(ctx, infraObj); err != nil {
+		return fmt.Errorf("patching infra cluster with gc annotation: %w", err)
+	}
+
+	return nil
+}