Merge pull request #3610 from richardcase/1718_gc_new_service

feat: external load balancer garbage collection (part 2) - new gc service

Author: k8s-ci-robot

controlplane/eks/controllers/awsmanagedcontrolplane_controller_unit_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

controlplane/eks/controllers/helpers_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

exp/api/v1beta1/awsfargateprofile_types.go

@@ -27,11 +27,6 @@ import (
 	"sigs.k8s.io/cluster-api/errors"
 )
 
-const (
-	// FargateProfileFinalizer allows the controller to clean up resources on delete.
-	FargateProfileFinalizer = "awsfargateprofile.infrastructure.cluster.x-k8s.io"
-)
-
 var (
 	// DefaultEKSFargateRole is the name of the default IAM role to use for fargate
 	// profiles if no other role is supplied in the spec and if iam role creation

exp/api/v1beta1/awsmachinepool_types.go

@@ -27,9 +27,6 @@ import (
 
 // Constants block.
 const (
-	// MachinePoolFinalizer is the finalizer for the machine pool.
-	MachinePoolFinalizer = "awsmachinepool.infrastructure.cluster.x-k8s.io"
-
 	// LaunchTemplateLatestVersion defines the launching of the latest version of the template.
 	LaunchTemplateLatestVersion = "$Latest"
 )

exp/api/v1beta1/awsmanagedmachinepool_types.go

@@ -27,11 +27,6 @@ import (
 	"sigs.k8s.io/cluster-api/errors"
 )
 
-const (
-	// ManagedMachinePoolFinalizer allows the controller to clean up resources on delete.
-	ManagedMachinePoolFinalizer = "awsmanagedmachinepools.infrastructure.cluster.x-k8s.io"
-)
-
 // ManagedMachineAMIType specifies which AWS AMI to use for a managed MachinePool.
 type ManagedMachineAMIType string
 

exp/api/v1beta1/finalizers.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+const (
+	// FargateProfileFinalizer allows the controller to clean up resources on delete.
+	FargateProfileFinalizer = "awsfargateprofile.infrastructure.cluster.x-k8s.io"
+
+	// MachinePoolFinalizer is the finalizer for the machine pool.
+	MachinePoolFinalizer = "awsmachinepool.infrastructure.cluster.x-k8s.io"
+
+	// ManagedMachinePoolFinalizer allows the controller to clean up resources on delete.
+	ManagedMachinePoolFinalizer = "awsmanagedmachinepools.infrastructure.cluster.x-k8s.io"
+)

exp/api/v1beta1/types.go

@@ -22,6 +22,12 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
 )
 
+const (
+	// ExternalResourceGCAnnotation is the name of an annotation that indicates if
+	// external resources should be garbage collected for the cluster.
+	ExternalResourceGCAnnotation = "aws.cluster.x-k8s.io/external-resource-gc"
+)
+
 // EBS can be used to automatically set up EBS volumes when an instance is launched.
 type EBS struct {
 	// Encrypted is whether the volume should be encrypted or not.

pkg/annotations/annotations.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Set will set the value of an annotation on the supplied object. If there is no annotation it will be created.
+func Set(obj metav1.Object, name, value string) {
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[name] = value
+	obj.SetAnnotations(annotations)
+}
+
+// Get will get the value of the supplied annotation.
+func Get(obj metav1.Object, name string) (value string, found bool) {
+	annotations := obj.GetAnnotations()
+	if len(annotations) == 0 {
+		return "", false
+	}
+
+	value, found = annotations[name]
+
+	return
+}
+
+// Has returns true if the supplied object has the supplied annotation.
+func Has(obj metav1.Object, name string) bool {
+	annotations := obj.GetAnnotations()
+	if len(annotations) == 0 {
+		return false
+	}
+
+	_, found := annotations[name]
+
+	return found
+}

pkg/cloud/scope/clients.go

@@ -28,6 +28,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks/eksiface"
 	"github.com/aws/aws-sdk-go/service/elb"
 	"github.com/aws/aws-sdk-go/service/elb/elbiface"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
 	"github.com/aws/aws-sdk-go/service/eventbridge"
 	"github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface"
 	"github.com/aws/aws-sdk-go/service/iam"
@@ -91,6 +93,18 @@ func NewELBClient(scopeUser cloud.ScopeUsage, session cloud.Session, logger clou
 	return elbClient
 }
 
+// NewELBv2Client creates a new ELB v2 API client for a given session.
+func NewELBv2Client(scopeUser cloud.ScopeUsage, session cloud.Session, logger cloud.Logger, target runtime.Object) elbv2iface.ELBV2API {
+	elbClient := elbv2.New(session.Session(), aws.NewConfig().WithLogLevel(awslogs.GetAWSLogLevel(logger)).WithLogger(awslogs.NewWrapLogr(logger)))
+	elbClient.Handlers.Build.PushFrontNamed(getUserAgentHandler())
+	elbClient.Handlers.Sign.PushFront(session.ServiceLimiter(elbv2.ServiceID).LimitRequest)
+	elbClient.Handlers.CompleteAttempt.PushFront(awsmetrics.CaptureRequestMetrics(scopeUser.ControllerName()))
+	elbClient.Handlers.CompleteAttempt.PushFront(session.ServiceLimiter(elbv2.ServiceID).ReviewResponse)
+	elbClient.Handlers.Complete.PushBack(recordAWSPermissionsIssue(target))
+
+	return elbClient
+}
+
 // NewEventBridgeClient creates a new EventBridge API client for a given session.
 func NewEventBridgeClient(scopeUser cloud.ScopeUsage, session cloud.Session, target runtime.Object) eventbridgeiface.EventBridgeAPI {
 	eventBridgeClient := eventbridge.New(session.Session())

pkg/cloud/scope/session.go

@@ -27,6 +27,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/go-logr/logr"
@@ -189,6 +190,7 @@ func newServiceLimiters() throttle.ServiceLimiters {
 	return throttle.ServiceLimiters{
 		ec2.ServiceID:                      newEC2ServiceLimiter(),
 		elb.ServiceID:                      newGenericServiceLimiter(),
+		elbv2.ServiceID:                    newGenericServiceLimiter(),
 		resourcegroupstaggingapi.ServiceID: newGenericServiceLimiter(),
 		secretsmanager.ServiceID:           newGenericServiceLimiter(),
 	}

pkg/cloud/services/awsnode/cni_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package awsnode
 
 import (

pkg/cloud/services/gc/cleanup.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gc
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/arn"
+	rgapi "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+
+	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
+	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/annotations"
+)
+
+const (
+	serviceNameTag    = "kubernetes.io/service-name"
+	eksClusterNameTag = "aws:eks:cluster-name"
+)
+
+// ReconcileDelete is responsible for determining if the infra cluster needs to be garbage collected. If
+// does then it will perform garbage collection. For example, it will delete the ELB/NLBs that where created
+// as a result of Services of type load balancer.
+func (s *Service) ReconcileDelete(ctx context.Context) error {
+	s.scope.Info("reconciling deletion for garbage collection")
+
+	val, found := annotations.Get(s.scope.InfraCluster(), expinfrav1.ExternalResourceGCAnnotation)
+	if !found {
+		val = "true"
+	}
+
+	shouldGC, err := strconv.ParseBool(val)
+	if err != nil {
+		return fmt.Errorf("converting value %s of annotation %s to bool: %w", val, expinfrav1.ExternalResourceGCAnnotation, err)
+	}
+
+	if !shouldGC {
+		s.scope.Info("cluster opted-out of garbage collection")
+
+		return nil
+	}
+
+	return s.deleteResources(ctx)
+}
+
+func (s *Service) deleteResources(ctx context.Context) error {
+	s.scope.Info("deleting aws resources created by tenant cluster")
+
+	serviceTag := infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName())
+	awsInput := rgapi.GetResourcesInput{
+		ResourceTypeFilters: nil,
+		TagFilters: []*rgapi.TagFilter{
+			{
+				Key:    aws.String(serviceTag),
+				Values: []*string{aws.String(string(infrav1.ResourceLifecycleOwned))},
+			},
+		},
+	}
+
+	awsOutput, err := s.resourceTaggingClient.GetResourcesWithContext(ctx, &awsInput)
+	if err != nil {
+		return fmt.Errorf("getting tagged resources: %w", err)
+	}
+
+	resources := []*AWSResource{}
+
+	for i := range awsOutput.ResourceTagMappingList {
+		mapping := awsOutput.ResourceTagMappingList[i]
+		parsedArn, err := arn.Parse(*mapping.ResourceARN)
+		if err != nil {
+			return fmt.Errorf("parsing resource arn %s: %w", *mapping.ResourceARN, err)
+		}
+
+		tags := map[string]string{}
+		for _, rgTag := range mapping.Tags {
+			tags[*rgTag.Key] = *rgTag.Value
+		}
+
+		resources = append(resources, &AWSResource{
+			ARN:  &parsedArn,
+			Tags: tags,
+		})
+	}
+
+	if deleteErr := s.cleanupFuncs.Execute(ctx, resources); deleteErr != nil {
+		return fmt.Errorf("deleting resources: %w", deleteErr)
+	}
+
+	return nil
+}
+
+func (s *Service) isMatchingResource(resource *AWSResource, serviceName, resourceName string) bool {
+	if resource.ARN.Service != serviceName {
+		s.scope.V(5).Info("Resource not for service", "arn", resource.ARN.String(), "service_name", serviceName, "resource_name", resourceName)
+		return false
+	}
+	if !strings.HasPrefix(resource.ARN.Resource, resourceName+"/") {
+		s.scope.V(5).Info("Resource type does not match", "arn", resource.ARN.String(), "service_name", serviceName, "resource_name", resourceName)
+		return false
+	}
+
+	return true
+}