Empty buckets during Delete()

Author: 0x2b3bfa0

.gitignore

@@ -41,4 +41,4 @@ terraform
 # Ignore the Go vendor directory, superseded by proxy.golang.org and go.{mod,sum}
 vendor
 
-./terraform-provider-iterative
+terraform-provider-iterative

task/aws/resources/resource_bucket.go

@@ -76,45 +76,6 @@ func (b *Bucket) Update(ctx context.Context) error {
 }
 
 func (b *Bucket) Delete(ctx context.Context) error {
-	listInput := s3.ListObjectsV2Input{
-		Bucket: aws.String(b.Identifier),
-	}
-
-	for paginator := s3.NewListObjectsV2Paginator(b.Client.Services.S3, &listInput); paginator.HasMorePages(); {
-		page, err := paginator.NextPage(ctx)
-
-		if err != nil {
-			var e smithy.APIError
-			if errors.As(err, &e) && e.ErrorCode() == "NoSuchBucket" {
-				b.Resource = nil
-				return nil
-			}
-			return err
-		}
-
-		if len(page.Contents) == 0 {
-			break
-		}
-
-		var objects []types.ObjectIdentifier
-		for _, object := range page.Contents {
-			objects = append(objects, types.ObjectIdentifier{
-				Key: object.Key,
-			})
-		}
-
-		input := s3.DeleteObjectsInput{
-			Bucket: aws.String(b.Identifier),
-			Delete: &types.Delete{
-				Objects: objects,
-			},
-		}
-
-		if _, err = b.Client.Services.S3.DeleteObjects(ctx, &input); err != nil {
-			return err
-		}
-	}
-
 	deleteInput := s3.DeleteBucketInput{
 		Bucket: aws.String(b.Identifier),
 	}

task/aws/task.go

@@ -220,6 +220,10 @@ func (t *Task) Delete(ctx context.Context) error {
 	if err := t.DataSources.Credentials.Read(ctx); err != nil {
 		return err
 	}
+	log.Println("[INFO] Emptying Bucket...")
+	if err := t.Empty(ctx); err != nil {
+		return err
+	}
 	log.Println("[INFO] Deleting Bucket...")
 	if err := t.Resources.Bucket.Delete(ctx); err != nil {
 		return err
@@ -252,6 +256,14 @@ func (t *Task) Push(ctx context.Context, source string) error {
 	return machine.Transfer(ctx, source, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]+"/data")
 }
 
+func (t *Task) Empty(ctx context.Context) error {
+	if err := t.Read(ctx); err != nil {
+		return err
+	}
+
+	return machine.Purge(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"])
+}
+
 func (t *Task) Start(ctx context.Context) error {
 	return t.Resources.AutoScalingGroup.Update(ctx)
 }

task/az/task.go

@@ -205,6 +205,10 @@ func (t *Task) Delete(ctx context.Context) error {
 	if err := t.Resources.VirtualNetwork.Delete(ctx); err != nil {
 		return err
 	}
+	log.Println("[INFO] Emptying BlobContainer...")
+	if err := t.Empty(ctx); err != nil {
+		return err
+	}
 	log.Println("[INFO] Deleting BlobContainer...")
 	if err := t.Resources.BlobContainer.Delete(ctx); err != nil {
 		return err
@@ -245,6 +249,14 @@ func (t *Task) Push(ctx context.Context, source string) error {
 	return machine.Transfer(ctx, source, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]+"/data")
 }
 
+func (t *Task) Empty(ctx context.Context) error {
+	if err := t.Read(ctx); err != nil {
+		return err
+	}
+
+	return machine.Purge(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"])
+}
+
 func (t *Task) Start(ctx context.Context) error {
 	return t.Resources.VirtualMachineScaleSet.Update(ctx)
 }

task/common/machine/storage.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/sync"
+	"github.com/rclone/rclone/fs/operations"
 
 	"terraform-provider-iterative/task/common"
 )
@@ -99,7 +100,16 @@ func Transfer(ctx context.Context, source, destination string) error {
 		return err
 	}
 
-	if err := sync.CopyDir(ctx, destinationFileSystem, sourceFileSystem, true); err != nil {
+	return sync.CopyDir(ctx, destinationFileSystem, sourceFileSystem, true)
+}
+
+func Purge(ctx context.Context, destination string) error {
+	destinationFileSystem, err := fs.NewFs(ctx, destination)
+	if err != nil {
+		return err
+	}
+
+	if err := operations.Purge(ctx, destinationFileSystem, ""); err.Error() != "resource not found" {
 		return err
 	}
 

task/gcp/resources/resource_bucket.go

@@ -65,19 +65,6 @@ func (b *Bucket) Delete(ctx context.Context) error {
 		return nil
 	}
 
-	deletePage := func(objects *storage.Objects) error {
-		for _, object := range objects.Items {
-			if err := b.Client.Services.Storage.Objects.Delete(b.Identifier, object.Name).Do(); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	if err := b.Client.Services.Storage.Objects.List(b.Identifier).Pages(ctx, deletePage); err != nil {
-		return err
-	}
-
 	if err := b.Client.Services.Storage.Buckets.Delete(b.Identifier).Do(); err != nil {
 		return err
 	}

task/gcp/task.go

@@ -300,6 +300,10 @@ func (t *Task) Delete(ctx context.Context) error {
 	if err := t.Resources.FirewallDenyIngress.Delete(ctx); err != nil {
 		return err
 	}
+	log.Println("[INFO] Emptying Bucket...")
+	if err := t.Empty(ctx); err != nil {
+		return err
+	}
 	log.Println("[INFO] Deleting Bucket...")
 	if err := t.Resources.Bucket.Delete(ctx); err != nil {
 		return err
@@ -332,6 +336,14 @@ func (t *Task) Push(ctx context.Context, source string) error {
 	return machine.Transfer(ctx, source, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"]+"/data")
 }
 
+func (t *Task) Empty(ctx context.Context) error {
+	if err := t.Read(ctx); err != nil {
+		return err
+	}
+
+	return machine.Purge(ctx, (*t.DataSources.Credentials.Resource)["RCLONE_REMOTE"])
+}
+
 func (t *Task) Start(ctx context.Context) error {
 	return t.Resources.InstanceGroupManager.Update(ctx)
 }