diff --git a/apis/v1alpha1/ack-generate-metadata.yaml b/apis/v1alpha1/ack-generate-metadata.yaml index aebbf43..1da6b78 100755 --- a/apis/v1alpha1/ack-generate-metadata.yaml +++ b/apis/v1alpha1/ack-generate-metadata.yaml @@ -1,13 +1,13 @@ ack_generate_info: - build_date: "2021-10-27T21:04:06Z" - build_hash: d2b063806d25cfcae4f2d4eb44f8e3f713b23e8e - go_version: go1.15 + build_date: "2021-12-22T13:40:06Z" + build_hash: 6f17f51682dc0d16c36aa456fd22855ce9282fbc + go_version: go1.16.4 version: v0.15.2 api_directory_checksum: c1d144a18336326f141e97e6800b47f64ed992cc api_version: v1alpha1 aws_sdk_go_version: v1.38.47 generator_config_info: - file_checksum: 3d4ab94742ecf92212b94a5e47bdda8258589718 + file_checksum: 2c5f9585b1b0356e601e17efc7f5c47bdabdf4ab original_file_name: generator.yaml last_modification: reason: API generation diff --git a/apis/v1alpha1/generator.yaml b/apis/v1alpha1/generator.yaml index d718c42..d0e2a9a 100644 --- a/apis/v1alpha1/generator.yaml +++ b/apis/v1alpha1/generator.yaml @@ -7,6 +7,10 @@ operations: primary_identifier_field_name: BackupArn resources: Table: + fields: + Tags: + compare: + is_ignored: true exceptions: errors: 404: @@ -15,11 +19,13 @@ resources: - InternalServerError - LimitExceededException - ResourceInUseException + update_operation: + custom_method_name: customUpdateTable hooks: + delta_pre_compare: + code: customPreCompare(delta, a, b) sdk_read_one_post_set_output: template_path: hooks/table/sdk_read_one_post_set_output.go.tpl - sdk_update_pre_build_request: - template_path: hooks/table/sdk_update_pre_build_request.go.tpl sdk_delete_pre_build_request: template_path: hooks/table/sdk_delete_pre_build_request.go.tpl GlobalTable: diff --git a/generator.yaml b/generator.yaml index d718c42..d0e2a9a 100644 --- a/generator.yaml +++ b/generator.yaml @@ -7,6 +7,10 @@ operations: primary_identifier_field_name: BackupArn resources: Table: + fields: + Tags: + compare: + is_ignored: true exceptions: errors: 404: @@ -15,11 +19,13 @@ resources: - InternalServerError - LimitExceededException - ResourceInUseException + update_operation: + custom_method_name: customUpdateTable hooks: + delta_pre_compare: + code: customPreCompare(delta, a, b) sdk_read_one_post_set_output: template_path: hooks/table/sdk_read_one_post_set_output.go.tpl - sdk_update_pre_build_request: - template_path: hooks/table/sdk_update_pre_build_request.go.tpl sdk_delete_pre_build_request: template_path: hooks/table/sdk_delete_pre_build_request.go.tpl GlobalTable: diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index b0ade59..904405b 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -75,3 +75,12 @@ spec: value: {{ join "," .Values.resourceTags | quote }} terminationGracePeriodSeconds: 10 nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }} + {{ if .Values.deployment.tolerations -}} + tolerations: {{ toYaml .Values.deployment.tolerations | nindent 8 }} + {{ end -}} + {{ if .Values.deployment.affinity -}} + affinity: {{ toYaml .Values.deployment.affinity | nindent 8 }} + {{ end -}} + {{ if .Values.deployment.priorityClassName -}} + priorityClassName: {{ .Values.deployment.priorityClassName -}} + {{ end -}} diff --git a/helm/values.yaml b/helm/values.yaml index a8990ee..879be1e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -15,9 +15,20 @@ deployment: annotations: {} labels: {} containerPort: 8080 + # Which nodeSelector to set? + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector nodeSelector: kubernetes.io/os: linux - + # Which tolerations to set? + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: {} + # What affinity to set? + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # Which priorityClassName to set? + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: + metrics: service: # Set to true to automatically create a Kubernetes Service resource for the diff --git a/pkg/resource/backup/sdk.go b/pkg/resource/backup/sdk.go index df68d7c..e95a091 100644 --- a/pkg/resource/backup/sdk.go +++ b/pkg/resource/backup/sdk.go @@ -144,8 +144,6 @@ func (rm *resourceManager) newDescribeRequestPayload( if r.ko.Status.ACKResourceMetadata != nil && r.ko.Status.ACKResourceMetadata.ARN != nil { res.SetBackupArn(string(*r.ko.Status.ACKResourceMetadata.ARN)) - } else { - res.SetBackupArn(rm.ARNFromName(*r.ko.Spec.BackupName)) } return res, nil @@ -277,8 +275,6 @@ func (rm *resourceManager) newDeleteRequestPayload( if r.ko.Status.ACKResourceMetadata != nil && r.ko.Status.ACKResourceMetadata.ARN != nil { res.SetBackupArn(string(*r.ko.Status.ACKResourceMetadata.ARN)) - } else { - res.SetBackupArn(rm.ARNFromName(*r.ko.Spec.BackupName)) } return res, nil diff --git a/pkg/resource/table/delta.go b/pkg/resource/table/delta.go index e3cf5c8..1388b27 100644 --- a/pkg/resource/table/delta.go +++ b/pkg/resource/table/delta.go @@ -40,6 +40,7 @@ func newResourceDelta( delta.Add("", a, b) return delta } + customPreCompare(delta, a, b) if !reflect.DeepEqual(a.ko.Spec.AttributeDefinitions, b.ko.Spec.AttributeDefinitions) { delta.Add("Spec.AttributeDefinitions", a.ko.Spec.AttributeDefinitions, b.ko.Spec.AttributeDefinitions) @@ -128,9 +129,6 @@ func newResourceDelta( delta.Add("Spec.TableName", a.ko.Spec.TableName, b.ko.Spec.TableName) } } - if !reflect.DeepEqual(a.ko.Spec.Tags, b.ko.Spec.Tags) { - delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags) - } return delta } diff --git a/pkg/resource/table/hooks.go b/pkg/resource/table/hooks.go index 73a7c76..c7015b2 100644 --- a/pkg/resource/table/hooks.go +++ b/pkg/resource/table/hooks.go @@ -14,11 +14,18 @@ package table import ( + "context" "errors" "time" - "github.com/aws-controllers-k8s/dynamodb-controller/apis/v1alpha1" + ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare" ackrequeue "github.com/aws-controllers-k8s/runtime/pkg/requeue" + ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log" + ackutil "github.com/aws-controllers-k8s/runtime/pkg/util" + svcsdk "github.com/aws/aws-sdk-go/service/dynamodb" + corev1 "k8s.io/api/core/v1" + + "github.com/aws-controllers-k8s/dynamodb-controller/apis/v1alpha1" ) var ( @@ -95,3 +102,242 @@ func isTableUpdating(r *resource) bool { dbis := *r.ko.Status.TableStatus return dbis == string(v1alpha1.TableStatus_SDK_UPDATING) } + +func (rm *resourceManager) customUpdateTable( + ctx context.Context, + desired *resource, + latest *resource, + delta *ackcompare.Delta, +) (updated *resource, err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.customUpdateTable") + defer exit(err) + + if isTableDeleting(latest) { + msg := "table is currently being deleted" + setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) + return desired, requeueWaitWhileDeleting + } + if isTableCreating(latest) { + msg := "table is currently being created" + setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) + return desired, requeueWaitWhileCreating + } + if isTableUpdating(latest) { + msg := "table is currently being updated" + setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) + return desired, requeueWaitWhileUpdating + } + if tableHasTerminalStatus(latest) { + msg := "table is in '" + *latest.ko.Status.TableStatus + "' status" + setTerminalCondition(desired, corev1.ConditionTrue, &msg, nil) + setSyncedCondition(desired, corev1.ConditionTrue, nil, nil) + return desired, nil + } + + // Merge in the information we read from the API call above to the copy of + // the original Kubernetes object we passed to the function + ko := desired.ko.DeepCopy() + rm.setStatusDefaults(ko) + + if delta.DifferentAt("Spec.Tags") { + if err := rm.syncTableTags(ctx, latest, desired); err != nil { + return nil, err + } + } + + // TODO(hilalymh): support updating all table field + return &resource{ko}, nil +} + +// syncTableTags updates a dynamodb table tags. +// +// TODO(hilalymh): move this function to a common utility file. This function can be reused +// to tag GlobalTable resources. +func (rm *resourceManager) syncTableTags( + ctx context.Context, + latest *resource, + desired *resource, +) (err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.syncTableTags") + defer exit(err) + + added, updated, removed := computeTagsDelta(latest.ko.Spec.Tags, desired.ko.Spec.Tags) + + // There are no API calls to update an existing tag. To update a tag we will have to first + // delete it and then recreate it with the new value. + + // Tags to remove + for _, updatedTag := range updated { + removed = append(removed, updatedTag.Key) + } + // Tags to create + added = append(added, updated...) + + if len(removed) > 0 { + _, err = rm.sdkapi.UntagResourceWithContext( + ctx, + &svcsdk.UntagResourceInput{ + ResourceArn: (*string)(latest.ko.Status.ACKResourceMetadata.ARN), + TagKeys: removed, + }, + ) + rm.metrics.RecordAPICall("GET", "UntagResource", err) + if err != nil { + return err + } + } + + if len(added) > 0 { + _, err = rm.sdkapi.TagResourceWithContext( + ctx, + &svcsdk.TagResourceInput{ + ResourceArn: (*string)(latest.ko.Status.ACKResourceMetadata.ARN), + Tags: sdkTagsFromResourceTags(added), + }, + ) + rm.metrics.RecordAPICall("GET", "UntagResource", err) + if err != nil { + return err + } + } + return nil +} + +// equalTags returns true if two Tag arrays are equal regardless of the order +// of their elements. +func equalTags( + a []*v1alpha1.Tag, + b []*v1alpha1.Tag, +) bool { + added, updated, removed := computeTagsDelta(a, b) + return len(added) == 0 && len(updated) == 0 && len(removed) == 0 +} + +// resourceTagsFromSDKTags transforms a *svcsdk.Tag array to a *v1alpha1.Tag array. +func resourceTagsFromSDKTags(svcTags []*svcsdk.Tag) []*v1alpha1.Tag { + tags := make([]*v1alpha1.Tag, len(svcTags)) + for i := range svcTags { + tags[i] = &v1alpha1.Tag{ + Key: svcTags[i].Key, + Value: svcTags[i].Value, + } + } + return tags +} + +// svcTagsFromResourceTags transforms a *v1alpha1.Tag array to a *svcsdk.Tag array. +func sdkTagsFromResourceTags(rTags []*v1alpha1.Tag) []*svcsdk.Tag { + tags := make([]*svcsdk.Tag, len(rTags)) + for i := range rTags { + tags[i] = &svcsdk.Tag{ + Key: rTags[i].Key, + Value: rTags[i].Value, + } + } + return tags +} + +// computeTagsDelta compares two Tag arrays and return three different list +// containing the added, updated and removed tags. +// The removed tags only contains the Key of tags +func computeTagsDelta( + a []*v1alpha1.Tag, + b []*v1alpha1.Tag, +) (added, updated []*v1alpha1.Tag, removed []*string) { + var visitedIndexes []string +mainLoop: + for _, aElement := range a { + visitedIndexes = append(visitedIndexes, *aElement.Key) + for _, bElement := range b { + if equalStrings(aElement.Key, bElement.Key) { + if !equalStrings(aElement.Value, bElement.Value) { + updated = append(updated, bElement) + } + continue mainLoop + } + } + removed = append(removed, aElement.Key) + } + for _, bElement := range b { + if !ackutil.InStrings(*bElement.Key, visitedIndexes) { + added = append(added, bElement) + } + } + return added, updated, removed +} + +func equalStrings(a, b *string) bool { + if a == nil { + return b == nil || *b == "" + } + return (*a == "" && b == nil) || *a == *b +} + +// setResourceAdditionalFields will describe the fields that are not return by +// DescribeTable calls +func (rm *resourceManager) setResourceAdditionalFields( + ctx context.Context, + ko *v1alpha1.Table, +) (err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.setResourceAdditionalFields") + defer exit(err) + + ko.Spec.Tags, err = rm.getResourceTagsPagesWithContext(ctx, string(*ko.Status.ACKResourceMetadata.ARN)) + if err != nil { + return err + } + + return nil +} + +// getResourceTagsPagesWithContext queries the list of tags of a given resource. +func (rm *resourceManager) getResourceTagsPagesWithContext(ctx context.Context, resourceARN string) ([]*v1alpha1.Tag, error) { + var err error + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.getResourceTagsPagesWithContext") + defer exit(err) + + tags := []*v1alpha1.Tag{} + + var token *string = nil + for { + var listTagsOfResourceOutput *svcsdk.ListTagsOfResourceOutput + listTagsOfResourceOutput, err = rm.sdkapi.ListTagsOfResourceWithContext( + ctx, + &svcsdk.ListTagsOfResourceInput{ + NextToken: token, + ResourceArn: &resourceARN, + }, + ) + rm.metrics.RecordAPICall("GET", "ListTagsOfResource", err) + if err != nil { + return nil, err + } + tags = append(tags, resourceTagsFromSDKTags(listTagsOfResourceOutput.Tags)...) + if listTagsOfResourceOutput.NextToken == nil { + break + } + token = listTagsOfResourceOutput.NextToken + } + return tags, nil +} + +func customPreCompare( + delta *ackcompare.Delta, + a *resource, + b *resource, +) { + // TODO(hilalymh): customDeltaFunctions for AttributeDefintions + // TODO(hilalymh): customDeltaFunctions for GlobalSecondaryIndexes + + if len(a.ko.Spec.Tags) != len(b.ko.Spec.Tags) { + delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags) + } else if a.ko.Spec.Tags != nil && b.ko.Spec.Tags != nil { + if !equalTags(a.ko.Spec.Tags, b.ko.Spec.Tags) { + delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags) + } + } +} diff --git a/pkg/resource/table/hooks_test.go b/pkg/resource/table/hooks_test.go new file mode 100644 index 0000000..4ed4815 --- /dev/null +++ b/pkg/resource/table/hooks_test.go @@ -0,0 +1,132 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package table + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/aws-controllers-k8s/dynamodb-controller/apis/v1alpha1" +) + +var ( + Tag1 = &v1alpha1.Tag{ + Key: aws.String("k1"), + Value: aws.String("v1"), + } + Tag2 = &v1alpha1.Tag{ + Key: aws.String("k2"), + Value: aws.String("v2"), + } + Tag2Updated = &v1alpha1.Tag{ + Key: aws.String("k2"), + Value: aws.String("v2-updated"), + } + Tag3 = &v1alpha1.Tag{ + Key: aws.String("k3"), + Value: aws.String("v3"), + } +) + +func Test_computeTagsDelta(t *testing.T) { + type args struct { + a []*v1alpha1.Tag + b []*v1alpha1.Tag + } + tests := []struct { + name string + args args + wantAdded []*v1alpha1.Tag + wantUpdated []*v1alpha1.Tag + wantRemoved []*string + }{ + { + name: "nil arrays", + args: args{ + a: nil, + b: nil, + }, + wantAdded: nil, + wantRemoved: nil, + wantUpdated: nil, + }, + { + name: "empty arrays", + args: args{ + a: []*v1alpha1.Tag{}, + b: []*v1alpha1.Tag{}, + }, + wantAdded: nil, + wantRemoved: nil, + wantUpdated: nil, + }, + { + name: "added tags", + args: args{ + a: []*v1alpha1.Tag{}, + b: []*v1alpha1.Tag{Tag1, Tag2}, + }, + wantAdded: []*v1alpha1.Tag{Tag1, Tag2}, + wantRemoved: nil, + wantUpdated: nil, + }, + { + name: "removed tags", + args: args{ + a: []*v1alpha1.Tag{Tag1, Tag2}, + b: nil, + }, + wantAdded: nil, + wantRemoved: []*string{aws.String("k1"), aws.String("k2")}, + wantUpdated: nil, + }, + { + name: "updated tags", + args: args{ + a: []*v1alpha1.Tag{Tag1, Tag2}, + b: []*v1alpha1.Tag{Tag1, Tag2Updated}, + }, + wantAdded: nil, + wantRemoved: nil, + wantUpdated: []*v1alpha1.Tag{Tag2Updated}, + }, + { + name: "added, updated and removed tags", + args: args{ + a: []*v1alpha1.Tag{Tag1, Tag2}, + // remove Tag1, update Tag2 and add Tag3 + b: []*v1alpha1.Tag{Tag2Updated, Tag3}, + }, + wantAdded: []*v1alpha1.Tag{Tag3}, + wantRemoved: []*string{aws.String("k1")}, + wantUpdated: []*v1alpha1.Tag{Tag2Updated}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotAdded, gotUpdated, gotRemoved := computeTagsDelta(tt.args.a, tt.args.b) + if !reflect.DeepEqual(gotAdded, tt.wantAdded) { + t.Errorf("computeTagsDelta() gotAdded = %v, want %v", gotAdded, tt.wantAdded) + } + if !reflect.DeepEqual(gotUpdated, tt.wantUpdated) { + t.Errorf("computeTagsDelta() gotUpdated = %v, want %v", gotUpdated, tt.wantUpdated) + } + if !reflect.DeepEqual(gotRemoved, tt.wantRemoved) { + t.Errorf("computeTagsDelta() gotRemoved = %v, want %v", gotRemoved, tt.wantRemoved) + } + }) + } +} diff --git a/pkg/resource/table/sdk.go b/pkg/resource/table/sdk.go index 7d55983..044166f 100644 --- a/pkg/resource/table/sdk.go +++ b/pkg/resource/table/sdk.go @@ -408,6 +408,9 @@ func (rm *resourceManager) sdkFind( if isTableUpdating(&resource{ko}) { return &resource{ko}, requeueWaitWhileUpdating } + if err := rm.setResourceAdditionalFields(ctx, ko); err != nil { + return nil, err + } return &resource{ko}, nil } @@ -978,435 +981,8 @@ func (rm *resourceManager) sdkUpdate( desired *resource, latest *resource, delta *ackcompare.Delta, -) (updated *resource, err error) { - rlog := ackrtlog.FromContext(ctx) - exit := rlog.Trace("rm.sdkUpdate") - defer exit(err) - if isTableDeleting(latest) { - msg := "table is currently being deleted" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileDeleting - } - if isTableCreating(latest) { - msg := "table is currently being created" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileCreating - } - if isTableUpdating(latest) { - msg := "table is currently being updated" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileUpdating - } - if tableHasTerminalStatus(latest) { - msg := "table is in '" + *latest.ko.Status.TableStatus + "' status" - setTerminalCondition(desired, corev1.ConditionTrue, &msg, nil) - setSyncedCondition(desired, corev1.ConditionTrue, nil, nil) - return desired, nil - } - input, err := rm.newUpdateRequestPayload(ctx, desired) - if err != nil { - return nil, err - } - - var resp *svcsdk.UpdateTableOutput - _ = resp - resp, err = rm.sdkapi.UpdateTableWithContext(ctx, input) - rm.metrics.RecordAPICall("UPDATE", "UpdateTable", err) - if err != nil { - return nil, err - } - // Merge in the information we read from the API call above to the copy of - // the original Kubernetes object we passed to the function - ko := desired.ko.DeepCopy() - - if resp.TableDescription.ArchivalSummary != nil { - f0 := &svcapitypes.ArchivalSummary{} - if resp.TableDescription.ArchivalSummary.ArchivalBackupArn != nil { - f0.ArchivalBackupARN = resp.TableDescription.ArchivalSummary.ArchivalBackupArn - } - if resp.TableDescription.ArchivalSummary.ArchivalDateTime != nil { - f0.ArchivalDateTime = &metav1.Time{*resp.TableDescription.ArchivalSummary.ArchivalDateTime} - } - if resp.TableDescription.ArchivalSummary.ArchivalReason != nil { - f0.ArchivalReason = resp.TableDescription.ArchivalSummary.ArchivalReason - } - ko.Status.ArchivalSummary = f0 - } else { - ko.Status.ArchivalSummary = nil - } - if resp.TableDescription.AttributeDefinitions != nil { - f1 := []*svcapitypes.AttributeDefinition{} - for _, f1iter := range resp.TableDescription.AttributeDefinitions { - f1elem := &svcapitypes.AttributeDefinition{} - if f1iter.AttributeName != nil { - f1elem.AttributeName = f1iter.AttributeName - } - if f1iter.AttributeType != nil { - f1elem.AttributeType = f1iter.AttributeType - } - f1 = append(f1, f1elem) - } - ko.Spec.AttributeDefinitions = f1 - } else { - ko.Spec.AttributeDefinitions = nil - } - if resp.TableDescription.BillingModeSummary != nil { - f2 := &svcapitypes.BillingModeSummary{} - if resp.TableDescription.BillingModeSummary.BillingMode != nil { - f2.BillingMode = resp.TableDescription.BillingModeSummary.BillingMode - } - if resp.TableDescription.BillingModeSummary.LastUpdateToPayPerRequestDateTime != nil { - f2.LastUpdateToPayPerRequestDateTime = &metav1.Time{*resp.TableDescription.BillingModeSummary.LastUpdateToPayPerRequestDateTime} - } - ko.Status.BillingModeSummary = f2 - } else { - ko.Status.BillingModeSummary = nil - } - if resp.TableDescription.CreationDateTime != nil { - ko.Status.CreationDateTime = &metav1.Time{*resp.TableDescription.CreationDateTime} - } else { - ko.Status.CreationDateTime = nil - } - if resp.TableDescription.GlobalSecondaryIndexes != nil { - f4 := []*svcapitypes.GlobalSecondaryIndex{} - for _, f4iter := range resp.TableDescription.GlobalSecondaryIndexes { - f4elem := &svcapitypes.GlobalSecondaryIndex{} - if f4iter.IndexName != nil { - f4elem.IndexName = f4iter.IndexName - } - if f4iter.KeySchema != nil { - f4elemf6 := []*svcapitypes.KeySchemaElement{} - for _, f4elemf6iter := range f4iter.KeySchema { - f4elemf6elem := &svcapitypes.KeySchemaElement{} - if f4elemf6iter.AttributeName != nil { - f4elemf6elem.AttributeName = f4elemf6iter.AttributeName - } - if f4elemf6iter.KeyType != nil { - f4elemf6elem.KeyType = f4elemf6iter.KeyType - } - f4elemf6 = append(f4elemf6, f4elemf6elem) - } - f4elem.KeySchema = f4elemf6 - } - if f4iter.Projection != nil { - f4elemf7 := &svcapitypes.Projection{} - if f4iter.Projection.NonKeyAttributes != nil { - f4elemf7f0 := []*string{} - for _, f4elemf7f0iter := range f4iter.Projection.NonKeyAttributes { - var f4elemf7f0elem string - f4elemf7f0elem = *f4elemf7f0iter - f4elemf7f0 = append(f4elemf7f0, &f4elemf7f0elem) - } - f4elemf7.NonKeyAttributes = f4elemf7f0 - } - if f4iter.Projection.ProjectionType != nil { - f4elemf7.ProjectionType = f4iter.Projection.ProjectionType - } - f4elem.Projection = f4elemf7 - } - if f4iter.ProvisionedThroughput != nil { - f4elemf8 := &svcapitypes.ProvisionedThroughput{} - if f4iter.ProvisionedThroughput.ReadCapacityUnits != nil { - f4elemf8.ReadCapacityUnits = f4iter.ProvisionedThroughput.ReadCapacityUnits - } - if f4iter.ProvisionedThroughput.WriteCapacityUnits != nil { - f4elemf8.WriteCapacityUnits = f4iter.ProvisionedThroughput.WriteCapacityUnits - } - f4elem.ProvisionedThroughput = f4elemf8 - } - f4 = append(f4, f4elem) - } - ko.Spec.GlobalSecondaryIndexes = f4 - } else { - ko.Spec.GlobalSecondaryIndexes = nil - } - if resp.TableDescription.GlobalTableVersion != nil { - ko.Status.GlobalTableVersion = resp.TableDescription.GlobalTableVersion - } else { - ko.Status.GlobalTableVersion = nil - } - if resp.TableDescription.ItemCount != nil { - ko.Status.ItemCount = resp.TableDescription.ItemCount - } else { - ko.Status.ItemCount = nil - } - if resp.TableDescription.KeySchema != nil { - f7 := []*svcapitypes.KeySchemaElement{} - for _, f7iter := range resp.TableDescription.KeySchema { - f7elem := &svcapitypes.KeySchemaElement{} - if f7iter.AttributeName != nil { - f7elem.AttributeName = f7iter.AttributeName - } - if f7iter.KeyType != nil { - f7elem.KeyType = f7iter.KeyType - } - f7 = append(f7, f7elem) - } - ko.Spec.KeySchema = f7 - } else { - ko.Spec.KeySchema = nil - } - if resp.TableDescription.LatestStreamArn != nil { - ko.Status.LatestStreamARN = resp.TableDescription.LatestStreamArn - } else { - ko.Status.LatestStreamARN = nil - } - if resp.TableDescription.LatestStreamLabel != nil { - ko.Status.LatestStreamLabel = resp.TableDescription.LatestStreamLabel - } else { - ko.Status.LatestStreamLabel = nil - } - if resp.TableDescription.LocalSecondaryIndexes != nil { - f10 := []*svcapitypes.LocalSecondaryIndex{} - for _, f10iter := range resp.TableDescription.LocalSecondaryIndexes { - f10elem := &svcapitypes.LocalSecondaryIndex{} - if f10iter.IndexName != nil { - f10elem.IndexName = f10iter.IndexName - } - if f10iter.KeySchema != nil { - f10elemf4 := []*svcapitypes.KeySchemaElement{} - for _, f10elemf4iter := range f10iter.KeySchema { - f10elemf4elem := &svcapitypes.KeySchemaElement{} - if f10elemf4iter.AttributeName != nil { - f10elemf4elem.AttributeName = f10elemf4iter.AttributeName - } - if f10elemf4iter.KeyType != nil { - f10elemf4elem.KeyType = f10elemf4iter.KeyType - } - f10elemf4 = append(f10elemf4, f10elemf4elem) - } - f10elem.KeySchema = f10elemf4 - } - if f10iter.Projection != nil { - f10elemf5 := &svcapitypes.Projection{} - if f10iter.Projection.NonKeyAttributes != nil { - f10elemf5f0 := []*string{} - for _, f10elemf5f0iter := range f10iter.Projection.NonKeyAttributes { - var f10elemf5f0elem string - f10elemf5f0elem = *f10elemf5f0iter - f10elemf5f0 = append(f10elemf5f0, &f10elemf5f0elem) - } - f10elemf5.NonKeyAttributes = f10elemf5f0 - } - if f10iter.Projection.ProjectionType != nil { - f10elemf5.ProjectionType = f10iter.Projection.ProjectionType - } - f10elem.Projection = f10elemf5 - } - f10 = append(f10, f10elem) - } - ko.Spec.LocalSecondaryIndexes = f10 - } else { - ko.Spec.LocalSecondaryIndexes = nil - } - if resp.TableDescription.ProvisionedThroughput != nil { - f11 := &svcapitypes.ProvisionedThroughput{} - if resp.TableDescription.ProvisionedThroughput.ReadCapacityUnits != nil { - f11.ReadCapacityUnits = resp.TableDescription.ProvisionedThroughput.ReadCapacityUnits - } - if resp.TableDescription.ProvisionedThroughput.WriteCapacityUnits != nil { - f11.WriteCapacityUnits = resp.TableDescription.ProvisionedThroughput.WriteCapacityUnits - } - ko.Spec.ProvisionedThroughput = f11 - } else { - ko.Spec.ProvisionedThroughput = nil - } - if resp.TableDescription.Replicas != nil { - f12 := []*svcapitypes.ReplicaDescription{} - for _, f12iter := range resp.TableDescription.Replicas { - f12elem := &svcapitypes.ReplicaDescription{} - if f12iter.GlobalSecondaryIndexes != nil { - f12elemf0 := []*svcapitypes.ReplicaGlobalSecondaryIndexDescription{} - for _, f12elemf0iter := range f12iter.GlobalSecondaryIndexes { - f12elemf0elem := &svcapitypes.ReplicaGlobalSecondaryIndexDescription{} - if f12elemf0iter.IndexName != nil { - f12elemf0elem.IndexName = f12elemf0iter.IndexName - } - if f12elemf0iter.ProvisionedThroughputOverride != nil { - f12elemf0elemf1 := &svcapitypes.ProvisionedThroughputOverride{} - if f12elemf0iter.ProvisionedThroughputOverride.ReadCapacityUnits != nil { - f12elemf0elemf1.ReadCapacityUnits = f12elemf0iter.ProvisionedThroughputOverride.ReadCapacityUnits - } - f12elemf0elem.ProvisionedThroughputOverride = f12elemf0elemf1 - } - f12elemf0 = append(f12elemf0, f12elemf0elem) - } - f12elem.GlobalSecondaryIndexes = f12elemf0 - } - if f12iter.KMSMasterKeyId != nil { - f12elem.KMSMasterKeyID = f12iter.KMSMasterKeyId - } - if f12iter.ProvisionedThroughputOverride != nil { - f12elemf2 := &svcapitypes.ProvisionedThroughputOverride{} - if f12iter.ProvisionedThroughputOverride.ReadCapacityUnits != nil { - f12elemf2.ReadCapacityUnits = f12iter.ProvisionedThroughputOverride.ReadCapacityUnits - } - f12elem.ProvisionedThroughputOverride = f12elemf2 - } - if f12iter.RegionName != nil { - f12elem.RegionName = f12iter.RegionName - } - if f12iter.ReplicaInaccessibleDateTime != nil { - f12elem.ReplicaInaccessibleDateTime = &metav1.Time{*f12iter.ReplicaInaccessibleDateTime} - } - if f12iter.ReplicaStatus != nil { - f12elem.ReplicaStatus = f12iter.ReplicaStatus - } - if f12iter.ReplicaStatusDescription != nil { - f12elem.ReplicaStatusDescription = f12iter.ReplicaStatusDescription - } - if f12iter.ReplicaStatusPercentProgress != nil { - f12elem.ReplicaStatusPercentProgress = f12iter.ReplicaStatusPercentProgress - } - f12 = append(f12, f12elem) - } - ko.Status.Replicas = f12 - } else { - ko.Status.Replicas = nil - } - if resp.TableDescription.RestoreSummary != nil { - f13 := &svcapitypes.RestoreSummary{} - if resp.TableDescription.RestoreSummary.RestoreDateTime != nil { - f13.RestoreDateTime = &metav1.Time{*resp.TableDescription.RestoreSummary.RestoreDateTime} - } - if resp.TableDescription.RestoreSummary.RestoreInProgress != nil { - f13.RestoreInProgress = resp.TableDescription.RestoreSummary.RestoreInProgress - } - if resp.TableDescription.RestoreSummary.SourceBackupArn != nil { - f13.SourceBackupARN = resp.TableDescription.RestoreSummary.SourceBackupArn - } - if resp.TableDescription.RestoreSummary.SourceTableArn != nil { - f13.SourceTableARN = resp.TableDescription.RestoreSummary.SourceTableArn - } - ko.Status.RestoreSummary = f13 - } else { - ko.Status.RestoreSummary = nil - } - if resp.TableDescription.SSEDescription != nil { - f14 := &svcapitypes.SSEDescription{} - if resp.TableDescription.SSEDescription.InaccessibleEncryptionDateTime != nil { - f14.InaccessibleEncryptionDateTime = &metav1.Time{*resp.TableDescription.SSEDescription.InaccessibleEncryptionDateTime} - } - if resp.TableDescription.SSEDescription.KMSMasterKeyArn != nil { - f14.KMSMasterKeyARN = resp.TableDescription.SSEDescription.KMSMasterKeyArn - } - if resp.TableDescription.SSEDescription.SSEType != nil { - f14.SSEType = resp.TableDescription.SSEDescription.SSEType - } - if resp.TableDescription.SSEDescription.Status != nil { - f14.Status = resp.TableDescription.SSEDescription.Status - } - ko.Status.SSEDescription = f14 - } else { - ko.Status.SSEDescription = nil - } - if resp.TableDescription.StreamSpecification != nil { - f15 := &svcapitypes.StreamSpecification{} - if resp.TableDescription.StreamSpecification.StreamEnabled != nil { - f15.StreamEnabled = resp.TableDescription.StreamSpecification.StreamEnabled - } - if resp.TableDescription.StreamSpecification.StreamViewType != nil { - f15.StreamViewType = resp.TableDescription.StreamSpecification.StreamViewType - } - ko.Spec.StreamSpecification = f15 - } else { - ko.Spec.StreamSpecification = nil - } - if ko.Status.ACKResourceMetadata == nil { - ko.Status.ACKResourceMetadata = &ackv1alpha1.ResourceMetadata{} - } - if resp.TableDescription.TableArn != nil { - arn := ackv1alpha1.AWSResourceName(*resp.TableDescription.TableArn) - ko.Status.ACKResourceMetadata.ARN = &arn - } - if resp.TableDescription.TableId != nil { - ko.Status.TableID = resp.TableDescription.TableId - } else { - ko.Status.TableID = nil - } - if resp.TableDescription.TableName != nil { - ko.Spec.TableName = resp.TableDescription.TableName - } else { - ko.Spec.TableName = nil - } - if resp.TableDescription.TableSizeBytes != nil { - ko.Status.TableSizeBytes = resp.TableDescription.TableSizeBytes - } else { - ko.Status.TableSizeBytes = nil - } - if resp.TableDescription.TableStatus != nil { - ko.Status.TableStatus = resp.TableDescription.TableStatus - } else { - ko.Status.TableStatus = nil - } - - rm.setStatusDefaults(ko) - return &resource{ko}, nil -} - -// newUpdateRequestPayload returns an SDK-specific struct for the HTTP request -// payload of the Update API call for the resource -func (rm *resourceManager) newUpdateRequestPayload( - ctx context.Context, - r *resource, -) (*svcsdk.UpdateTableInput, error) { - res := &svcsdk.UpdateTableInput{} - - if r.ko.Spec.AttributeDefinitions != nil { - f0 := []*svcsdk.AttributeDefinition{} - for _, f0iter := range r.ko.Spec.AttributeDefinitions { - f0elem := &svcsdk.AttributeDefinition{} - if f0iter.AttributeName != nil { - f0elem.SetAttributeName(*f0iter.AttributeName) - } - if f0iter.AttributeType != nil { - f0elem.SetAttributeType(*f0iter.AttributeType) - } - f0 = append(f0, f0elem) - } - res.SetAttributeDefinitions(f0) - } - if r.ko.Spec.BillingMode != nil { - res.SetBillingMode(*r.ko.Spec.BillingMode) - } - if r.ko.Spec.ProvisionedThroughput != nil { - f3 := &svcsdk.ProvisionedThroughput{} - if r.ko.Spec.ProvisionedThroughput.ReadCapacityUnits != nil { - f3.SetReadCapacityUnits(*r.ko.Spec.ProvisionedThroughput.ReadCapacityUnits) - } - if r.ko.Spec.ProvisionedThroughput.WriteCapacityUnits != nil { - f3.SetWriteCapacityUnits(*r.ko.Spec.ProvisionedThroughput.WriteCapacityUnits) - } - res.SetProvisionedThroughput(f3) - } - if r.ko.Spec.SSESpecification != nil { - f5 := &svcsdk.SSESpecification{} - if r.ko.Spec.SSESpecification.Enabled != nil { - f5.SetEnabled(*r.ko.Spec.SSESpecification.Enabled) - } - if r.ko.Spec.SSESpecification.KMSMasterKeyID != nil { - f5.SetKMSMasterKeyId(*r.ko.Spec.SSESpecification.KMSMasterKeyID) - } - if r.ko.Spec.SSESpecification.SSEType != nil { - f5.SetSSEType(*r.ko.Spec.SSESpecification.SSEType) - } - res.SetSSESpecification(f5) - } - if r.ko.Spec.StreamSpecification != nil { - f6 := &svcsdk.StreamSpecification{} - if r.ko.Spec.StreamSpecification.StreamEnabled != nil { - f6.SetStreamEnabled(*r.ko.Spec.StreamSpecification.StreamEnabled) - } - if r.ko.Spec.StreamSpecification.StreamViewType != nil { - f6.SetStreamViewType(*r.ko.Spec.StreamSpecification.StreamViewType) - } - res.SetStreamSpecification(f6) - } - if r.ko.Spec.TableName != nil { - res.SetTableName(*r.ko.Spec.TableName) - } - - return res, nil +) (*resource, error) { + return rm.customUpdateTable(ctx, desired, latest, delta) } // sdkDelete deletes the supplied resource in the backend AWS service API diff --git a/templates/hooks/table/sdk_read_one_post_set_output.go.tpl b/templates/hooks/table/sdk_read_one_post_set_output.go.tpl index 2189a8d..e1f8d57 100644 --- a/templates/hooks/table/sdk_read_one_post_set_output.go.tpl +++ b/templates/hooks/table/sdk_read_one_post_set_output.go.tpl @@ -3,4 +3,7 @@ } if isTableUpdating(&resource{ko}) { return &resource{ko}, requeueWaitWhileUpdating + } + if err := rm.setResourceAdditionalFields(ctx, ko); err != nil { + return nil, err } \ No newline at end of file diff --git a/templates/hooks/table/sdk_update_pre_build_request.go.tpl b/templates/hooks/table/sdk_update_pre_build_request.go.tpl deleted file mode 100644 index 5374e3a..0000000 --- a/templates/hooks/table/sdk_update_pre_build_request.go.tpl +++ /dev/null @@ -1,21 +0,0 @@ - if isTableDeleting(latest) { - msg := "table is currently being deleted" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileDeleting - } - if isTableCreating(latest) { - msg := "table is currently being created" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileCreating - } - if isTableUpdating(latest) { - msg := "table is currently being updated" - setSyncedCondition(desired, corev1.ConditionFalse, &msg, nil) - return desired, requeueWaitWhileUpdating - } - if tableHasTerminalStatus(latest) { - msg := "table is in '"+*latest.ko.Status.TableStatus+"' status" - setTerminalCondition(desired, corev1.ConditionTrue, &msg, nil) - setSyncedCondition(desired, corev1.ConditionTrue, nil, nil) - return desired, nil - } \ No newline at end of file diff --git a/test/e2e/__init__.py b/test/e2e/__init__.py index 6c75319..997c30c 100644 --- a/test/e2e/__init__.py +++ b/test/e2e/__init__.py @@ -20,6 +20,7 @@ from acktest.k8s import resource as k8s from acktest.resources import load_resource_file +from acktest.aws.identity import get_region SERVICE_NAME = "dynamodb" CRD_GROUP = "dynamodb.services.k8s.aws" @@ -60,4 +61,21 @@ def wait_for_cr_status( f"Wait for status: {desired_status} timed out. Actual status: {actual_status}" ) - assert actual_status == desired_status \ No newline at end of file + assert actual_status == desired_status + +def get_resource_tags(resource_arn: str): + region = get_region() + ddb_client = boto3.client('dynamodb', region_name=region) + tags = [] + next_token = "" + while True: + resp = ddb_client.list_tags_of_resource( + ResourceArn=resource_arn, + NextToken=next_token, + ) + tags += resp['Tags'] + if not 'NextToken' in resp.keys(): + break + next_token = resp['NextToken'] + + return tags \ No newline at end of file diff --git a/test/e2e/resources/table_forums.yaml b/test/e2e/resources/table_forums.yaml index f15e1b3..49e6438 100644 --- a/test/e2e/resources/table_forums.yaml +++ b/test/e2e/resources/table_forums.yaml @@ -7,10 +7,10 @@ spec: attributeDefinitions: - attributeName: ForumName attributeType: S - - attributeName: Subject - attributeType: S - attributeName: LastPostDateTime attributeType: S + - attributeName: Subject + attributeType: S keySchema: - attributeName: ForumName keyType: HASH diff --git a/test/e2e/tests/test_table.py b/test/e2e/tests/test_table.py index 37a7f24..7deb744 100644 --- a/test/e2e/tests/test_table.py +++ b/test/e2e/tests/test_table.py @@ -22,12 +22,17 @@ from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s -from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_dynamodb_resource, wait_for_cr_status +from e2e import ( + service_marker, CRD_GROUP, CRD_VERSION, + load_dynamodb_resource, wait_for_cr_status, + get_resource_tags, +) from e2e.replacement_values import REPLACEMENT_VALUES RESOURCE_PLURAL = "tables" DELETE_WAIT_AFTER_SECONDS = 15 +UPDATE_TAGS_WAIT_AFTER_SECONDS = 5 @pytest.fixture(scope="module") def dynamodb_client(): @@ -50,7 +55,7 @@ def get_table(self, dynamodb_client, table_name: str) -> dict: def table_exists(self, dynamodb_client, table_name: str) -> bool: return self.get_table(dynamodb_client, table_name) is not None - def test_smoke(self, dynamodb_client): + def test_create_delete(self, dynamodb_client): resource_name = random_suffix_name("table", 32) replacements = REPLACEMENT_VALUES.copy() @@ -92,6 +97,73 @@ def test_smoke(self, dynamodb_client): time.sleep(DELETE_WAIT_AFTER_SECONDS) + # Check DynamoDB Table doesn't exists + exists = self.table_exists(dynamodb_client, resource_name) + assert not exists + + def test_table_update_tags(self, dynamodb_client): + resource_name = random_suffix_name("table", 32) + + replacements = REPLACEMENT_VALUES.copy() + replacements["TABLE_NAME"] = resource_name + + # Load Table CR + resource_data = load_dynamodb_resource( + "table_forums", + additional_replacements=replacements, + ) + logging.debug(resource_data) + + # Create k8s resource + ref = k8s.CustomResourceReference( + CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL, + resource_name, namespace="default", + ) + k8s.create_custom_resource(ref, resource_data) + cr = k8s.wait_resource_consumed_by_controller(ref) + + assert cr is not None + assert k8s.get_resource_exists(ref) + + wait_for_cr_status( + ref, + "tableStatus", + "ACTIVE", + 10, + 5, + ) + + # Check DynamoDB Table exists + exists = self.table_exists(dynamodb_client, resource_name) + assert exists + + # Get CR latest revision + cr = k8s.wait_resource_consumed_by_controller(ref) + + # Update table list of tags + tags = [ + { + "key": "key1", + "value": "value1", + }, + ] + cr["spec"]["tags"] = tags + + # Patch k8s resource + k8s.patch_custom_resource(ref, cr) + time.sleep(UPDATE_TAGS_WAIT_AFTER_SECONDS) + + table_tags = get_resource_tags(cr["status"]["ackResourceMetadata"]["arn"]) + assert len(table_tags) == len(tags) + assert table_tags[0]['Key'] == tags[0]['key'] + assert table_tags[0]['Value'] == tags[0]['value'] + + # Delete k8s resource + _, deleted = k8s.delete_custom_resource(ref) + assert deleted is True + + time.sleep(DELETE_WAIT_AFTER_SECONDS) + # Check DynamoDB Table doesn't exists exists = self.table_exists(dynamodb_client, resource_name) assert not exists \ No newline at end of file