diff --git a/apis/v1alpha1/ack-generate-metadata.yaml b/apis/v1alpha1/ack-generate-metadata.yaml index 0a084e78..70a48ec9 100755 --- a/apis/v1alpha1/ack-generate-metadata.yaml +++ b/apis/v1alpha1/ack-generate-metadata.yaml @@ -1,5 +1,5 @@ ack_generate_info: - build_date: "2022-09-21T13:36:34Z" + build_date: "2022-09-27T22:29:46Z" build_hash: 6e2ffbc3b16a30ac59be6719918c601c2c864064 go_version: go1.18.1 version: v0.20.1-3-g6e2ffbc @@ -7,7 +7,7 @@ api_directory_checksum: 127aa0f51fbcdde596b8f42f93bcdf3b6dee8488 api_version: v1alpha1 aws_sdk_go_version: v1.44.93 generator_config_info: - file_checksum: 8573a89b8220023263610579bfb84b762d99f5fa + file_checksum: a31177552ce6ea5ce3b624520711382f113cf05b original_file_name: generator.yaml last_modification: reason: API generation diff --git a/apis/v1alpha1/generator.yaml b/apis/v1alpha1/generator.yaml index 3ba9bdd2..4d1f99eb 100644 --- a/apis/v1alpha1/generator.yaml +++ b/apis/v1alpha1/generator.yaml @@ -205,6 +205,10 @@ resources: hooks: sdk_create_post_build_request: template_path: hooks/instance/sdk_create_post_build_request.go.tpl + sdk_create_post_set_output: + template_path: hooks/instance/sdk_create_post_set_output.go.tpl + sdk_read_many_post_set_output: + template_path: hooks/instance/sdk_read_many_post_set_output.go.tpl sdk_delete_post_build_request: template_path: hooks/instance/sdk_delete_post_build_request.go.tpl sdk_file_end: diff --git a/generator.yaml b/generator.yaml index 3ba9bdd2..4d1f99eb 100644 --- a/generator.yaml +++ b/generator.yaml @@ -205,6 +205,10 @@ resources: hooks: sdk_create_post_build_request: template_path: hooks/instance/sdk_create_post_build_request.go.tpl + sdk_create_post_set_output: + template_path: hooks/instance/sdk_create_post_set_output.go.tpl + sdk_read_many_post_set_output: + template_path: hooks/instance/sdk_read_many_post_set_output.go.tpl sdk_delete_post_build_request: template_path: hooks/instance/sdk_delete_post_build_request.go.tpl sdk_file_end: diff --git a/pkg/resource/instance/sdk.go b/pkg/resource/instance/sdk.go index ae38d042..b70677a1 100644 --- a/pkg/resource/instance/sdk.go +++ b/pkg/resource/instance/sdk.go @@ -629,6 +629,14 @@ func (rm *resourceManager) sdkFind( } rm.setStatusDefaults(ko) + + toAdd, toDelete := computeTagsDelta(r.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if resource's initial tags and response tags are equal, + // then assign resource's tags to maintain tag order + ko.Spec.Tags = r.ko.Spec.Tags + } + return &resource{ko}, nil } @@ -1227,6 +1235,12 @@ func (rm *resourceManager) sdkCreate( } rm.setStatusDefaults(ko) + toAdd, toDelete := computeTagsDelta(desired.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if desired tags and response tags are equal, + // then assign desired tags to maintain tag order + ko.Spec.Tags = desired.ko.Spec.Tags + } return &resource{ko}, nil } diff --git a/pkg/resource/route_table/sdk.go b/pkg/resource/route_table/sdk.go index c91a52da..37b6dda4 100644 --- a/pkg/resource/route_table/sdk.go +++ b/pkg/resource/route_table/sdk.go @@ -228,6 +228,12 @@ func (rm *resourceManager) sdkFind( if found { rm.addRoutesToStatus(ko, resp.RouteTables[0]) } + toAdd, toDelete := computeTagsDelta(r.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if resource's initial tags and response tags are equal, + // then assign resource's tags to maintain tag order + ko.Spec.Tags = r.ko.Spec.Tags + } return &resource{ko}, nil } @@ -430,6 +436,13 @@ func (rm *resourceManager) sdkCreate( return nil, err } } + + toAdd, toDelete := computeTagsDelta(desired.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if desired tags and response tags are equal, + // then assign desired tags to maintain tag order + ko.Spec.Tags = desired.ko.Spec.Tags + } return &resource{ko}, nil } diff --git a/templates/hooks/instance/sdk_create_post_set_output.go.tpl b/templates/hooks/instance/sdk_create_post_set_output.go.tpl new file mode 100644 index 00000000..f64ad6b2 --- /dev/null +++ b/templates/hooks/instance/sdk_create_post_set_output.go.tpl @@ -0,0 +1,6 @@ + toAdd, toDelete := computeTagsDelta(desired.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if desired tags and response tags are equal, + // then assign desired tags to maintain tag order + ko.Spec.Tags = desired.ko.Spec.Tags + } \ No newline at end of file diff --git a/templates/hooks/instance/sdk_read_many_post_set_output.go.tpl b/templates/hooks/instance/sdk_read_many_post_set_output.go.tpl new file mode 100644 index 00000000..8ae2ef98 --- /dev/null +++ b/templates/hooks/instance/sdk_read_many_post_set_output.go.tpl @@ -0,0 +1,8 @@ + + toAdd, toDelete := computeTagsDelta(r.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if resource's initial tags and response tags are equal, + // then assign resource's tags to maintain tag order + ko.Spec.Tags = r.ko.Spec.Tags + } + diff --git a/templates/hooks/route_table/sdk_create_post_set_output.go.tpl b/templates/hooks/route_table/sdk_create_post_set_output.go.tpl index 94d6d632..96b41236 100644 --- a/templates/hooks/route_table/sdk_create_post_set_output.go.tpl +++ b/templates/hooks/route_table/sdk_create_post_set_output.go.tpl @@ -10,4 +10,11 @@ if err := rm.createRoutes(ctx, &resource{ko}); err != nil { return nil, err } + } + + toAdd, toDelete := computeTagsDelta(desired.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if desired tags and response tags are equal, + // then assign desired tags to maintain tag order + ko.Spec.Tags = desired.ko.Spec.Tags } \ No newline at end of file diff --git a/templates/hooks/route_table/sdk_read_many_post_set_output.go.tpl b/templates/hooks/route_table/sdk_read_many_post_set_output.go.tpl index 4a80fab8..d669b0f3 100644 --- a/templates/hooks/route_table/sdk_read_many_post_set_output.go.tpl +++ b/templates/hooks/route_table/sdk_read_many_post_set_output.go.tpl @@ -2,4 +2,10 @@ if found { rm.addRoutesToStatus(ko, resp.RouteTables[0]) } + toAdd, toDelete := computeTagsDelta(r.ko.Spec.Tags, ko.Spec.Tags) + if len(toAdd) == 0 && len(toDelete) == 0 { + // if resource's initial tags and response tags are equal, + // then assign resource's tags to maintain tag order + ko.Spec.Tags = r.ko.Spec.Tags + } diff --git a/test/e2e/requirements.txt b/test/e2e/requirements.txt index 9b734ba5..3d1b5d5c 100644 --- a/test/e2e/requirements.txt +++ b/test/e2e/requirements.txt @@ -1 +1 @@ -acktest @ git+https://github.com/aws-controllers-k8s/test-infra.git@5220331d003e3a23de4470e68d02c99b16c81989 \ No newline at end of file +acktest @ git+https://github.com/aws-controllers-k8s/test-infra.git@73eb0d5af8ac19d17fad8787d6462d56f4e3e37d \ No newline at end of file diff --git a/test/e2e/tests/helper.py b/test/e2e/tests/helper.py index 7468fa7a..788d1d59 100644 --- a/test/e2e/tests/helper.py +++ b/test/e2e/tests/helper.py @@ -21,6 +21,15 @@ class EC2Validator: def __init__(self, ec2_client): self.ec2_client = ec2_client + def get_dhcp_options(self, dhcp_options_id: str): + try: + aws_res = self.ec2_client.describe_dhcp_options(DhcpOptionsIds=[dhcp_options_id]) + if len(aws_res["DhcpOptions"]) > 0: + return aws_res["DhcpOptions"][0] + return None + except self.ec2_client.exceptions.ClientError: + return None + def assert_dhcp_options(self, dhcp_options_id: str, exists=True): res_found = False try: @@ -42,6 +51,15 @@ def get_internet_gateway(self, igw_id: str): def assert_internet_gateway(self, igw_id: str, exists=True): assert (self.get_internet_gateway(igw_id) is not None) == exists + def get_nat_gateway(self, ngw_id: str): + try: + aws_res = self.ec2_client.describe_nat_gateways(NatGatewayIds=[ngw_id]) + if len(aws_res["NatGateways"]) > 0: + return aws_res["NatGateways"][0] + return None + except self.ec2_client.exceptions.ClientError: + return None + def assert_nat_gateway(self, ngw_id: str, exists=True): res_found = False try: @@ -116,6 +134,15 @@ def get_subnet(self, subnet_id: str) -> Union[None, Dict]: def assert_subnet(self, subnet_id: str, exists=True): assert (self.get_subnet(subnet_id) is not None) == exists + def get_transit_gateway(self, tgw_id: str) -> Union[None, Dict]: + try: + aws_res = self.ec2_client.describe_transit_gateways(TransitGatewayIds=[tgw_id]) + if len(aws_res["TransitGateways"]) > 0: + return aws_res["TransitGateways"][0] + return None + except self.ec2_client.exceptions.ClientError: + return None + def assert_transit_gateway(self, tgw_id: str, exists=True): res_found = False try: @@ -146,6 +173,15 @@ def assert_vpc(self, vpc_id: str, exists=True): pass assert res_found is exists + def get_vpc_endpoint(self, vpc_endpoint_id: str) -> Union[None, Dict]: + try: + aws_res = self.ec2_client.describe_vpc_endpoints(VpcEndpointIds=[vpc_endpoint_id]) + if len(aws_res["VpcEndpoints"]) > 0: + return aws_res["VpcEndpoints"][0] + return None + except self.ec2_client.exceptions.ClientError: + return None + def assert_vpc_endpoint(self, vpc_endpoint_id: str, exists=True): res_found = False try: diff --git a/test/e2e/tests/test_dhcp_options.py b/test/e2e/tests/test_dhcp_options.py index 92fd28cb..f34208c3 100755 --- a/test/e2e/tests/test_dhcp_options.py +++ b/test/e2e/tests/test_dhcp_options.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -131,23 +132,36 @@ def test_crud_tags(self, ec2_client, simple_dhcp_options): # Check dhcpOptions exists in AWS ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_dhcp_options(resource_id) + + # Check system and user tags exist for dhcpOptions resource + dhcp_options = ec2_validator.get_dhcp_options(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=dhcp_options["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=dhcp_options["Tags"], + ) - # Check tags exist for dhcpOptions resource + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the dhcpOptions, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -156,8 +170,22 @@ def test_crud_tags(self, ec2_client, simple_dhcp_options): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for dhcpOptions resource + # Check for updated user tags; system tags should persist + dhcp_options = ec2_validator.get_dhcp_options(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=dhcp_options["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=dhcp_options["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -172,9 +200,19 @@ def test_crud_tags(self, ec2_client, simple_dhcp_options): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + dhcp_options = ec2_validator.get_dhcp_options(resource_id) + tags.assert_ack_system_tags( + tags=dhcp_options["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=dhcp_options["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_elastic_ip_address.py b/test/e2e/tests/test_elastic_ip_address.py index 82288395..2f842925 100644 --- a/test/e2e/tests/test_elastic_ip_address.py +++ b/test/e2e/tests/test_elastic_ip_address.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -153,13 +154,27 @@ def test_crud_tags(self, ec2_client, simple_elastic_ip_address): # Check Address exists exists = address_exists(ec2_client, resource_id) assert exists - - # Check tags exist for elasticipaddress resource + + # Check system and user tags exist for elasticIP resource + elastic_ip = get_address(ec2_client, resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=elastic_ip["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=elastic_ip["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", @@ -169,7 +184,7 @@ def test_crud_tags(self, ec2_client, simple_elastic_ip_address): # Patch the elasticipaddress, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -177,9 +192,23 @@ def test_crud_tags(self, ec2_client, simple_elastic_ip_address): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - - # Assert tags are updated for elasticipaddress resource + + # Check for updated user tags; system tags should persist + elastic_ip = get_address(ec2_client, resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=elastic_ip["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=elastic_ip["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -194,9 +223,19 @@ def test_crud_tags(self, ec2_client, simple_elastic_ip_address): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + elastic_ip = get_address(ec2_client, resource_id) + tags.assert_ack_system_tags( + tags=elastic_ip["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=elastic_ip["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_instance.py b/test/e2e/tests/test_instance.py index 0d44f16c..0998c6e1 100644 --- a/test/e2e/tests/test_instance.py +++ b/test/e2e/tests/test_instance.py @@ -19,6 +19,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -88,17 +89,6 @@ def get_ami_id(ec2_client): except Exception as e: logging.debug(e) -def contains_tag(resource, tag): - try: - tag_key, tag_val = tag.popitem() - for t in resource["spec"]["tags"]: - if t["key"] == tag_key and t["value"] == tag_val: - return True - except: - pass - - return False - @pytest.fixture def instance(ec2_client): @@ -188,22 +178,34 @@ def test_crud_tags(self, ec2_client, instance): instance = get_instance(ec2_client, resource_id) assert instance is not None - # Check tags exist for Instance resource - assert contains_tag(resource, {INSTANCE_TAG_KEY: INSTANCE_TAG_VAL}) - - # Fetch all tags including ACK tags applied by default - new_tags = resource['spec']['tags'] - - # New pair of tags - new_tag = { + # Check system and user tags exist for instance resource + user_tags = { + INSTANCE_TAG_KEY: INSTANCE_TAG_VAL + } + tags.assert_ack_system_tags( + tags=instance["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=instance["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 + assert resource["spec"]["tags"][0]["key"] == INSTANCE_TAG_KEY + assert resource["spec"]["tags"][0]["value"] == INSTANCE_TAG_VAL + + # Update tags + update_tags = [ + { "key": "updatedtagkey", "value": "updatedtagvalue", - } - new_tags.append(new_tag) + } + ] # Patch the Instance, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -212,14 +214,28 @@ def test_crud_tags(self, ec2_client, instance): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for Instance resource + # Check for updated user tags; system tags should persist + instance = get_instance(ec2_client, resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=instance["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=instance["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) - assert contains_tag(resource, {"updatedtagkey": "updatedtagvalue"}) + assert len(resource["spec"]["tags"]) == 1 + assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" + assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" - new_tags = [] # Patch the Instance resource, deleting the tags updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": []}, } k8s.patch_custom_resource(ref, updates) @@ -228,9 +244,19 @@ def test_crud_tags(self, ec2_client, instance): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + instance = get_instance(ec2_client, resource_id) + tags.assert_ack_system_tags( + tags=instance["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=instance["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert contains_tag(resource, {"updatedtagkey": "updatedtagvalue"}) is False + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_internet_gateway.py b/test/e2e/tests/test_internet_gateway.py index 40fca9f4..2af5dc34 100644 --- a/test/e2e/tests/test_internet_gateway.py +++ b/test/e2e/tests/test_internet_gateway.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -160,22 +161,35 @@ def test_crud_tags(self, ec2_client, simple_internet_gateway): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_internet_gateway(resource_id) - # Check tags exist for IGW resource + # Check system and user tags exist for igw resource + internet_gateway = ec2_validator.get_internet_gateway(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=internet_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=internet_gateway["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the IGW, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -184,8 +198,22 @@ def test_crud_tags(self, ec2_client, simple_internet_gateway): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for IGW resource + # Check for updated user tags; system tags should persist + internet_gateway = ec2_validator.get_internet_gateway(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=internet_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=internet_gateway["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -199,10 +227,20 @@ def test_crud_tags(self, ec2_client, simple_internet_gateway): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) + + # Check for removed user tags; system tags should persist + internet_gateway = ec2_validator.get_internet_gateway(resource_id) + tags.assert_ack_system_tags( + tags=internet_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=internet_gateway["Tags"], + ) - # Assert tags are deleted + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_nat_gateway.py b/test/e2e/tests/test_nat_gateway.py index 08b8178a..68ba9407 100644 --- a/test/e2e/tests/test_nat_gateway.py +++ b/test/e2e/tests/test_nat_gateway.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -156,29 +157,56 @@ def test_crud_tags(self, ec2_client, simple_nat_gateway): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_nat_gateway(resource_id) - # Check tags exist for natGateway resource + # Check system and user tags exist for natGateway resource + nat_gateway = ec2_validator.get_nat_gateway(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=nat_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=nat_gateway["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the natGateway, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) time.sleep(MODIFY_WAIT_AFTER_SECONDS) - # Assert tags are updated for natGateway resource + # Check for updated user tags; system tags should persist + nat_gateway = ec2_validator.get_nat_gateway(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=nat_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=nat_gateway["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -189,10 +217,23 @@ def test_crud_tags(self, ec2_client, simple_nat_gateway): k8s.patch_custom_resource(ref, updates) time.sleep(MODIFY_WAIT_AFTER_SECONDS) + + # Check resource synced successfully + assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) + + # Check for removed user tags; system tags should persist + nat_gateway = ec2_validator.get_nat_gateway(resource_id) + tags.assert_ack_system_tags( + tags=nat_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=nat_gateway["Tags"], + ) - # Assert tags are deleted + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_route_table.py b/test/e2e/tests/test_route_table.py index cf7bf665..d84c42ab 100644 --- a/test/e2e/tests/test_route_table.py +++ b/test/e2e/tests/test_route_table.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -32,17 +33,6 @@ DELETE_WAIT_AFTER_SECONDS = 10 MODIFY_WAIT_AFTER_SECONDS = 10 -def contains_tag(resource, tag): - try: - tag_key, tag_val = tag.popitem() - for t in resource["spec"]["tags"]: - if t["key"] == tag_key and t["value"] == tag_val: - return True - except: - pass - - return False - @pytest.fixture def simple_route_table(request): replacements = REPLACEMENT_VALUES.copy() @@ -177,22 +167,35 @@ def test_crud_tags(self, ec2_client, simple_route_table): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_route_table(resource_id) - # Check tags exist for Route Table resource - assert contains_tag(resource, {"initialtagkey": "initialtagvalue"}) - - # Fetch all tags including ACK tags applied by default - new_tags = resource['spec']['tags'] - - # New tag - new_tag = { + # Check system and user tags exist for route table resource + route_table = ec2_validator.get_route_table(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=route_table["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=route_table["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 + assert resource["spec"]["tags"][0]["key"] == "initialtagkey" + assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" + + # Update tags + update_tags = [ + { "key": "updatedtagkey", "value": "updatedtagvalue", - } - new_tags.append(new_tag) + } + ] # Patch the Route Table, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -200,15 +203,29 @@ def test_crud_tags(self, ec2_client, simple_route_table): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - - # Assert tags are updated for Route Table resource + + # Check for updated user tags; system tags should persist + route_table = ec2_validator.get_route_table(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=route_table["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=route_table["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) - assert contains_tag(resource, {"updatedtagkey": "updatedtagvalue"}) + assert len(resource["spec"]["tags"]) == 1 + assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" + assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" - new_tags = [] # Patch the Route Table resource, deleting the tags updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": []}, } k8s.patch_custom_resource(ref, updates) @@ -217,9 +234,19 @@ def test_crud_tags(self, ec2_client, simple_route_table): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + route_table = ec2_validator.get_route_table(resource_id) + tags.assert_ack_system_tags( + tags=route_table["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=route_table["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert contains_tag(resource, {"updatedtagkey": "updatedtagvalue"}) is False + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_security_group.py b/test/e2e/tests/test_security_group.py index d461ed8d..8ba83a65 100644 --- a/test/e2e/tests/test_security_group.py +++ b/test/e2e/tests/test_security_group.py @@ -19,6 +19,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -207,23 +208,37 @@ def test_crud_tags(self, ec2_client, simple_security_group): # Check SecurityGroup exists in AWS ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_security_group(resource_id) + + + # Check system and user tags exist for security group resource + security_group = ec2_validator.get_security_group(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=security_group["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=security_group["Tags"], + ) - # Check tags exist for SecurityGroup resource + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the SecurityGroup, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -232,8 +247,22 @@ def test_crud_tags(self, ec2_client, simple_security_group): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for SecurityGroup resource + # Check for updated user tags; system tags should persist + security_group = ec2_validator.get_security_group(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=security_group["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=security_group["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -248,9 +277,19 @@ def test_crud_tags(self, ec2_client, simple_security_group): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + security_group = ec2_validator.get_security_group(resource_id) + tags.assert_ack_system_tags( + tags=security_group["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=security_group["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_subnet.py b/test/e2e/tests/test_subnet.py index 921f65da..f4b0192b 100644 --- a/test/e2e/tests/test_subnet.py +++ b/test/e2e/tests/test_subnet.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -178,23 +179,36 @@ def test_crud_tags(self, ec2_client): # Check Subnet exists in AWS ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_subnet(resource_id) + + # Check system and user tags exist for subnet resource + subnet = ec2_validator.get_subnet(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=subnet["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=subnet["Tags"], + ) - # Check tags exist for subnet resource + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the subnet, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -203,8 +217,22 @@ def test_crud_tags(self, ec2_client): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for subnet resource + # Check for updated user tags; system tags should persist + subnet = ec2_validator.get_subnet(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=subnet["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=subnet["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -219,9 +247,19 @@ def test_crud_tags(self, ec2_client): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + subnet = ec2_validator.get_subnet(resource_id) + tags.assert_ack_system_tags( + tags=subnet["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=subnet["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_transit_gateway.py b/test/e2e/tests/test_transit_gateway.py index f25f54af..9524b746 100644 --- a/test/e2e/tests/test_transit_gateway.py +++ b/test/e2e/tests/test_transit_gateway.py @@ -19,6 +19,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -111,22 +112,35 @@ def test_crud_tags(self, ec2_client, simple_transit_gateway): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_transit_gateway(resource_id) - # Check tags exist for TransitGateway resource + # Check system and user tags exist for transit gateway resource + transit_gateway = ec2_validator.get_transit_gateway(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=transit_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=transit_gateway["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the TransitGateway, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -135,8 +149,22 @@ def test_crud_tags(self, ec2_client, simple_transit_gateway): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for TransitGateway resource + # Check for updated user tags; system tags should persist + transit_gateway = ec2_validator.get_transit_gateway(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=transit_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=transit_gateway["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -151,9 +179,19 @@ def test_crud_tags(self, ec2_client, simple_transit_gateway): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + transit_gateway = ec2_validator.get_transit_gateway(resource_id) + tags.assert_ack_system_tags( + tags=transit_gateway["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=transit_gateway["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_vpc.py b/test/e2e/tests/test_vpc.py index 135c4b3d..ce12b4e4 100644 --- a/test/e2e/tests/test_vpc.py +++ b/test/e2e/tests/test_vpc.py @@ -18,6 +18,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -97,22 +98,35 @@ def test_crud_tags(self, ec2_client, simple_vpc): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_vpc(resource_id) - # Check tags exist for VPC resource + # Check system and user tags exist for vpc resource + vpc = ec2_validator.get_vpc(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=vpc["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=vpc["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the VPC, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -121,8 +135,22 @@ def test_crud_tags(self, ec2_client, simple_vpc): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for VPC resource + # Check for updated user tags; system tags should persist + vpc = ec2_validator.get_vpc(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=vpc["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=vpc["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -137,9 +165,19 @@ def test_crud_tags(self, ec2_client, simple_vpc): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + vpc = ec2_validator.get_vpc(resource_id) + tags.assert_ack_system_tags( + tags=vpc["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=vpc["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref) diff --git a/test/e2e/tests/test_vpc_endpoint.py b/test/e2e/tests/test_vpc_endpoint.py index 5e31644c..9369ac09 100644 --- a/test/e2e/tests/test_vpc_endpoint.py +++ b/test/e2e/tests/test_vpc_endpoint.py @@ -19,6 +19,7 @@ import time import logging +from acktest import tags from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource @@ -117,22 +118,35 @@ def test_crud_tags(self, ec2_client, simple_vpc_endpoint): ec2_validator = EC2Validator(ec2_client) ec2_validator.assert_vpc_endpoint(resource_id) - # Check tags exist for vpcEndpoint resource + # Check system and user tags exist for vpc endpoint resource + vpc_endpoint = ec2_validator.get_vpc_endpoint(resource_id) + user_tags = { + "initialtagkey": "initialtagvalue" + } + tags.assert_ack_system_tags( + tags=vpc_endpoint["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=user_tags, + actual=vpc_endpoint["Tags"], + ) + + # Only user tags should be present in Spec + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "initialtagkey" assert resource["spec"]["tags"][0]["value"] == "initialtagvalue" - # New pair of tags - new_tags = [ + # Update tags + update_tags = [ { "key": "updatedtagkey", "value": "updatedtagvalue", } - ] # Patch the vpcEndpoint, updating the tags with new pair updates = { - "spec": {"tags": new_tags}, + "spec": {"tags": update_tags}, } k8s.patch_custom_resource(ref, updates) @@ -141,8 +155,22 @@ def test_crud_tags(self, ec2_client, simple_vpc_endpoint): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are updated for vpcEndpoint resource + # Check for updated user tags; system tags should persist + vpc_endpoint = ec2_validator.get_vpc_endpoint(resource_id) + updated_tags = { + "updatedtagkey": "updatedtagvalue" + } + tags.assert_ack_system_tags( + tags=vpc_endpoint["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=updated_tags, + actual=vpc_endpoint["Tags"], + ) + + # Only user tags should be present in Spec resource = k8s.get_resource(ref) + assert len(resource["spec"]["tags"]) == 1 assert resource["spec"]["tags"][0]["key"] == "updatedtagkey" assert resource["spec"]["tags"][0]["value"] == "updatedtagvalue" @@ -157,9 +185,19 @@ def test_crud_tags(self, ec2_client, simple_vpc_endpoint): # Check resource synced successfully assert k8s.wait_on_condition(ref, "ACK.ResourceSynced", "True", wait_periods=5) - # Assert tags are deleted + # Check for removed user tags; system tags should persist + vpc_endpoint = ec2_validator.get_vpc_endpoint(resource_id) + tags.assert_ack_system_tags( + tags=vpc_endpoint["Tags"], + ) + tags.assert_equal_without_ack_tags( + expected=[], + actual=vpc_endpoint["Tags"], + ) + + # Check user tags are removed from Spec resource = k8s.get_resource(ref) - assert len(resource['spec']['tags']) == 0 + assert len(resource["spec"]["tags"]) == 0 # Delete k8s resource _, deleted = k8s.delete_custom_resource(ref)