From 8c0911b83b14d9044afd51f841ccd438dea34304 Mon Sep 17 00:00:00 2001 From: brycahta Date: Wed, 23 Mar 2022 15:36:34 -0500 Subject: [PATCH 1/7] add vpc endpoint reference test --- test/e2e/resources/security_group_ref.yaml | 10 ++ test/e2e/resources/subnet_ref.yaml | 9 ++ test/e2e/resources/vpc_endpoint_ref.yaml | 16 ++ test/e2e/tests/test_references.py | 163 +++++++++++++++++++++ 4 files changed, 198 insertions(+) create mode 100644 test/e2e/resources/security_group_ref.yaml create mode 100644 test/e2e/resources/subnet_ref.yaml create mode 100644 test/e2e/resources/vpc_endpoint_ref.yaml create mode 100644 test/e2e/tests/test_references.py diff --git a/test/e2e/resources/security_group_ref.yaml b/test/e2e/resources/security_group_ref.yaml new file mode 100644 index 00000000..95090027 --- /dev/null +++ b/test/e2e/resources/security_group_ref.yaml @@ -0,0 +1,10 @@ +apiVersion: ec2.services.k8s.aws/v1alpha1 +kind: SecurityGroup +metadata: + name: $SECURITY_GROUP_REF_NAME +spec: + description: $SECURITY_GROUP_DESCRIPTION + name: $SECURITY_GROUP_REF_NAME + vpcRef: + from: + name: $VPC_NAME \ No newline at end of file diff --git a/test/e2e/resources/subnet_ref.yaml b/test/e2e/resources/subnet_ref.yaml new file mode 100644 index 00000000..48e96758 --- /dev/null +++ b/test/e2e/resources/subnet_ref.yaml @@ -0,0 +1,9 @@ +apiVersion: ec2.services.k8s.aws/v1alpha1 +kind: Subnet +metadata: + name: $SUBNET_REF_NAME +spec: + cidrBlock: $SUBNET_CIDR_BLOCK + vpcRef: + from: + name: $VPC_NAME \ No newline at end of file diff --git a/test/e2e/resources/vpc_endpoint_ref.yaml b/test/e2e/resources/vpc_endpoint_ref.yaml new file mode 100644 index 00000000..71bf6653 --- /dev/null +++ b/test/e2e/resources/vpc_endpoint_ref.yaml @@ -0,0 +1,16 @@ +apiVersion: ec2.services.k8s.aws/v1alpha1 +kind: VPCEndpoint +metadata: + name: $VPC_ENDPOINT_REF_NAME +spec: + serviceName: $SERVICE_NAME + vpcEndpointType: $VPC_ENDPOINT_TYPE + vpcRef: + from: + name: $VPC_NAME + subnetRefs: + - from: + name: $SUBNET_REF_NAME + securityGroupRefs: + - from: + name: $SECURITY_GROUP_REF_NAME \ No newline at end of file diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py new file mode 100644 index 00000000..2efe5add --- /dev/null +++ b/test/e2e/tests/test_references.py @@ -0,0 +1,163 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""Integration tests for EC2 resource references +""" + +from os import environ +import pytest +import time + +from acktest.resources import random_suffix_name +from acktest.k8s import resource as k8s +from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource +from e2e.replacement_values import REPLACEMENT_VALUES +from e2e.tests.helper import EC2Validator + +# Default to us-west-2 since that's where prow is deployed +REGION = "us-west-2" if environ.get('AWS_DEFAULT_REGION') is None else environ.get('AWS_DEFAULT_REGION') +RESOURCE_PLURAL = "vpcendpoints" +ENDPOINT_SERVICE_NAME = "com.amazonaws.%s.s3" % REGION + +CREATE_WAIT_AFTER_SECONDS = 10 +DELETE_WAIT_AFTER_SECONDS = 10 + +@service_marker +@pytest.mark.canary +class TestEC2References: + def test_references(self, ec2_client): + vpc_endpoint_name = random_suffix_name("vpc-endpoint-test", 24) + vpc_name = random_suffix_name("vpc-ref-test", 24) + subnet_name = random_suffix_name("subnet-ref-test", 24) + security_group_name = random_suffix_name("sec-group-ref-test", 24) + + test_values = REPLACEMENT_VALUES.copy() + test_values["VPC_ENDPOINT_REF_NAME"] = vpc_endpoint_name + # Type 'Interface' allows the use of Security Groups and Subnet + test_values["VPC_ENDPOINT_TYPE"] = "Interface" + test_values["SERVICE_NAME"] = ENDPOINT_SERVICE_NAME + test_values["VPC_NAME"] = vpc_name + test_values["CIDR_BLOCK"] = "10.0.0.0/16" + test_values["SUBNET_CIDR_BLOCK"] = "10.0.255.0/24" + test_values["SUBNET_REF_NAME"] = subnet_name + test_values["SECURITY_GROUP_REF_NAME"] = security_group_name + test_values["SECURITY_GROUP_DESCRIPTION"] = "TestingSecurityGroup-ack-ref" + + # Load CRs + vpc_endpoint_resource_data = load_ec2_resource( + "vpc_endpoint_ref", + additional_replacements=test_values, + ) + sg_resource_data = load_ec2_resource( + "security_group_ref", + additional_replacements=test_values, + ) + subnet_resource_data = load_ec2_resource( + "subnet_ref", + additional_replacements=test_values, + ) + vpc_resource_data = load_ec2_resource( + "vpc", + additional_replacements=test_values, + ) + + # This test creates resources in reverse order (VPC last) so that reference + # resolution fails upon resource creation. Eventually, resources become synced + # and references resolve. + + # Create VPC Endpoint. Requires: VPC, Subnet, and SecurityGroup + vpc_endpoint_ref = k8s.CustomResourceReference( + CRD_GROUP, CRD_VERSION, 'vpcendpoints', + vpc_endpoint_name, namespace="default", + ) + k8s.create_custom_resource(vpc_endpoint_ref, vpc_endpoint_resource_data) + vpc_endpoint_cr = k8s.wait_resource_consumed_by_controller(vpc_endpoint_ref) + assert vpc_endpoint_cr is not None + + # Create Subnet. Requires: VPC + subnet_ref = k8s.CustomResourceReference( + CRD_GROUP, CRD_VERSION, 'subnets', + subnet_name, namespace="default", + ) + k8s.create_custom_resource(subnet_ref, subnet_resource_data) + subnet_cr = k8s.wait_resource_consumed_by_controller(subnet_ref) + assert subnet_cr is not None + + # Create SecurityGroups. Requires: VPC + sg_ref = k8s.CustomResourceReference( + CRD_GROUP, CRD_VERSION, 'securitygroups', + security_group_name, namespace="default", + ) + k8s.create_custom_resource(sg_ref, sg_resource_data) + sg_cr = k8s.wait_resource_consumed_by_controller(sg_ref) + assert sg_cr is not None + + # Create VPC. Requires: None + vpc_ref = k8s.CustomResourceReference( + CRD_GROUP, CRD_VERSION, 'vpcs', + vpc_name, namespace="default", + ) + k8s.create_custom_resource(vpc_ref, vpc_resource_data) + vpc_cr = k8s.wait_resource_consumed_by_controller(vpc_ref) + assert vpc_cr is not None + + # Check resources sync & resolve + assert k8s.wait_on_condition(vpc_ref, "ACK.ResourceSynced", "True", wait_periods=5) + assert k8s.wait_on_condition(sg_ref, "ACK.ResourceSynced", "True", wait_periods=5) + assert k8s.wait_on_condition(subnet_ref, "ACK.ResourceSynced", "True", wait_periods=5) + + assert k8s.wait_on_condition(sg_ref, "ACK.ReferencesResolved", "True", wait_periods=5) + assert k8s.wait_on_condition(subnet_ref, "ACK.ReferencesResolved", "True", wait_periods=5) + assert k8s.wait_on_condition(vpc_endpoint_ref, "ACK.ReferencesResolved", "True", wait_periods=5) + + time.sleep(CREATE_WAIT_AFTER_SECONDS) + + # Acquire resource IDs + vpc_endpoint_cr = k8s.get_resource(vpc_endpoint_ref) + vpc_endpoint_id = vpc_endpoint_cr["status"]["vpcEndpointID"] + subnet_cr = k8s.get_resource(subnet_ref) + subnet_id = subnet_cr["status"]["subnetID"] + sg_cr = k8s.get_resource(sg_ref) + sg_id = sg_cr["status"]["id"] + vpc_cr = k8s.get_resource(vpc_ref) + vpc_id = vpc_cr["status"]["vpcID"] + + # Check resources exist in AWS + ec2_validator = EC2Validator(ec2_client) + ec2_validator.assert_vpc_endpoint(vpc_endpoint_id) + ec2_validator.assert_subnet(subnet_id) + ec2_validator.assert_security_group(sg_id) + ec2_validator.assert_vpc(vpc_id) + + # Delete resources + _, deleted = k8s.delete_custom_resource(vpc_endpoint_ref, 2, 5) + assert deleted is True + time.sleep(DELETE_WAIT_AFTER_SECONDS) + + _, deleted = k8s.delete_custom_resource(subnet_ref, 2, 5) + assert deleted is True + time.sleep(DELETE_WAIT_AFTER_SECONDS) + + _, deleted = k8s.delete_custom_resource(sg_ref, 2, 5) + assert deleted is True + time.sleep(DELETE_WAIT_AFTER_SECONDS) + + _, deleted = k8s.delete_custom_resource(vpc_ref, 2, 5) + assert deleted is True + time.sleep(DELETE_WAIT_AFTER_SECONDS) + + # Check resources no longer exist in AWS + ec2_validator.assert_vpc_endpoint(vpc_endpoint_id, exists=False) + ec2_validator.assert_subnet(subnet_id, exists=False) + ec2_validator.assert_security_group(sg_id, exists=False) + ec2_validator.assert_vpc(vpc_id, exists=False) \ No newline at end of file From a1ff54788e60b0351ba5e0af19bd9103093ea6b5 Mon Sep 17 00:00:00 2001 From: brycahta Date: Wed, 23 Mar 2022 15:51:38 -0500 Subject: [PATCH 2/7] increase delete wait period in reference test --- test/e2e/tests/test_references.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index 2efe5add..5953af1f 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -140,19 +140,19 @@ def test_references(self, ec2_client): ec2_validator.assert_vpc(vpc_id) # Delete resources - _, deleted = k8s.delete_custom_resource(vpc_endpoint_ref, 2, 5) + _, deleted = k8s.delete_custom_resource(vpc_endpoint_ref, 6, 5) assert deleted is True time.sleep(DELETE_WAIT_AFTER_SECONDS) - _, deleted = k8s.delete_custom_resource(subnet_ref, 2, 5) + _, deleted = k8s.delete_custom_resource(subnet_ref, 6, 5) assert deleted is True time.sleep(DELETE_WAIT_AFTER_SECONDS) - _, deleted = k8s.delete_custom_resource(sg_ref, 2, 5) + _, deleted = k8s.delete_custom_resource(sg_ref, 6, 5) assert deleted is True time.sleep(DELETE_WAIT_AFTER_SECONDS) - _, deleted = k8s.delete_custom_resource(vpc_ref, 2, 5) + _, deleted = k8s.delete_custom_resource(vpc_ref, 6, 5) assert deleted is True time.sleep(DELETE_WAIT_AFTER_SECONDS) From d39515f3ae5552d693b9ef057a3e6e057aa2b4bc Mon Sep 17 00:00:00 2001 From: brycahta Date: Mon, 28 Mar 2022 12:50:49 -0400 Subject: [PATCH 3/7] use common test helper for region --- test/e2e/conftest.py | 4 +++- test/e2e/tests/test_references.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e/conftest.py b/test/e2e/conftest.py index 879f4464..4d2ca032 100644 --- a/test/e2e/conftest.py +++ b/test/e2e/conftest.py @@ -13,6 +13,7 @@ import boto3 import pytest +from acktest.aws.identity import get_region def pytest_addoption(parser): parser.addoption("--runslow", action="store_true", default=False, help="run slow tests") @@ -40,4 +41,5 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="module") def ec2_client(): - return boto3.client("ec2") \ No newline at end of file + region = get_region() + return boto3.client("ec2", region) \ No newline at end of file diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index 5953af1f..e0039d6d 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -20,14 +20,15 @@ from acktest.resources import random_suffix_name from acktest.k8s import resource as k8s +from acktest.aws.identity import get_region from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_ec2_resource from e2e.replacement_values import REPLACEMENT_VALUES from e2e.tests.helper import EC2Validator # Default to us-west-2 since that's where prow is deployed -REGION = "us-west-2" if environ.get('AWS_DEFAULT_REGION') is None else environ.get('AWS_DEFAULT_REGION') +REGION = get_region() RESOURCE_PLURAL = "vpcendpoints" -ENDPOINT_SERVICE_NAME = "com.amazonaws.%s.s3" % REGION +ENDPOINT_SERVICE_NAME = f'com.amazonaws.{REGION}.s3' CREATE_WAIT_AFTER_SECONDS = 10 DELETE_WAIT_AFTER_SECONDS = 10 From d773b9dda093c4339187b8c6381a3b7ccaf36890 Mon Sep 17 00:00:00 2001 From: brycahta Date: Mon, 28 Mar 2022 12:56:01 -0400 Subject: [PATCH 4/7] reduce reference test flakiness --- test/e2e/tests/test_references.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index e0039d6d..8c787efe 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -30,7 +30,7 @@ RESOURCE_PLURAL = "vpcendpoints" ENDPOINT_SERVICE_NAME = f'com.amazonaws.{REGION}.s3' -CREATE_WAIT_AFTER_SECONDS = 10 +CREATE_WAIT_AFTER_SECONDS = 20 DELETE_WAIT_AFTER_SECONDS = 10 @service_marker @@ -82,8 +82,6 @@ def test_references(self, ec2_client): vpc_endpoint_name, namespace="default", ) k8s.create_custom_resource(vpc_endpoint_ref, vpc_endpoint_resource_data) - vpc_endpoint_cr = k8s.wait_resource_consumed_by_controller(vpc_endpoint_ref) - assert vpc_endpoint_cr is not None # Create Subnet. Requires: VPC subnet_ref = k8s.CustomResourceReference( @@ -91,8 +89,6 @@ def test_references(self, ec2_client): subnet_name, namespace="default", ) k8s.create_custom_resource(subnet_ref, subnet_resource_data) - subnet_cr = k8s.wait_resource_consumed_by_controller(subnet_ref) - assert subnet_cr is not None # Create SecurityGroups. Requires: VPC sg_ref = k8s.CustomResourceReference( @@ -100,8 +96,6 @@ def test_references(self, ec2_client): security_group_name, namespace="default", ) k8s.create_custom_resource(sg_ref, sg_resource_data) - sg_cr = k8s.wait_resource_consumed_by_controller(sg_ref) - assert sg_cr is not None # Create VPC. Requires: None vpc_ref = k8s.CustomResourceReference( @@ -109,8 +103,9 @@ def test_references(self, ec2_client): vpc_name, namespace="default", ) k8s.create_custom_resource(vpc_ref, vpc_resource_data) - vpc_cr = k8s.wait_resource_consumed_by_controller(vpc_ref) - assert vpc_cr is not None + + # Wait a few seconds so resources get persisted in etcd + time.sleep(CREATE_WAIT_AFTER_SECONDS) # Check resources sync & resolve assert k8s.wait_on_condition(vpc_ref, "ACK.ResourceSynced", "True", wait_periods=5) @@ -121,8 +116,6 @@ def test_references(self, ec2_client): assert k8s.wait_on_condition(subnet_ref, "ACK.ReferencesResolved", "True", wait_periods=5) assert k8s.wait_on_condition(vpc_endpoint_ref, "ACK.ReferencesResolved", "True", wait_periods=5) - time.sleep(CREATE_WAIT_AFTER_SECONDS) - # Acquire resource IDs vpc_endpoint_cr = k8s.get_resource(vpc_endpoint_ref) vpc_endpoint_id = vpc_endpoint_cr["status"]["vpcEndpointID"] From 7939113149316c85ebad82cee5a5097415d4d823 Mon Sep 17 00:00:00 2001 From: brycahta Date: Tue, 29 Mar 2022 10:32:01 -0500 Subject: [PATCH 5/7] clean up region and unused consts in ref test --- test/e2e/tests/test_references.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index 8c787efe..5b7017fd 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -25,11 +25,6 @@ from e2e.replacement_values import REPLACEMENT_VALUES from e2e.tests.helper import EC2Validator -# Default to us-west-2 since that's where prow is deployed -REGION = get_region() -RESOURCE_PLURAL = "vpcendpoints" -ENDPOINT_SERVICE_NAME = f'com.amazonaws.{REGION}.s3' - CREATE_WAIT_AFTER_SECONDS = 20 DELETE_WAIT_AFTER_SECONDS = 10 @@ -46,7 +41,7 @@ def test_references(self, ec2_client): test_values["VPC_ENDPOINT_REF_NAME"] = vpc_endpoint_name # Type 'Interface' allows the use of Security Groups and Subnet test_values["VPC_ENDPOINT_TYPE"] = "Interface" - test_values["SERVICE_NAME"] = ENDPOINT_SERVICE_NAME + test_values["SERVICE_NAME"] = f'com.amazonaws.{get_region()}.s3' test_values["VPC_NAME"] = vpc_name test_values["CIDR_BLOCK"] = "10.0.0.0/16" test_values["SUBNET_CIDR_BLOCK"] = "10.0.255.0/24" From 4ee9755f19760d0d03f20f433da91b8bb80f74bd Mon Sep 17 00:00:00 2001 From: brycahta Date: Tue, 29 Mar 2022 12:28:09 -0500 Subject: [PATCH 6/7] increase wait duration after deleting vpc endpoint in ref test --- test/e2e/tests/test_references.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index 5b7017fd..25c970bc 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -131,7 +131,11 @@ def test_references(self, ec2_client): # Delete resources _, deleted = k8s.delete_custom_resource(vpc_endpoint_ref, 6, 5) assert deleted is True - time.sleep(DELETE_WAIT_AFTER_SECONDS) + # Deleting an interface endpoint also deletes the endpoint network interfaces + # and therefore requires more time to resolve server-side. Increasing the sleep + # to a longer duration allows VPC Endpoint to be removed completely. Then, + # Subnet and other dependent resources can be deleted successfully. + time.sleep(70) _, deleted = k8s.delete_custom_resource(subnet_ref, 6, 5) assert deleted is True From 5448112c2bf1d4683c8bd5fba09d8d0bf57e9919 Mon Sep 17 00:00:00 2001 From: brycahta Date: Tue, 29 Mar 2022 12:57:44 -0500 Subject: [PATCH 7/7] check for vpc deletion server-side in ref test --- test/e2e/tests/test_references.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/e2e/tests/test_references.py b/test/e2e/tests/test_references.py index 25c970bc..3bc0a393 100644 --- a/test/e2e/tests/test_references.py +++ b/test/e2e/tests/test_references.py @@ -15,6 +15,7 @@ """ from os import environ +import datetime import pytest import time @@ -27,6 +28,17 @@ CREATE_WAIT_AFTER_SECONDS = 20 DELETE_WAIT_AFTER_SECONDS = 10 +DELETE_TIMEOUT_SECONDS = 300 + +def wait_for_delete_or_die(ec2_client, vpc_endpoint_id, timeout): + while True: + if datetime.datetime.now() >= timeout: + pytest.fail("Timed out waiting for VPC Endpoint to be deleted from EC2") + time.sleep(DELETE_WAIT_AFTER_SECONDS) + try: + ec2_client.describe_vpc_endpoints(VpcEndpointIds=[vpc_endpoint_id]) + except ec2_client.exceptions.ClientError: + break @service_marker @pytest.mark.canary @@ -131,11 +143,12 @@ def test_references(self, ec2_client): # Delete resources _, deleted = k8s.delete_custom_resource(vpc_endpoint_ref, 6, 5) assert deleted is True - # Deleting an interface endpoint also deletes the endpoint network interfaces - # and therefore requires more time to resolve server-side. Increasing the sleep - # to a longer duration allows VPC Endpoint to be removed completely. Then, - # Subnet and other dependent resources can be deleted successfully. - time.sleep(70) + + # If VPC Endpoint is not completely removed server-side, then remaining + # resources will NOT delete successfully due to dependency exceptions + now = datetime.datetime.now() + timeout = now + datetime.timedelta(seconds=DELETE_TIMEOUT_SECONDS) + wait_for_delete_or_die(ec2_client, vpc_endpoint_id, timeout) _, deleted = k8s.delete_custom_resource(subnet_ref, 6, 5) assert deleted is True