Skip to content

Commit 3244b3b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #3648 from richardcase/1718_gc_add_e2e
feat: external load balancer garbage collection (part 4) - e2e tests
2 parents a87fc10 + c507f32 commit 3244b3b

19 files changed

+1111
-6
lines changed

‎Makefile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,14 @@ test-e2e: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) generate-test-flavors e2e
412412
test-e2e-eks: generate-test-flavors $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
413413
time $(GINKGO) -tags=e2e $(GINKGO_ARGS) ./test/e2e/suites/managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS)
414414

415+
.PHONY: test-e2e-gc ## Run garbage collection e2e tests using clusterctl
416+
test-e2e-gc: generate-test-flavors $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
417+
time $(GINKGO) -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" $(GINKGO_ARGS) -p ./test/e2e/suites/gc_unmanaged/... -- -config-path="$(E2E_CONF_PATH)" $(E2E_ARGS)
418+
419+
.PHONY: test-e2e-eks-gc ## Run EKS garbage collection e2e tests using clusterctl
420+
test-e2e-eks-gc: generate-test-flavors $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
421+
time $(GINKGO) -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" $(GINKGO_ARGS) ./test/e2e/suites/gc_managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS)
422+
415423

416424
CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
417425
CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
@@ -435,6 +443,8 @@ compile-e2e: ## Test e2e compilation
435443
go test -c -o /dev/null -tags=e2e ./test/e2e/suites/unmanaged
436444
go test -c -o /dev/null -tags=e2e ./test/e2e/suites/conformance
437445
go test -c -o /dev/null -tags=e2e ./test/e2e/suites/managed
446+
go test -c -o /dev/null -tags=e2e ./test/e2e/suites/gc_managed
447+
go test -c -o /dev/null -tags=e2e ./test/e2e/suites/gc_unmanaged
438448

439449

440450
.PHONY: docker-pull-e2e-preloads

‎pkg/cloud/services/gc/service.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,9 @@ func NewService(clusterScope cloud.ClusterScoper, opts ...ServiceOption) *Servic
6060

6161
func addDefaultCleanupFuncs(s *Service) {
6262
s.cleanupFuncs = []ResourceCleanupFunc{
63-
s.deleteSecurityGroups,
6463
s.deleteLoadBalancers,
6564
s.deleteTargetGroups,
65+
s.deleteSecurityGroups,
6666
}
6767
}
6868

‎scripts/ci-e2e-eks-gc.sh

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/bash
2+
3+
# Copyright 2022 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
################################################################################
18+
# usage: e2e.sh
19+
# This program runs the e2e tests.
20+
#
21+
# ENVIRONMENT VARIABLES
22+
# JANITOR_ENABLED
23+
# Set to 1 to run the aws-janitor command after running the e2e tests.
24+
################################################################################
25+
26+
set -o nounset
27+
set -o pipefail
28+
29+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
30+
cd "${REPO_ROOT}" || exit 1
31+
32+
# shellcheck source=../hack/ensure-go.sh
33+
source "${REPO_ROOT}/hack/ensure-go.sh"
34+
# shellcheck source=../hack/ensure-kind.sh
35+
source "${REPO_ROOT}/hack/ensure-kind.sh"
36+
# shellcheck source=../hack/ensure-kubectl.sh
37+
source "${REPO_ROOT}/hack/ensure-kubectl.sh"
38+
39+
ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
40+
mkdir -p "$ARTIFACTS/logs/"
41+
42+
# our exit handler (trap)
43+
cleanup() {
44+
# stop boskos heartbeat
45+
[[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
46+
}
47+
trap cleanup EXIT
48+
49+
#Install requests module explicitly for HTTP calls
50+
python3 -m pip install requests
51+
52+
# If BOSKOS_HOST is set then acquire an AWS account from Boskos.
53+
if [ -n "${BOSKOS_HOST:-}" ]; then
54+
# Check out the account from Boskos and store the produced environment
55+
# variables in a temporary file.
56+
account_env_var_file="$(mktemp)"
57+
python3 hack/boskos.py --get 1>"${account_env_var_file}"
58+
checkout_account_status="${?}"
59+
60+
# If the checkout process was a success then load the account's
61+
# environment variables into this process.
62+
# shellcheck disable=SC1090
63+
[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
64+
65+
# Always remove the account environment variable file. It contains
66+
# sensitive information.
67+
rm -f "${account_env_var_file}"
68+
69+
if [ ! "${checkout_account_status}" = "0" ]; then
70+
echo "error getting account from boskos" 1>&2
71+
exit "${checkout_account_status}"
72+
fi
73+
74+
# run the heart beat process to tell boskos that we are still
75+
# using the checked out account periodically
76+
python3 -u hack/boskos.py --heartbeat >>$ARTIFACTS/logs/boskos.log 2>&1 &
77+
HEART_BEAT_PID=$(echo $!)
78+
fi
79+
80+
# Prevent a disallowed AWS key from being used.
81+
if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" |
82+
{ md5sum 2>/dev/null || md5; } |
83+
awk '{print $1}')" hack/e2e-aws-disallowed.txt; then
84+
echo "The provided AWS key is not allowed" 1>&2
85+
exit 1
86+
fi
87+
88+
EXP_EXTERNAL_RESOURCE_GC="true" GC_WORKLOAD="../../data/gcworkload.yaml" make test-e2e-eks-gc ARTIFACTS=$ARTIFACTS
89+
90+
test_status="${?}"
91+
92+
# If Boskos is being used then release the AWS account back to Boskos.
93+
[ -z "${BOSKOS_HOST:-}" ] || python3 -u hack/boskos.py --release
94+
95+
# The janitor is typically not run as part of the e2e process, but rather
96+
# in a parallel process via a service on the same cluster that runs Prow and
97+
# Boskos.
98+
#
99+
# However, setting JANITOR_ENABLED=1 tells this program to run the janitor
100+
# after the e2e test is executed.
101+
if [ "${JANITOR_ENABLED:-0}" = "1" ]; then
102+
if ! command -v aws-janitor >/dev/null 2>&1; then
103+
echo "skipping janitor; aws-janitor not found" 1>&2
104+
else
105+
aws-janitor -all -v 2
106+
fi
107+
else
108+
echo "skipping janitor; JANITOR_ENABLED=${JANITOR_ENABLED:-0}" 1>&2
109+
fi
110+
111+
exit "${test_status}"

‎scripts/ci-e2e-gc.sh

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/bash
2+
3+
# Copyright 2022 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
################################################################################
18+
# usage: e2e.sh
19+
# This program runs the e2e tests.
20+
#
21+
# ENVIRONMENT VARIABLES
22+
# JANITOR_ENABLED
23+
# Set to 1 to run the aws-janitor command after running the e2e tests.
24+
################################################################################
25+
26+
set -o nounset
27+
set -o pipefail
28+
29+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
30+
cd "${REPO_ROOT}" || exit 1
31+
32+
# shellcheck source=../hack/ensure-go.sh
33+
source "${REPO_ROOT}/hack/ensure-go.sh"
34+
# shellcheck source=../hack/ensure-kind.sh
35+
source "${REPO_ROOT}/hack/ensure-kind.sh"
36+
# shellcheck source=../hack/ensure-kubectl.sh
37+
source "${REPO_ROOT}/hack/ensure-kubectl.sh"
38+
39+
ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
40+
mkdir -p "$ARTIFACTS/logs/"
41+
42+
# our exit handler (trap)
43+
cleanup() {
44+
# stop boskos heartbeat
45+
[[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
46+
}
47+
trap cleanup EXIT
48+
49+
#Install requests module explicitly for HTTP calls
50+
python3 -m pip install requests
51+
52+
# If BOSKOS_HOST is set then acquire an AWS account from Boskos.
53+
if [ -n "${BOSKOS_HOST:-}" ]; then
54+
# Check out the account from Boskos and store the produced environment
55+
# variables in a temporary file.
56+
account_env_var_file="$(mktemp)"
57+
python3 hack/boskos.py --get 1>"${account_env_var_file}"
58+
checkout_account_status="${?}"
59+
60+
# If the checkout process was a success then load the account's
61+
# environment variables into this process.
62+
# shellcheck disable=SC1090
63+
[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
64+
65+
# Always remove the account environment variable file. It contains
66+
# sensitive information.
67+
rm -f "${account_env_var_file}"
68+
69+
if [ ! "${checkout_account_status}" = "0" ]; then
70+
echo "error getting account from boskos" 1>&2
71+
exit "${checkout_account_status}"
72+
fi
73+
74+
# run the heart beat process to tell boskos that we are still
75+
# using the checked out account periodically
76+
python3 -u hack/boskos.py --heartbeat >>$ARTIFACTS/logs/boskos.log 2>&1 &
77+
HEART_BEAT_PID=$(echo $!)
78+
fi
79+
80+
# Prevent a disallowed AWS key from being used.
81+
if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" |
82+
{ md5sum 2>/dev/null || md5; } |
83+
awk '{print $1}')" hack/e2e-aws-disallowed.txt; then
84+
echo "The provided AWS key is not allowed" 1>&2
85+
exit 1
86+
fi
87+
88+
EXP_EXTERNAL_RESOURCE_GC="true" GC_WORKLOAD="../../data/gcworkload.yaml" make test-e2e-gc ARTIFACTS=$ARTIFACTS
89+
90+
test_status="${?}"
91+
92+
# If Boskos is being used then release the AWS account back to Boskos.
93+
[ -z "${BOSKOS_HOST:-}" ] || python3 -u hack/boskos.py --release
94+
95+
# The janitor is typically not run as part of the e2e process, but rather
96+
# in a parallel process via a service on the same cluster that runs Prow and
97+
# Boskos.
98+
#
99+
# However, setting JANITOR_ENABLED=1 tells this program to run the janitor
100+
# after the e2e test is executed.
101+
if [ "${JANITOR_ENABLED:-0}" = "1" ]; then
102+
if ! command -v aws-janitor >/dev/null 2>&1; then
103+
echo "skipping janitor; aws-janitor not found" 1>&2
104+
else
105+
aws-janitor -all -v 2
106+
fi
107+
else
108+
echo "skipping janitor; JANITOR_ENABLED=${JANITOR_ENABLED:-0}" 1>&2
109+
fi
110+
111+
exit "${test_status}"

‎test/e2e/data/e2e_conf.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ variables:
197197
MULTI_TENANCY_ROLE_NAME: "multi-tenancy-role"
198198
MULTI_TENANCY_NESTED_ROLE_NAME: "multi-tenancy-nested-role"
199199
IP_FAMILY: "IPv4"
200+
CAPA_LOGLEVEL: "4"
200201
# Enabling the feature flags by setting the env variables.
201202
EXP_CLUSTER_RESOURCE_SET: "true"
202203
EXP_MACHINE_POOL: "true"
@@ -226,3 +227,5 @@ intervals:
226227
default/wait-machine-pool-upgrade: [ "50m", "10s" ]
227228
default/wait-create-identity: ["1m", "10s"]
228229
default/wait-job: ["10m", "10s"]
230+
default/wait-deployment-ready: ["5m", "10s"]
231+
default/wait-loadbalancer-ready: ["5m", "30s"]

‎test/e2e/data/e2e_eks_conf.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ variables:
188188
CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "1.22.9"
189189
AUTO_CONTROLLER_IDENTITY_CREATOR: "false"
190190
IP_FAMILY: "IPv4"
191+
CAPA_LOGLEVEL: "4"
191192

192193
intervals:
193194
default/wait-cluster: ["30m", "10s"]
@@ -204,3 +205,5 @@ intervals:
204205
default/wait-control-plane-upgrade: ["35m", "30s"]
205206
default/wait-addon-status: ["10m", "30s"]
206207
default/wait-create-identity: ["1m", "10s"]
208+
default/wait-deployment-ready: ["5m", "10s"]
209+
default/wait-loadbalancer-ready: ["5m", "30s"]

0 commit comments

Comments
 (0)