Skip to content

Commit f8e8806

Browse files
damdodamdo
committed
test: e2e: make managed suite more robust to errors with Eventually()
1 parent fef3eea commit f8e8806

File tree

4 files changed

+85
-45
lines changed

4 files changed

+85
-45
lines changed

‎test/e2e/suites/managed/addon.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package managed
2222
import (
2323
"context"
2424
"fmt"
25+
"time"
2526

2627
"github.com/aws/aws-sdk-go/aws/client"
2728
"github.com/aws/aws-sdk-go/service/eks"
@@ -62,8 +63,9 @@ func CheckAddonExistsSpec(ctx context.Context, inputGetter func() CheckAddonExis
6263

6364
By(fmt.Sprintf("Getting control plane: %s", controlPlaneName))
6465
controlPlane := &ekscontrolplanev1.AWSManagedControlPlane{}
65-
err := mgmtClient.Get(ctx, crclient.ObjectKey{Namespace: input.Namespace.Name, Name: controlPlaneName}, controlPlane)
66-
Expect(err).ToNot(HaveOccurred())
66+
Eventually(func() error {
67+
return mgmtClient.Get(ctx, crclient.ObjectKey{Namespace: input.Namespace.Name, Name: controlPlaneName}, controlPlane)
68+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), "eventually failed trying to get the AWSManagedControlPlane")
6769

6870
By(fmt.Sprintf("Checking EKS addon %s is installed on cluster %s and is active", input.AddonName, input.ClusterName))
6971
waitForEKSAddonToHaveStatus(waitForEKSAddonToHaveStatusInput{

‎test/e2e/suites/managed/aws_node_env.go

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ package managed
2121

2222
import (
2323
"context"
24+
"errors"
2425
"fmt"
26+
"time"
2527

2628
"github.com/aws/aws-sdk-go/aws/client"
2729
. "github.com/onsi/ginkgo/v2"
@@ -57,27 +59,34 @@ func CheckAwsNodeEnvVarsSet(ctx context.Context, inputGetter func() UpdateAwsNod
5759

5860
By(fmt.Sprintf("Getting control plane: %s", controlPlaneName))
5961
controlPlane := &ekscontrolplanev1.AWSManagedControlPlane{}
60-
err := mgmtClient.Get(ctx, crclient.ObjectKey{Namespace: input.Namespace.Name, Name: controlPlaneName}, controlPlane)
61-
Expect(err).ToNot(HaveOccurred())
62+
Eventually(func() error {
63+
return mgmtClient.Get(ctx, crclient.ObjectKey{Namespace: input.Namespace.Name, Name: controlPlaneName}, controlPlane)
64+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), "eventually failed trying to get the AWSManagedControlPlane")
6265

6366
By(fmt.Sprintf("Checking environment variables are set on AWSManagedControlPlane: %s", controlPlaneName))
6467
Expect(controlPlane.Spec.VpcCni.Env).NotTo(BeNil())
6568
Expect(len(controlPlane.Spec.VpcCni.Env)).Should(BeNumerically(">", 1))
6669

6770
By("Checking if aws-node has been updated with the defined environment variables on the workload cluster")
6871
daemonSet := &appsv1.DaemonSet{}
69-
7072
clusterClient := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName).GetClient()
71-
err = clusterClient.Get(ctx, crclient.ObjectKey{Namespace: "kube-system", Name: "aws-node"}, daemonSet)
72-
Expect(err).ToNot(HaveOccurred())
73-
74-
for _, container := range daemonSet.Spec.Template.Spec.Containers {
75-
if container.Name == "aws-node" {
76-
Expect(matchEnvVar(container.Env, corev1.EnvVar{Name: "FOO", Value: "BAR"})).Should(BeTrue())
77-
Expect(matchEnvVar(container.Env, corev1.EnvVar{Name: "ENABLE_PREFIX_DELEGATION", Value: "true"})).Should(BeTrue())
78-
break
73+
74+
Eventually(func() error {
75+
if err := clusterClient.Get(ctx, crclient.ObjectKey{Namespace: "kube-system", Name: "aws-node"}, daemonSet); err != nil {
76+
return fmt.Errorf("unable to get aws-node: %w", err)
7977
}
80-
}
78+
79+
for _, container := range daemonSet.Spec.Template.Spec.Containers {
80+
if container.Name == "aws-node" {
81+
if matchEnvVar(container.Env, corev1.EnvVar{Name: "FOO", Value: "BAR"}) &&
82+
matchEnvVar(container.Env, corev1.EnvVar{Name: "ENABLE_PREFIX_DELEGATION", Value: "true"}) {
83+
return nil
84+
}
85+
}
86+
}
87+
88+
return errors.New("unable to find the expected environment variables on the aws-node DaemonSet's container")
89+
}, 20*time.Minute, 5*time.Second).Should(Succeed(), "should have been able to find the expected aws-node DaemonSet")
8190
}
8291

8392
func matchEnvVar(s []corev1.EnvVar, ev corev1.EnvVar) bool {

‎test/e2e/suites/managed/eks_test.go

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ package managed
2222
import (
2323
"context"
2424
"fmt"
25-
"time"
2625

2726
"github.com/onsi/ginkgo/v2"
2827
. "github.com/onsi/gomega"
@@ -75,22 +74,15 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() {
7574
})
7675

7776
ginkgo.By("should set environment variables on the aws-node daemonset")
78-
Eventually(func() error {
79-
defer ginkgo.GinkgoRecover()
80-
CheckAwsNodeEnvVarsSet(ctx, func() UpdateAwsNodeVersionSpecInput {
81-
return UpdateAwsNodeVersionSpecInput{
82-
E2EConfig: e2eCtx.E2EConfig,
83-
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
84-
AWSSession: e2eCtx.BootstrapUserAWSSession,
85-
Namespace: namespace,
86-
ClusterName: clusterName,
87-
}
88-
})
89-
return nil
90-
}).WithTimeout(5*time.Minute).WithPolling(10*time.Second).WithContext(ctx).Should(
91-
Succeed(),
92-
"Failed to verify AWS Node environment variables after 5 minutes of retries",
93-
)
77+
CheckAwsNodeEnvVarsSet(ctx, func() UpdateAwsNodeVersionSpecInput {
78+
return UpdateAwsNodeVersionSpecInput{
79+
E2EConfig: e2eCtx.E2EConfig,
80+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
81+
AWSSession: e2eCtx.BootstrapUserAWSSession,
82+
Namespace: namespace,
83+
ClusterName: clusterName,
84+
}
85+
})
9486

9587
ginkgo.By("should have the VPC CNI installed")
9688
CheckAddonExistsSpec(ctx, func() CheckAddonExistsSpecInput {

‎test/e2e/suites/managed/helpers.go

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
. "github.com/onsi/gomega"
3333
corev1 "k8s.io/api/core/v1"
3434
apimachinerytypes "k8s.io/apimachinery/pkg/types"
35+
"k8s.io/utils/ptr"
3536
crclient "sigs.k8s.io/controller-runtime/pkg/client"
3637

3738
infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
@@ -74,14 +75,19 @@ func getASGName(clusterName string) string {
7475
}
7576

7677
func verifyClusterActiveAndOwned(eksClusterName string, sess client.ConfigProvider) {
77-
cluster, err := getEKSCluster(eksClusterName, sess)
78-
Expect(err).NotTo(HaveOccurred())
78+
var (
79+
cluster *eks.Cluster
80+
err error
81+
)
82+
Eventually(func() error {
83+
cluster, err = getEKSCluster(eksClusterName, sess)
84+
return err
85+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), fmt.Sprintf("eventually failed trying to get EKS Cluster %q", eksClusterName))
7986

8087
tagName := infrav1.ClusterTagKey(eksClusterName)
8188
tagValue, ok := cluster.Tags[tagName]
8289
Expect(ok).To(BeTrue(), "expecting the cluster owned tag to exist")
8390
Expect(*tagValue).To(BeEquivalentTo(string(infrav1.ResourceLifecycleOwned)))
84-
8591
Expect(*cluster.Status).To(BeEquivalentTo(eks.ClusterStatusActive))
8692
}
8793

@@ -102,6 +108,7 @@ func getEKSClusterAddon(eksClusterName, addonName string, sess client.ConfigProv
102108
AddonName: &addonName,
103109
ClusterName: &eksClusterName,
104110
}
111+
105112
describeOutput, err := eksClient.DescribeAddon(describeInput)
106113
if err != nil {
107114
return nil, fmt.Errorf("describing eks addon %s: %w", addonName, err)
@@ -112,16 +119,16 @@ func getEKSClusterAddon(eksClusterName, addonName string, sess client.ConfigProv
112119

113120
func verifySecretExists(ctx context.Context, secretName, namespace string, k8sclient crclient.Client) {
114121
secret := &corev1.Secret{}
115-
err := k8sclient.Get(ctx, apimachinerytypes.NamespacedName{Name: secretName, Namespace: namespace}, secret)
116-
117-
Expect(err).ShouldNot(HaveOccurred())
122+
Eventually(func() error {
123+
return k8sclient.Get(ctx, apimachinerytypes.NamespacedName{Name: secretName, Namespace: namespace}, secret)
124+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), fmt.Sprintf("eventually failed trying to verify Secret %q exists", secretName))
118125
}
119126

120127
func verifyConfigMapExists(ctx context.Context, name, namespace string, k8sclient crclient.Client) {
121128
cm := &corev1.ConfigMap{}
122129
Eventually(func() error {
123130
return k8sclient.Get(ctx, apimachinerytypes.NamespacedName{Name: name, Namespace: namespace}, cm)
124-
}, 2*time.Minute, 5*time.Second).Should(Succeed())
131+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), fmt.Sprintf("eventually failed trying to verify ConfigMap %q exists", name))
125132
}
126133

127134
func VerifyRoleExistsAndOwned(roleName string, eksClusterName string, checkOwned bool, sess client.ConfigProvider) {
@@ -130,8 +137,15 @@ func VerifyRoleExistsAndOwned(roleName string, eksClusterName string, checkOwned
130137
RoleName: aws.String(roleName),
131138
}
132139

133-
output, err := iamClient.GetRole(input)
134-
Expect(err).ShouldNot(HaveOccurred())
140+
var (
141+
output *iam.GetRoleOutput
142+
err error
143+
)
144+
145+
Eventually(func() error {
146+
output, err = iamClient.GetRole(input)
147+
return err
148+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), fmt.Sprintf("eventually failed trying to get IAM Role %q", roleName))
135149

136150
if checkOwned {
137151
found := false
@@ -152,9 +166,24 @@ func verifyManagedNodeGroup(eksClusterName, nodeGroupName string, checkOwned boo
152166
ClusterName: aws.String(eksClusterName),
153167
NodegroupName: aws.String(nodeGroupName),
154168
}
155-
result, err := eksClient.DescribeNodegroup(input)
156-
Expect(err).NotTo(HaveOccurred())
157-
Expect(*result.Nodegroup.Status).To(BeEquivalentTo(eks.NodegroupStatusActive))
169+
var (
170+
result *eks.DescribeNodegroupOutput
171+
err error
172+
)
173+
174+
Eventually(func() error {
175+
result, err = eksClient.DescribeNodegroup(input)
176+
if err != nil {
177+
return fmt.Errorf("error describing nodegroup: %w", err)
178+
}
179+
180+
nodeGroupStatus := ptr.Deref(result.Nodegroup.Status, "")
181+
if nodeGroupStatus != eks.NodegroupStatusActive {
182+
return fmt.Errorf("expected nodegroup.Status to be %q, was %q instead", eks.NodegroupStatusActive, nodeGroupStatus)
183+
}
184+
185+
return nil
186+
}, 2*time.Minute, 5*time.Second).Should(Succeed(), "eventually failed trying to describe EKS Node group")
158187

159188
if checkOwned {
160189
tagName := infrav1.ClusterAWSCloudProviderTagKey(eksClusterName)
@@ -172,8 +201,16 @@ func verifyASG(eksClusterName, asgName string, checkOwned bool, sess client.Conf
172201
},
173202
}
174203

175-
result, err := asgClient.DescribeAutoScalingGroups(input)
176-
Expect(err).NotTo(HaveOccurred())
204+
var (
205+
result *autoscaling.DescribeAutoScalingGroupsOutput
206+
err error
207+
)
208+
209+
Eventually(func() error {
210+
result, err = asgClient.DescribeAutoScalingGroups(input)
211+
return err
212+
}, 2*time.Minute, 5*time.Second).Should(Succeed())
213+
177214
for _, instance := range result.AutoScalingGroups[0].Instances {
178215
Expect(*instance.LifecycleState).To(Equal("InService"), "expecting the instance in service")
179216
}

0 commit comments

Comments
 (0)