Skip to content

Commit 1a88f55

Browse files
openshift-ci[bot]openshift-ci[bot]
authored
Merge pull request #62 from alexander-demichev/e2eaws
Bug 2099340: Add a basic e2e suite for AWS
2 parents af87743 + 2a0bc29 commit 1a88f55

File tree

185 files changed

+15712
-8023
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

185 files changed

+15712
-8023
lines changed

‎Makefile

+11-2
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,12 @@ build: operator
3232
operator:
3333
go build -o bin/cluster-capi-operator cmd/cluster-capi-operator/main.go
3434

35-
unit: envtest
36-
KUBEBUILDER_ASSETS=$(shell $(ENVTEST) --bin-dir=$(shell pwd)/bin use $(ENVTEST_K8S_VERSION) -p path) go test ./... -coverprofile cover.out
35+
unit: ginkgo envtest
36+
KUBEBUILDER_ASSETS=$(shell $(ENVTEST) --bin-dir=$(shell pwd)/bin use $(ENVTEST_K8S_VERSION) -p path) ./hack/test.sh "./pkg/... ./assets/..." 5m
37+
38+
.PHONY: e2e
39+
e2e: ginkgo
40+
./hack/test.sh "./e2e/..." 30m
3741

3842
# Run against the configured Kubernetes cluster in ~/.kube/config
3943
run: verify
@@ -63,6 +67,11 @@ ENVTEST = $(shell pwd)/bin/setup-envtest
6367
envtest: # Download envtest-setup locally if necessary.
6468
GOBIN=$(PROJECT_DIR)/bin go install -mod=readonly sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
6569

70+
.PHONY: ginkgo
71+
GINKGO = $(shell pwd)/bin/GINKGO
72+
ginkgo: # Download envtest-setup locally if necessary.
73+
GOBIN=$(PROJECT_DIR)/bin go install -mod=readonly github.com/onsi/ginkgo/v2/ginkgo@latest
74+
6675
.PHONY: assets
6776
assets:
6877
./hack/assets.sh

‎assets/embed_test.go

+25-27
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
package assets
22

33
import (
4-
"testing"
5-
4+
. "github.com/onsi/ginkgo/v2"
65
. "github.com/onsi/gomega"
76
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
87
"k8s.io/client-go/kubernetes/scheme"
@@ -13,36 +12,35 @@ func init() {
1312
utilruntime.Must(operatorv1.AddToScheme(scheme.Scheme))
1413
}
1514

16-
func TestReadCoreProviderAssets(t *testing.T) {
17-
g := NewGomegaWithT(t)
18-
19-
objs, err := ReadCoreProviderAssets(scheme.Scheme)
20-
g.Expect(err).NotTo(HaveOccurred())
15+
var _ = Describe("Read assets suite", func() {
16+
It("should read core provider assets", func() {
2117

22-
g.Expect(objs).To(HaveLen(2))
18+
objs, err := ReadCoreProviderAssets(scheme.Scheme)
19+
Expect(err).NotTo(HaveOccurred())
2320

24-
g.Expect(objs).Should(HaveKey(CoreProviderKey))
25-
g.Expect(objs[CoreProviderKey]).ToNot(BeNil())
26-
g.Expect(objs[CoreProviderKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("CoreProvider"))
21+
Expect(objs).To(HaveLen(2))
2722

28-
g.Expect(objs).Should(HaveKey(CoreProviderConfigMapKey))
29-
g.Expect(objs[CoreProviderConfigMapKey]).ToNot(BeNil())
30-
g.Expect(objs[CoreProviderConfigMapKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("ConfigMap"))
31-
}
23+
Expect(objs).Should(HaveKey(CoreProviderKey))
24+
Expect(objs[CoreProviderKey]).ToNot(BeNil())
25+
Expect(objs[CoreProviderKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("CoreProvider"))
3226

33-
func TestReadInfrastructureProviderAssets(t *testing.T) {
34-
g := NewGomegaWithT(t)
27+
Expect(objs).Should(HaveKey(CoreProviderConfigMapKey))
28+
Expect(objs[CoreProviderConfigMapKey]).ToNot(BeNil())
29+
Expect(objs[CoreProviderConfigMapKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("ConfigMap"))
30+
})
3531

36-
objs, err := ReadInfrastructureProviderAssets(scheme.Scheme, "aws")
37-
g.Expect(err).NotTo(HaveOccurred())
32+
It("should read infra provider assets", func() {
33+
objs, err := ReadInfrastructureProviderAssets(scheme.Scheme, "aws")
34+
Expect(err).NotTo(HaveOccurred())
3835

39-
g.Expect(objs).To(HaveLen(2))
36+
Expect(objs).To(HaveLen(2))
4037

41-
g.Expect(objs).Should(HaveKey(InfrastructureProviderKey))
42-
g.Expect(objs[InfrastructureProviderKey]).ToNot(BeNil())
43-
g.Expect(objs[InfrastructureProviderKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("InfrastructureProvider"))
38+
Expect(objs).Should(HaveKey(InfrastructureProviderKey))
39+
Expect(objs[InfrastructureProviderKey]).ToNot(BeNil())
40+
Expect(objs[InfrastructureProviderKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("InfrastructureProvider"))
4441

45-
g.Expect(objs).Should(HaveKey(InfrastructureProviderConfigMapKey))
46-
g.Expect(objs[InfrastructureProviderConfigMapKey]).ToNot(BeNil())
47-
g.Expect(objs[InfrastructureProviderConfigMapKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("ConfigMap"))
48-
}
42+
Expect(objs).Should(HaveKey(InfrastructureProviderConfigMapKey))
43+
Expect(objs[InfrastructureProviderConfigMapKey]).ToNot(BeNil())
44+
Expect(objs[InfrastructureProviderConfigMapKey].GetObjectKind().GroupVersionKind().Kind).To(Equal("ConfigMap"))
45+
})
46+
})

‎assets/suite_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package assets
2+
3+
import (
4+
"testing"
5+
6+
. "github.com/onsi/ginkgo/v2"
7+
. "github.com/onsi/gomega"
8+
)
9+
10+
func TestAPIs(t *testing.T) {
11+
RegisterFailHandler(Fail)
12+
RunSpecs(t, "Assets Suite")
13+
}

‎e2e/aws_test.go

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package e2e
2+
3+
import (
4+
. "github.com/onsi/ginkgo/v2"
5+
. "github.com/onsi/gomega"
6+
corev1 "k8s.io/api/core/v1"
7+
apierrors "k8s.io/apimachinery/pkg/api/errors"
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
awsv1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
10+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
11+
"sigs.k8s.io/controller-runtime/pkg/client"
12+
yaml "sigs.k8s.io/yaml"
13+
14+
configv1 "github.com/openshift/api/config/v1"
15+
mapiv1 "github.com/openshift/api/machine/v1beta1"
16+
"github.com/openshift/cluster-capi-operator/e2e/framework"
17+
)
18+
19+
const (
20+
awsMachineTemplateName = "aws-machine-template"
21+
)
22+
23+
var _ = Describe("Cluster API AWS MachineSet", Ordered, func() {
24+
var awsMachineTemplate *awsv1.AWSMachineTemplate
25+
var machineSet *clusterv1.MachineSet
26+
27+
BeforeAll(func() {
28+
if platform != configv1.AWSPlatformType {
29+
Skip("Skipping AWS E2E tests")
30+
}
31+
framework.CreateCoreCluster(cl, clusterName, "AWSCluster")
32+
createAWSCluster(cl, getAWSMAPIProviderSpec(cl))
33+
})
34+
35+
AfterEach(func() {
36+
framework.DeleteMachineSets(cl, machineSet)
37+
framework.WaitForMachineSetsDeleted(cl, machineSet)
38+
framework.DeleteObjects(cl, awsMachineTemplate)
39+
})
40+
41+
It("should be able to run a machine", func() {
42+
awsMachineTemplate = createAWSMachineTemplate(cl, getAWSMAPIProviderSpec(cl))
43+
44+
machineSet = framework.CreateMachineSet(cl, framework.NewMachineSetParams(
45+
"aws-machineset",
46+
clusterName,
47+
1,
48+
corev1.ObjectReference{
49+
Kind: "AWSMachineTemplate",
50+
APIVersion: infraAPIVersion,
51+
Name: awsMachineTemplateName,
52+
},
53+
))
54+
55+
framework.WaitForMachineSet(cl, machineSet.Name)
56+
})
57+
})
58+
59+
func getAWSMAPIProviderSpec(cl client.Client) *mapiv1.AWSMachineProviderConfig {
60+
machineSetList := &mapiv1.MachineSetList{}
61+
Expect(cl.List(ctx, machineSetList, client.InNamespace(framework.MAPINamespace))).To(Succeed())
62+
63+
Expect(machineSetList.Items).ToNot(HaveLen(0))
64+
machineSet := machineSetList.Items[0]
65+
Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil())
66+
67+
providerSpec := &mapiv1.AWSMachineProviderConfig{}
68+
Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed())
69+
70+
return providerSpec
71+
}
72+
73+
func createAWSCluster(cl client.Client, mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSCluster {
74+
By("Creating AWS cluster")
75+
76+
awsCluster := &awsv1.AWSCluster{
77+
ObjectMeta: metav1.ObjectMeta{
78+
Name: clusterName,
79+
Namespace: framework.CAPINamespace,
80+
},
81+
Spec: awsv1.AWSClusterSpec{
82+
Region: mapiProviderSpec.Placement.Region,
83+
},
84+
}
85+
86+
if err := cl.Create(ctx, awsCluster); err != nil && !apierrors.IsAlreadyExists(err) {
87+
Expect(err).ToNot(HaveOccurred())
88+
}
89+
90+
Eventually(func() (bool, error) {
91+
patchedAWSCluster := &awsv1.AWSCluster{}
92+
err := cl.Get(ctx, client.ObjectKeyFromObject(awsCluster), patchedAWSCluster)
93+
if err != nil {
94+
return false, err
95+
}
96+
97+
if patchedAWSCluster.Annotations == nil {
98+
return false, nil
99+
}
100+
101+
if _, ok := patchedAWSCluster.Annotations[clusterv1.ManagedByAnnotation]; !ok {
102+
return false, nil
103+
}
104+
105+
return patchedAWSCluster.Status.Ready, nil
106+
}, framework.WaitShort).Should(BeTrue())
107+
108+
return awsCluster
109+
}
110+
111+
func createAWSMachineTemplate(cl client.Client, mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate {
112+
By("Creating AWS machine template")
113+
114+
Expect(mapiProviderSpec).ToNot(BeNil())
115+
Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil())
116+
Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil())
117+
Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty())
118+
Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty())
119+
Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil())
120+
Expect(mapiProviderSpec.Subnet.Filters).ToNot(HaveLen(0))
121+
Expect(mapiProviderSpec.Subnet.Filters[0].Values).ToNot(HaveLen(0))
122+
Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0))
123+
Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0))
124+
Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0))
125+
126+
uncompressedUserData := true
127+
128+
awsMachineSpec := awsv1.AWSMachineSpec{
129+
UncompressedUserData: &uncompressedUserData,
130+
IAMInstanceProfile: *mapiProviderSpec.IAMInstanceProfile.ID,
131+
InstanceType: mapiProviderSpec.InstanceType,
132+
FailureDomain: &mapiProviderSpec.Placement.AvailabilityZone,
133+
CloudInit: awsv1.CloudInit{
134+
InsecureSkipSecretsManager: true,
135+
},
136+
AMI: awsv1.AMIReference{
137+
ID: mapiProviderSpec.AMI.ID,
138+
},
139+
Subnet: &awsv1.AWSResourceReference{
140+
Filters: []awsv1.Filter{
141+
{
142+
Name: "tag:Name",
143+
Values: mapiProviderSpec.Subnet.Filters[0].Values,
144+
},
145+
},
146+
},
147+
AdditionalSecurityGroups: []awsv1.AWSResourceReference{
148+
{
149+
Filters: []awsv1.Filter{
150+
{
151+
Name: "tag:Name",
152+
Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values,
153+
},
154+
},
155+
},
156+
},
157+
}
158+
159+
awsMachineTemplate := &awsv1.AWSMachineTemplate{
160+
ObjectMeta: metav1.ObjectMeta{
161+
Name: awsMachineTemplateName,
162+
Namespace: framework.CAPINamespace,
163+
},
164+
Spec: awsv1.AWSMachineTemplateSpec{
165+
Template: awsv1.AWSMachineTemplateResource{
166+
Spec: awsMachineSpec,
167+
},
168+
},
169+
}
170+
171+
if err := cl.Create(ctx, awsMachineTemplate); err != nil && !apierrors.IsAlreadyExists(err) {
172+
Expect(err).ToNot(HaveOccurred())
173+
}
174+
175+
return awsMachineTemplate
176+
}

‎e2e/e2e_test.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
10+
"k8s.io/client-go/kubernetes/scheme"
11+
awsv1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
12+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
13+
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
14+
"sigs.k8s.io/controller-runtime/pkg/client/config"
15+
16+
configv1 "github.com/openshift/api/config/v1"
17+
mapiv1 "github.com/openshift/api/machine/v1beta1"
18+
)
19+
20+
const (
21+
infrastructureName = "cluster"
22+
infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta1"
23+
)
24+
25+
var (
26+
cl runtimeclient.Client
27+
ctx = context.Background()
28+
platform configv1.PlatformType
29+
clusterName string
30+
)
31+
32+
func init() {
33+
utilruntime.Must(configv1.Install(scheme.Scheme))
34+
utilruntime.Must(awsv1.AddToScheme(scheme.Scheme))
35+
utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme))
36+
utilruntime.Must(mapiv1.AddToScheme(scheme.Scheme))
37+
}
38+
39+
func TestAPIs(t *testing.T) {
40+
RegisterFailHandler(Fail)
41+
RunSpecs(t, "Cluster API Suite")
42+
}
43+
44+
var _ = BeforeSuite(func() {
45+
cfg, err := config.GetConfig()
46+
Expect(err).ToNot(HaveOccurred())
47+
48+
cl, err = runtimeclient.New(cfg, runtimeclient.Options{})
49+
Expect(err).ToNot(HaveOccurred())
50+
51+
infra := &configv1.Infrastructure{}
52+
infraName := runtimeclient.ObjectKey{
53+
Name: infrastructureName,
54+
}
55+
Expect(cl.Get(ctx, infraName, infra)).To(Succeed())
56+
Expect(infra.Status.PlatformStatus).ToNot(BeNil())
57+
clusterName = infra.Status.InfrastructureName
58+
platform = infra.Status.PlatformStatus.Type
59+
})

‎e2e/framework/cluster.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package framework
2+
3+
import (
4+
. "github.com/onsi/ginkgo/v2"
5+
. "github.com/onsi/gomega"
6+
corev1 "k8s.io/api/core/v1"
7+
apierrors "k8s.io/apimachinery/pkg/api/errors"
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
10+
"sigs.k8s.io/cluster-api/util/conditions"
11+
"sigs.k8s.io/controller-runtime/pkg/client"
12+
)
13+
14+
// CreateCluster creates a cluster with the given name and returns the cluster object.
15+
func CreateCoreCluster(cl client.Client, clusterName, infraClusterKind string) *clusterv1.Cluster {
16+
By("Creating core cluster")
17+
18+
cluster := &clusterv1.Cluster{
19+
ObjectMeta: metav1.ObjectMeta{
20+
Name: clusterName,
21+
Namespace: CAPINamespace,
22+
},
23+
Spec: clusterv1.ClusterSpec{
24+
InfrastructureRef: &corev1.ObjectReference{
25+
APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
26+
Kind: infraClusterKind,
27+
Name: clusterName,
28+
Namespace: CAPINamespace,
29+
},
30+
},
31+
}
32+
33+
if err := cl.Create(ctx, cluster); err != nil && !apierrors.IsAlreadyExists(err) {
34+
Expect(err).ToNot(HaveOccurred())
35+
}
36+
37+
Eventually(func() (bool, error) {
38+
patchedCluster := &clusterv1.Cluster{}
39+
err := cl.Get(ctx, client.ObjectKeyFromObject(cluster), patchedCluster)
40+
if err != nil {
41+
return false, err
42+
}
43+
44+
return conditions.IsTrue(patchedCluster, clusterv1.ControlPlaneInitializedCondition), nil
45+
}, WaitShort).Should(BeTrue())
46+
47+
return cluster
48+
}

0 commit comments

Comments
 (0)