Skip to content

Commit 5478a1b

Browse files
sbueringersbueringer
committed
Add e2e test to reproduce SSA apiVersion issue
Signed-off-by: Stefan Büringer [email protected]
1 parent a4c34f3 commit 5478a1b

File tree

12 files changed

+694
-91
lines changed

12 files changed

+694
-91
lines changed

‎Makefile

+5-1
Original file line numberDiff line numberDiff line change
@@ -511,11 +511,15 @@ generate-doctoc:
511511
TRACE=$(TRACE) ./hack/generate-doctoc.sh
512512

513513
.PHONY: generate-e2e-templates
514-
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
514+
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
515515

516516
DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker
517517
INMEMORY_TEMPLATES := test/e2e/data/infrastructure-inmemory
518518

519+
.PHONY: generate-e2e-templates-v0.4
520+
generate-e2e-templates-v0.4: $(KUSTOMIZE)
521+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml
522+
519523
.PHONY: generate-e2e-templates-v1.0
520524
generate-e2e-templates-v1.0: $(KUSTOMIZE)
521525
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.0/cluster-template.yaml

‎test/e2e/clusterctl_upgrade.go

+330-56
Large diffs are not rendered by default.

‎test/e2e/clusterctl_upgrade_test.go

+53
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,59 @@ var (
3434
providerDockerPrefix = "docker:v%s"
3535
)
3636

37+
var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current) [PR-Blocking]", func() {
38+
// Get v0.4 latest stable release
39+
version04 := "0.4"
40+
stableRelease04, err := GetStableReleaseOfMinor(ctx, version04)
41+
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version04)
42+
43+
// Get v1.6 latest stable release
44+
version16 := "1.6"
45+
stableRelease16, err := GetStableReleaseOfMinor(ctx, version16)
46+
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version16)
47+
48+
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
49+
return ClusterctlUpgradeSpecInput{
50+
E2EConfig: e2eConfig,
51+
ClusterctlConfigPath: clusterctlConfigPath,
52+
BootstrapClusterProxy: bootstrapClusterProxy,
53+
ArtifactFolder: artifactFolder,
54+
SkipCleanup: skipCleanup,
55+
InfrastructureProvider: ptr.To("docker"),
56+
// ### Versions for the initial deployment of providers ###
57+
InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease04),
58+
InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease04),
59+
InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
60+
InitWithControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
61+
InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease04)},
62+
InitWithRuntimeExtensionProviders: []string{},
63+
// ### Versions for the first upgrade of providers ### // FIXME: move upgrades to slice & add hook
64+
UpgradeWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease16),
65+
CoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease16),
66+
BootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
67+
ControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
68+
InfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease16)},
69+
RuntimeExtensionProviders: []string{},
70+
// Run a final upgrade to latest
71+
AdditionalUpgrade: true,
72+
73+
// Some notes about the version pinning: (FIXME)
74+
// We have to pin the providers because with `InitWithProvidersContract` the test would
75+
// use the latest version for the contract (which is v1.6.X for v1beta1).
76+
// We have to set this to an empty array as clusterctl v1.0 doesn't support
77+
// runtime extension providers. If we don't do this the test will automatically
78+
// try to deploy the latest version of our test-extension from docker.yaml.
79+
80+
// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.0/bases.
81+
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
82+
InitWithKubernetesVersion: "v1.23.17",
83+
WorkloadKubernetesVersion: "v1.23.17",
84+
MgmtFlavor: "topology",
85+
WorkloadFlavor: "",
86+
}
87+
})
88+
})
89+
3790
var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
3891
// Get v1.0 latest stable release
3992
version := "1.0"

‎test/e2e/config/docker.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,15 @@ providers:
3535
- name: cluster-api
3636
type: CoreProvider
3737
versions:
38+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
39+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
40+
type: "url"
41+
contract: v1alpha4
42+
replacements:
43+
- old: --metrics-addr=127.0.0.1:8080
44+
new: --metrics-addr=:8080
45+
files:
46+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
3847
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
3948
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
4049
type: "url"
@@ -73,6 +82,15 @@ providers:
7382
- name: kubeadm
7483
type: BootstrapProvider
7584
versions:
85+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
86+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
87+
type: "url"
88+
contract: v1alpha4
89+
replacements:
90+
- old: --metrics-addr=127.0.0.1:8080
91+
new: --metrics-addr=:8080
92+
files:
93+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
7694
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
7795
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
7896
type: "url"
@@ -111,6 +129,15 @@ providers:
111129
- name: kubeadm
112130
type: ControlPlaneProvider
113131
versions:
132+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
133+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
134+
type: "url"
135+
contract: v1alpha4
136+
replacements:
137+
- old: --metrics-addr=127.0.0.1:8080
138+
new: --metrics-addr=:8080
139+
files:
140+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
114141
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
115142
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
116143
type: "url"
@@ -149,6 +176,16 @@ providers:
149176
- name: docker
150177
type: InfrastructureProvider
151178
versions:
179+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
180+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
181+
type: "url"
182+
contract: v1alpha4
183+
replacements:
184+
- old: --metrics-addr=127.0.0.1:8080
185+
new: --metrics-addr=:8080
186+
files:
187+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
188+
- sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml"
152189
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
153190
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
154191
type: "url"

‎test/e2e/data/infrastructure-docker/v0.4/bases/cluster-with-kcp.yaml

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
---
2+
# DockerCluster object referenced by the Cluster object
3+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
4+
kind: DockerCluster
5+
metadata:
6+
name: '${CLUSTER_NAME}'
7+
---
8+
# Cluster object with
9+
# - Reference to the KubeadmControlPlane object
10+
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
11+
apiVersion: cluster.x-k8s.io/v1alpha4
12+
kind: Cluster
13+
metadata:
14+
name: '${CLUSTER_NAME}'
15+
labels:
16+
cni: "${CLUSTER_NAME}-crs-0"
17+
spec:
18+
clusterNetwork:
19+
services:
20+
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
21+
pods:
22+
cidrBlocks: ['${DOCKER_POD_CIDRS}']
23+
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
24+
infrastructureRef:
25+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
26+
kind: DockerCluster
27+
name: '${CLUSTER_NAME}'
28+
controlPlaneRef:
29+
kind: KubeadmControlPlane
30+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
31+
name: "${CLUSTER_NAME}-control-plane"
32+
---
33+
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
34+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
35+
kind: DockerMachineTemplate
36+
metadata:
37+
name: "${CLUSTER_NAME}-control-plane"
38+
spec:
39+
template:
40+
spec:
41+
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
42+
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
43+
extraMounts:
44+
- containerPath: "/var/run/docker.sock"
45+
hostPath: "/var/run/docker.sock"
46+
---
47+
# KubeadmControlPlane referenced by the Cluster object with
48+
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
49+
kind: KubeadmControlPlane
50+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
51+
metadata:
52+
name: "${CLUSTER_NAME}-control-plane"
53+
labels:
54+
kcp-adoption.step2: ""
55+
spec:
56+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
57+
machineTemplate:
58+
infrastructureRef:
59+
kind: DockerMachineTemplate
60+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
61+
name: "${CLUSTER_NAME}-control-plane"
62+
kubeadmConfigSpec:
63+
clusterConfiguration:
64+
controllerManager:
65+
extraArgs: {enable-hostpath-provisioner: 'true'}
66+
apiServer:
67+
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
68+
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
69+
initConfiguration:
70+
nodeRegistration:
71+
criSocket: unix:///var/run/containerd/containerd.sock
72+
kubeletExtraArgs:
73+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
74+
# This cluster is used in tests where the Kubernetes version is < 1.24
75+
cgroup-driver: cgroupfs
76+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
77+
fail-swap-on: "false"
78+
joinConfiguration:
79+
nodeRegistration:
80+
criSocket: unix:///var/run/containerd/containerd.sock
81+
kubeletExtraArgs:
82+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
83+
# This cluster is used in tests where the Kubernetes version is < 1.24
84+
cgroup-driver: cgroupfs
85+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
86+
fail-swap-on: "false"
87+
version: "${KUBERNETES_VERSION}"

‎test/e2e/data/infrastructure-docker/v0.4/bases/crs.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
# ConfigMap object referenced by the ClusterResourceSet object and with
3+
# the CNI resource defined in the test config file
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: "cni-${CLUSTER_NAME}-crs-0"
8+
data: ${CNI_RESOURCES}
9+
binaryData:
10+
---
11+
# ClusterResourceSet object with
12+
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
13+
apiVersion: addons.cluster.x-k8s.io/v1alpha4
14+
kind: ClusterResourceSet
15+
metadata:
16+
name: "${CLUSTER_NAME}-crs-0"
17+
spec:
18+
strategy: ApplyOnce
19+
clusterSelector:
20+
matchLabels:
21+
cni: "${CLUSTER_NAME}-crs-0"
22+
resources:
23+
- name: "cni-${CLUSTER_NAME}-crs-0"
24+
kind: ConfigMap

‎test/e2e/data/infrastructure-docker/v0.4/bases/md.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
---
2+
# DockerMachineTemplate referenced by the MachineDeployment and with
3+
# - extraMounts for the docker sock, thus allowing self-hosting test
4+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
5+
kind: DockerMachineTemplate
6+
metadata:
7+
name: "${CLUSTER_NAME}-md-0"
8+
spec:
9+
template:
10+
spec:
11+
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
12+
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
13+
extraMounts:
14+
- containerPath: "/var/run/docker.sock"
15+
hostPath: "/var/run/docker.sock"
16+
---
17+
# KubeadmConfigTemplate referenced by the MachineDeployment
18+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
19+
kind: KubeadmConfigTemplate
20+
metadata:
21+
name: "${CLUSTER_NAME}-md-0"
22+
spec:
23+
template:
24+
spec:
25+
joinConfiguration:
26+
nodeRegistration:
27+
criSocket: unix:///var/run/containerd/containerd.sock
28+
kubeletExtraArgs:
29+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
30+
# This cluster is used in tests where the Kubernetes version is < 1.24
31+
cgroup-driver: cgroupfs
32+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
33+
fail-swap-on: "false"
34+
---
35+
# MachineDeployment object
36+
apiVersion: cluster.x-k8s.io/v1alpha4
37+
kind: MachineDeployment
38+
metadata:
39+
name: "${CLUSTER_NAME}-md-0"
40+
spec:
41+
clusterName: "${CLUSTER_NAME}"
42+
replicas: ${WORKER_MACHINE_COUNT}
43+
selector:
44+
matchLabels:
45+
template:
46+
spec:
47+
clusterName: "${CLUSTER_NAME}"
48+
version: "${KUBERNETES_VERSION}"
49+
bootstrap:
50+
configRef:
51+
name: "${CLUSTER_NAME}-md-0"
52+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
53+
kind: KubeadmConfigTemplate
54+
infrastructureRef:
55+
name: "${CLUSTER_NAME}-md-0"
56+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
57+
kind: DockerMachineTemplate

‎test/e2e/data/infrastructure-docker/v0.4/cluster-template/kustomization.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
bases:
2+
- ../bases/cluster-with-kcp.yaml
3+
- ../bases/md.yaml
4+
- ../bases/crs.yaml

‎test/e2e/data/shared/v0.4/metadata.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
2+
kind: Metadata
3+
releaseSeries:
4+
- major: 0
5+
minor: 4
6+
contract: v1alpha4
7+
- major: 0
8+
minor: 3
9+
contract: v1alpha3
10+
- major: 0
11+
minor: 2
12+
contract: v1alpha2

‎test/e2e/quick_start_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ var _ = Describe("When following the Cluster API quick-start", func() {
6161
})
6262
})
6363

64-
var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", func() {
64+
var _ = Describe("When following the Cluster API quick-start with ClusterClass [ClusterClass]", func() {
6565
QuickStartSpec(ctx, func() QuickStartSpecInput {
6666
return QuickStartSpecInput{
6767
E2EConfig: e2eConfig,

0 commit comments

Comments
 (0)