Skip to content

Commit 2ce619a

Browse files
sbueringersbueringer
committed
Add e2e test to reproduce SSA apiVersion issue
Signed-off-by: Stefan Büringer [email protected]
1 parent 6249008 commit 2ce619a

File tree

13 files changed

+756
-210
lines changed

13 files changed

+756
-210
lines changed

‎Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -562,11 +562,15 @@ generate-doctoc:
562562
TRACE=$(TRACE) ./hack/generate-doctoc.sh
563563

564564
.PHONY: generate-e2e-templates
565-
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
565+
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
566566

567567
DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker
568568
INMEMORY_TEMPLATES := test/e2e/data/infrastructure-inmemory
569569

570+
.PHONY: generate-e2e-templates-v0.4
571+
generate-e2e-templates-v0.4: $(KUSTOMIZE)
572+
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml
573+
570574
.PHONY: generate-e2e-templates-v1.0
571575
generate-e2e-templates-v1.0: $(KUSTOMIZE)
572576
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.0/cluster-template.yaml

‎test/e2e/clusterctl_upgrade.go

Lines changed: 383 additions & 174 deletions
Large diffs are not rendered by default.

‎test/e2e/clusterctl_upgrade_test.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import (
2525
. "github.com/onsi/ginkgo/v2"
2626
. "github.com/onsi/gomega"
2727
"k8s.io/utils/ptr"
28+
29+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2830
)
2931

3032
var (
@@ -34,6 +36,60 @@ var (
3436
providerDockerPrefix = "docker:v%s"
3537
)
3638

39+
var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current) [PR-Blocking]", func() {
40+
// Get v0.4 latest stable release
41+
version04 := "0.4"
42+
stableRelease04, err := GetStableReleaseOfMinor(ctx, version04)
43+
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version04)
44+
45+
// Get v1.6 latest stable release
46+
version16 := "1.6"
47+
stableRelease16, err := GetStableReleaseOfMinor(ctx, version16)
48+
Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version16)
49+
50+
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
51+
return ClusterctlUpgradeSpecInput{
52+
E2EConfig: e2eConfig,
53+
ClusterctlConfigPath: clusterctlConfigPath,
54+
BootstrapClusterProxy: bootstrapClusterProxy,
55+
ArtifactFolder: artifactFolder,
56+
SkipCleanup: skipCleanup,
57+
InfrastructureProvider: ptr.To("docker"),
58+
// Configuration for the initial provider deployment.
59+
InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease04),
60+
// We have to pin the providers because with `InitWithProvidersContract` the test would
61+
// use the latest version for the contract (which is v1.3.X for v1beta1).
62+
InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease04),
63+
InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
64+
InitWithControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
65+
InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease04)},
66+
// We have to set this to an empty array as clusterctl v0.4 doesn't support
67+
// runtime extension providers. If we don't do this the test will automatically
68+
// try to deploy the latest version of our test-extension from docker.yaml.
69+
InitWithRuntimeExtensionProviders: []string{},
70+
// Configuration for the provider upgrades.
71+
Upgrades: []ClusterctlUpgradeSpecInputUpgrade{
72+
{ // Upgrade to 1.6.
73+
WithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease16),
74+
CoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease16),
75+
BootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
76+
ControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
77+
InfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease16)},
78+
},
79+
{ // Upgrade to latest v1beta1.
80+
Contract: clusterv1.GroupVersion.Version,
81+
},
82+
},
83+
// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.4/bases.
84+
// Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
85+
InitWithKubernetesVersion: "v1.23.17",
86+
WorkloadKubernetesVersion: "v1.23.17",
87+
MgmtFlavor: "topology",
88+
WorkloadFlavor: "",
89+
}
90+
})
91+
})
92+
3793
var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
3894
// Get v1.0 latest stable release
3995
version := "1.0"

‎test/e2e/config/docker.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,15 @@ providers:
3535
- name: cluster-api
3636
type: CoreProvider
3737
versions:
38+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
39+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
40+
type: "url"
41+
contract: v1alpha4
42+
replacements:
43+
- old: --metrics-addr=127.0.0.1:8080
44+
new: --metrics-addr=:8080
45+
files:
46+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
3847
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
3948
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
4049
type: "url"
@@ -73,6 +82,15 @@ providers:
7382
- name: kubeadm
7483
type: BootstrapProvider
7584
versions:
85+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
86+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
87+
type: "url"
88+
contract: v1alpha4
89+
replacements:
90+
- old: --metrics-addr=127.0.0.1:8080
91+
new: --metrics-addr=:8080
92+
files:
93+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
7694
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
7795
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
7896
type: "url"
@@ -111,6 +129,15 @@ providers:
111129
- name: kubeadm
112130
type: ControlPlaneProvider
113131
versions:
132+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
133+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
134+
type: "url"
135+
contract: v1alpha4
136+
replacements:
137+
- old: --metrics-addr=127.0.0.1:8080
138+
new: --metrics-addr=:8080
139+
files:
140+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
114141
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
115142
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
116143
type: "url"
@@ -149,6 +176,16 @@ providers:
149176
- name: docker
150177
type: InfrastructureProvider
151178
versions:
179+
- name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
180+
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
181+
type: "url"
182+
contract: v1alpha4
183+
replacements:
184+
- old: --metrics-addr=127.0.0.1:8080
185+
new: --metrics-addr=:8080
186+
files:
187+
- sourcePath: "../data/shared/v0.4/metadata.yaml"
188+
- sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml"
152189
- name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
153190
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
154191
type: "url"

‎test/e2e/data/infrastructure-docker/v0.4/bases/cluster-with-kcp.yaml

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
---
2+
# DockerCluster object referenced by the Cluster object
3+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
4+
kind: DockerCluster
5+
metadata:
6+
name: '${CLUSTER_NAME}'
7+
---
8+
# Cluster object with
9+
# - Reference to the KubeadmControlPlane object
10+
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
11+
apiVersion: cluster.x-k8s.io/v1alpha4
12+
kind: Cluster
13+
metadata:
14+
name: '${CLUSTER_NAME}'
15+
labels:
16+
cni: "${CLUSTER_NAME}-crs-0"
17+
spec:
18+
clusterNetwork:
19+
services:
20+
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
21+
pods:
22+
cidrBlocks: ['${DOCKER_POD_CIDRS}']
23+
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
24+
infrastructureRef:
25+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
26+
kind: DockerCluster
27+
name: '${CLUSTER_NAME}'
28+
controlPlaneRef:
29+
kind: KubeadmControlPlane
30+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
31+
name: "${CLUSTER_NAME}-control-plane"
32+
---
33+
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
34+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
35+
kind: DockerMachineTemplate
36+
metadata:
37+
name: "${CLUSTER_NAME}-control-plane"
38+
spec:
39+
template:
40+
spec:
41+
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
42+
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
43+
extraMounts:
44+
- containerPath: "/var/run/docker.sock"
45+
hostPath: "/var/run/docker.sock"
46+
---
47+
# KubeadmControlPlane referenced by the Cluster object with
48+
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
49+
kind: KubeadmControlPlane
50+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
51+
metadata:
52+
name: "${CLUSTER_NAME}-control-plane"
53+
labels:
54+
kcp-adoption.step2: ""
55+
spec:
56+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
57+
machineTemplate:
58+
infrastructureRef:
59+
kind: DockerMachineTemplate
60+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
61+
name: "${CLUSTER_NAME}-control-plane"
62+
kubeadmConfigSpec:
63+
clusterConfiguration:
64+
controllerManager:
65+
extraArgs: {enable-hostpath-provisioner: 'true'}
66+
apiServer:
67+
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
68+
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
69+
initConfiguration:
70+
nodeRegistration:
71+
criSocket: unix:///var/run/containerd/containerd.sock
72+
kubeletExtraArgs:
73+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
74+
# This cluster is used in tests where the Kubernetes version is < 1.24
75+
cgroup-driver: cgroupfs
76+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
77+
fail-swap-on: "false"
78+
joinConfiguration:
79+
nodeRegistration:
80+
criSocket: unix:///var/run/containerd/containerd.sock
81+
kubeletExtraArgs:
82+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
83+
# This cluster is used in tests where the Kubernetes version is < 1.24
84+
cgroup-driver: cgroupfs
85+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
86+
fail-swap-on: "false"
87+
version: "${KUBERNETES_VERSION}"

‎test/e2e/data/infrastructure-docker/v0.4/bases/crs.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
# ConfigMap object referenced by the ClusterResourceSet object and with
3+
# the CNI resource defined in the test config file
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: "cni-${CLUSTER_NAME}-crs-0"
8+
data: ${CNI_RESOURCES}
9+
binaryData:
10+
---
11+
# ClusterResourceSet object with
12+
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
13+
apiVersion: addons.cluster.x-k8s.io/v1alpha4
14+
kind: ClusterResourceSet
15+
metadata:
16+
name: "${CLUSTER_NAME}-crs-0"
17+
spec:
18+
strategy: ApplyOnce
19+
clusterSelector:
20+
matchLabels:
21+
cni: "${CLUSTER_NAME}-crs-0"
22+
resources:
23+
- name: "cni-${CLUSTER_NAME}-crs-0"
24+
kind: ConfigMap

‎test/e2e/data/infrastructure-docker/v0.4/bases/md.yaml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
---
2+
# DockerMachineTemplate referenced by the MachineDeployment and with
3+
# - extraMounts for the docker sock, thus allowing self-hosting test
4+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
5+
kind: DockerMachineTemplate
6+
metadata:
7+
name: "${CLUSTER_NAME}-md-0"
8+
spec:
9+
template:
10+
spec:
11+
# NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
12+
customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
13+
extraMounts:
14+
- containerPath: "/var/run/docker.sock"
15+
hostPath: "/var/run/docker.sock"
16+
---
17+
# KubeadmConfigTemplate referenced by the MachineDeployment
18+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
19+
kind: KubeadmConfigTemplate
20+
metadata:
21+
name: "${CLUSTER_NAME}-md-0"
22+
spec:
23+
template:
24+
spec:
25+
joinConfiguration:
26+
nodeRegistration:
27+
criSocket: unix:///var/run/containerd/containerd.sock
28+
kubeletExtraArgs:
29+
# We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
30+
# This cluster is used in tests where the Kubernetes version is < 1.24
31+
cgroup-driver: cgroupfs
32+
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
33+
fail-swap-on: "false"
34+
---
35+
# MachineDeployment object
36+
apiVersion: cluster.x-k8s.io/v1alpha4
37+
kind: MachineDeployment
38+
metadata:
39+
name: "${CLUSTER_NAME}-md-0"
40+
spec:
41+
clusterName: "${CLUSTER_NAME}"
42+
replicas: ${WORKER_MACHINE_COUNT}
43+
selector:
44+
matchLabels:
45+
template:
46+
spec:
47+
clusterName: "${CLUSTER_NAME}"
48+
version: "${KUBERNETES_VERSION}"
49+
bootstrap:
50+
configRef:
51+
name: "${CLUSTER_NAME}-md-0"
52+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
53+
kind: KubeadmConfigTemplate
54+
infrastructureRef:
55+
name: "${CLUSTER_NAME}-md-0"
56+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
57+
kind: DockerMachineTemplate

‎test/e2e/data/infrastructure-docker/v0.4/cluster-template/kustomization.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
bases:
2+
- ../bases/cluster-with-kcp.yaml
3+
- ../bases/md.yaml
4+
- ../bases/crs.yaml

‎test/e2e/data/shared/v0.4/metadata.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
2+
kind: Metadata
3+
releaseSeries:
4+
- major: 0
5+
minor: 4
6+
contract: v1alpha4
7+
- major: 0
8+
minor: 3
9+
contract: v1alpha3
10+
- major: 0
11+
minor: 2
12+
contract: v1alpha2

‎test/e2e/quick_start_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ var _ = Describe("When following the Cluster API quick-start", func() {
6161
})
6262
})
6363

64-
var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", func() {
64+
var _ = Describe("When following the Cluster API quick-start with ClusterClass [ClusterClass]", func() {
6565
QuickStartSpec(ctx, func() QuickStartSpecInput {
6666
return QuickStartSpecInput{
6767
E2EConfig: e2eConfig,

0 commit comments

Comments
 (0)