diff --git a/Makefile b/Makefile index 08c76db641..a8990b0568 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,8 @@ e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \ cluster-template-multi-network.yaml \ cluster-template-without-lb.yaml \ cluster-template.yaml \ - cluster-template-flatcar.yaml) + cluster-template-flatcar.yaml \ + cluster-template-k8s-upgrade.yaml) # Currently no templates that require CI artifacts # $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \ diff --git a/README.md b/README.md index 46a3fe3ed2..b683047e91 100644 --- a/README.md +++ b/README.md @@ -48,25 +48,28 @@ This provider's versions are compatible with the following versions of Cluster A This provider's versions are able to install and manage the following versions of Kubernetes: -| | v1.22 | v1.23 | v1.24 | v1.25 | v1.26 | v1.27 | v1.28 | -|------------------------------------| ----- | ----- | ----- | ----- | ----- | ----- | ----- | -| OpenStack Provider v1alpha5 (v0.6) | + | ✓ | ✓ | ✓ | + | + | + | -| OpenStack Provider v1alpha6 (v0.7) | + | ✓ | ✓ | ✓ | ✓ | ✓ | + | -| OpenStack Provider v1alpha7 | + | + | + | + | ✓ | ✓ | ✓ | +| | v1.25 | v1.26 | v1.27 | v1.28 | +|------------------------------------| ----- | ----- | ----- | ----- | +| OpenStack Provider v1alpha5 (v0.6) | ✓ | + | + | + | +| OpenStack Provider v1alpha6 (v0.7) | ✓ | ✓ | ✓ | + | +| OpenStack Provider v1alpha7 | + | ✓ | ✓ | ★ | This provider's versions are able to install Kubernetes to the following versions of OpenStack: | | Queens | Rocky | Stein | Train | Ussuri | Victoria | Wallaby | Xena | Yoga | |------------------------------------| ------ | ----- | ----- | ----- | ------ | -------- | ------- | ---- | ---- | -| OpenStack Provider v1alpha5 (v0.6) | + | + | + | + | + | ✓ | ✓ | ✓ | ✓ | -| OpenStack Provider v1alpha6 (v0.7) | + | + | + | + | + | ✓ | ✓ | ✓ | ✓ | -| OpenStack Provider v1alpha7 | | + | + | + | + | ✓ | ✓ | ✓ | ✓ | +| OpenStack Provider v1alpha5 (v0.6) | + | + | + | + | + | ✓ | ✓ | ✓ | ★ | +| OpenStack Provider v1alpha6 (v0.7) | + | + | + | + | + | ✓ | ✓ | ✓ | ★ | +| OpenStack Provider v1alpha7 | | + | + | + | + | ✓ | ✓ | ✓ | ★ | Test status: -- `✓` tested +- `★` currently testing +- `✓` previously tested - `+` should work, but we weren't able to test it +Older versions may also work but we have not verified. + Each version of Cluster API for OpenStack will attempt to support two Kubernetes versions. **NOTE:** As the versioning for this project is tied to the versioning of Cluster API, future modifications to this diff --git a/hack/ci/cloud-init/controller.yaml.tpl b/hack/ci/cloud-init/controller.yaml.tpl index ea373b5b7d..4b235cb692 100644 --- a/hack/ci/cloud-init/controller.yaml.tpl +++ b/hack/ci/cloud-init/controller.yaml.tpl @@ -62,6 +62,7 @@ # https://docs.openstack.org/glance/latest/admin/quotas.html IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2," IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img," + IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.27.2.img," IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.28.2.img," IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.28.2.img" diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 13d1e03dca..507cb13ad4 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -165,6 +165,12 @@ variables: # used to ensure we deploy to the correct management cluster KUBE_CONTEXT: "kind-capo-e2e" KUBERNETES_VERSION: "v1.28.2" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.27.2" + KUBERNETES_VERSION_UPGRADE_TO: "v1.28.2" + ETCD_VERSION_UPGRADE_TO: "3.5.9-0" + COREDNS_VERSION_UPGRADE_TO: "v1.10.1" + CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "upgrade-to-control-plane" + WORKERS_MACHINE_TEMPLATE_UPGRADE_TO: "upgrade-to-md-0" CNI: "../../data/cni/calico.yaml" CCM: "../../data/ccm/cloud-controller-manager.yaml" EXP_CLUSTER_RESOURCE_SET: "true" @@ -179,6 +185,7 @@ variables: OPENSTACK_FAILURE_DOMAIN: "testaz1" OPENSTACK_FAILURE_DOMAIN_ALT: "testaz2" OPENSTACK_IMAGE_NAME: "ubuntu-2204-kube-v1.28.2" + OPENSTACK_IMAGE_NAME_UPGRADE_FROM: "ubuntu-2204-kube-v1.27.2" OPENSTACK_NODE_MACHINE_FLAVOR: "m1.small" OPENSTACK_SSH_KEY_NAME: "cluster-api-provider-openstack-sigs-k8s-io" OPENSTACK_EXTERNAL_NETWORK_ID: "" diff --git a/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml b/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml index df7aa70474..07cf642d5d 100644 --- a/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml +++ b/test/e2e/data/kustomize/common-patches/ccm/kustomization.yaml @@ -6,9 +6,6 @@ kind: Component resources: - ccm.yaml -patchesStrategicMerge: -- patch-ccm.yaml - patches: - target: kind: OpenStackCluster @@ -18,3 +15,4 @@ patches: kind: KubeadmControlPlane name: \${CLUSTER_NAME}-control-plane path: patch-ccm-cloud-config.yaml +- path: patch-ccm.yaml diff --git a/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml b/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml index d01d1cb1e8..60fe867342 100644 --- a/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml +++ b/test/e2e/data/kustomize/common-patches/cni/kustomization.yaml @@ -6,11 +6,9 @@ kind: Component resources: - cni.yaml -patchesStrategicMerge: -- patch-cni.yaml - patches: - target: kind: OpenStackCluster name: \${CLUSTER_NAME} path: patch-cluster.yaml +- path: patch-cni.yaml diff --git a/test/e2e/data/kustomize/k8s-upgrade/kcp-patch.yaml b/test/e2e/data/kustomize/k8s-upgrade/kcp-patch.yaml new file mode 100644 index 0000000000..b3561dc8e1 --- /dev/null +++ b/test/e2e/data/kustomize/k8s-upgrade/kcp-patch.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-upgrade-from-control-plane diff --git a/test/e2e/data/kustomize/k8s-upgrade/kustomization.yaml b/test/e2e/data/kustomize/k8s-upgrade/kustomization.yaml new file mode 100644 index 0000000000..5a9e9011df --- /dev/null +++ b/test/e2e/data/kustomize/k8s-upgrade/kustomization.yaml @@ -0,0 +1,9 @@ +--- +resources: +- ../default +- upgrade-from-template.yaml +- upgrade-to-template.yaml + +patches: +- path: kcp-patch.yaml +- path: md-patch.yaml diff --git a/test/e2e/data/kustomize/k8s-upgrade/md-patch.yaml b/test/e2e/data/kustomize/k8s-upgrade/md-patch.yaml new file mode 100644 index 0000000000..bde877415b --- /dev/null +++ b/test/e2e/data/kustomize/k8s-upgrade/md-patch.yaml @@ -0,0 +1,11 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 + kind: OpenStackMachineTemplate + name: ${CLUSTER_NAME}-upgrade-from-md-0 diff --git a/test/e2e/data/kustomize/k8s-upgrade/upgrade-from-template.yaml b/test/e2e/data/kustomize/k8s-upgrade/upgrade-from-template.yaml new file mode 100644 index 0000000000..fe20fcc5f2 --- /dev/null +++ b/test/e2e/data/kustomize/k8s-upgrade/upgrade-from-template.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-upgrade-from-control-plane +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_IMAGE_NAME_UPGRADE_FROM} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} + tags: + - control-plane +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: ${CLUSTER_NAME}-upgrade-from-md-0 +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_IMAGE_NAME_UPGRADE_FROM} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} + tags: + - machine diff --git a/test/e2e/data/kustomize/k8s-upgrade/upgrade-to-template.yaml b/test/e2e/data/kustomize/k8s-upgrade/upgrade-to-template.yaml new file mode 100644 index 0000000000..734946741d --- /dev/null +++ b/test/e2e/data/kustomize/k8s-upgrade/upgrade-to-template.yaml @@ -0,0 +1,43 @@ +# This is just a copy of the normal cluster template OSMTs. +# It is needed because we need a static name that we can put in the e2e config. +# The normal templates has the cluster name as part of the OSMT name. +# See CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO and WORKERS_MACHINE_TEMPLATE_UPGRADE_TO +# in the e2e config. +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: upgrade-to-control-plane + labels: + clusterctl.cluster.x-k8s.io/move: "" +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} + tags: + - control-plane +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 +kind: OpenStackMachineTemplate +metadata: + name: upgrade-to-md-0 + labels: + clusterctl.cluster.x-k8s.io/move: "" +spec: + template: + spec: + cloudName: ${OPENSTACK_CLOUD} + flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR} + identityRef: + kind: Secret + name: ${CLUSTER_NAME}-cloud-config + image: ${OPENSTACK_IMAGE_NAME} + sshKeyName: ${OPENSTACK_SSH_KEY_NAME} + tags: + - machine diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 732c81952e..857bd134fa 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -53,6 +53,7 @@ const ( FlavorMDRemediation = "md-remediation" FlavorKCPRemediation = "kcp-remediation" FlavorFlatcar = "flatcar" + FlavorKubernetesUpgrade = "k8s-upgrade" ) // DefaultScheme returns the default scheme to use for testing. diff --git a/test/e2e/suites/e2e/self_hosted_test.go b/test/e2e/suites/e2e/self_hosted_test.go index 4a7e4a3770..d3160077c9 100644 --- a/test/e2e/suites/e2e/self_hosted_test.go +++ b/test/e2e/suites/e2e/self_hosted_test.go @@ -30,23 +30,16 @@ import ( var _ = Describe("When testing Cluster API provider Openstack working on [self-hosted] clusters", func() { ctx := context.TODO() - shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false) shared.SetEnvVar("DOWNLOAD_E2E_IMAGE", "true", false) - // TODO(lentzi90): Since we currently rely on USE_CI_ARTIFACTS to get kubernetes set up, - // we are stuck with one version only and cannot do the k8s upgrade that is part of the - // SelfHostedSpec. The reason for this is that the script used to install kubernetes tools comes - // with the KubeadmControlPlane and that does not change when upgrading the k8s version - // (only the machineTemplate changes). - // Until we have a way to work around this, the k8s upgrade test is disabled. capie2e.SelfHostedSpec(ctx, func() capie2e.SelfHostedSpecInput { return capie2e.SelfHostedSpecInput{ E2EConfig: e2eCtx.E2EConfig, ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, ArtifactFolder: e2eCtx.Settings.ArtifactFolder, - SkipUpgrade: true, + SkipUpgrade: false, SkipCleanup: false, - Flavor: shared.FlavorDefault, + Flavor: shared.FlavorKubernetesUpgrade, } }) })