Skip to content

[installer-tests] Support tests against multiple version of k8s and ubuntu images #11568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 1, 2022
Merged

[installer-tests] Support tests against multiple version of k8s and ubuntu images #11568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 60 additions & 21 deletions .werft/installer-tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@ interface TestConfig {
CLOUD: string;
}

const k8s_version: string = randK8sVersion(testConfig)
const os_version: string = randOsVersion() // applicable only for k3s

// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_GKE_TEST: {
CLOUD: "gcp",
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
DESCRIPTION: `Deploy Gitpod on GKE(version ${k8s_version})`,
PHASES: [
"STANDARD_GKE_CLUSTER",
"CERT_MANAGER",
Expand All @@ -52,8 +55,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_K3S_TEST: {
CLOUD: "gcp", // the cloud provider is still GCP
DESCRIPTION:
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
" with managed DNS and run integrations tests",
`Deploy Gitpod on a K3s cluster(version ${k8s_version}), on a GCP instance with ubuntu ${os_version}`,
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"CERT_MANAGER",
Expand All @@ -65,7 +67,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
},
STANDARD_AKS_TEST: {
CLOUD: "azure",
DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
DESCRIPTION: `Deploy Gitpod on AKS(version ${k8s_version})`,
PHASES: [
"STANDARD_AKS_CLUSTER",
"CERT_MANAGER",
Expand All @@ -79,7 +81,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
},
STANDARD_EKS_TEST: {
CLOUD: "aws",
DESCRIPTION: "Create an EKS cluster",
DESCRIPTION: `Create an EKS cluster(version ${k8s_version})`,
PHASES: [
"STANDARD_EKS_CLUSTER",
"CERT_MANAGER",
Expand Down Expand Up @@ -110,23 +112,23 @@ const cloud: string = config.CLOUD;
const INFRA_PHASES: { [name: string]: InfraConfig } = {
STANDARD_GKE_CLUSTER: {
phase: "create-std-gke-cluster",
makeTarget: "gke-standard-cluster",
description: "Creating a GKE cluster with 1 nodepool each for workspace and server",
makeTarget: `gke-standard-cluster`,
description: `Creating a GCP GKE cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
},
STANDARD_K3S_CLUSTER_ON_GCP: {
phase: "create-std-k3s-cluster",
makeTarget: "k3s-standard-cluster",
description: "Creating a k3s cluster on GCP with 1 node",
makeTarget: `k3s-standard-cluster os_version=${os_version}`,
description: `Creating a k3s(version: ${k8s_version}) cluster on GCP with 1 node`,
},
STANDARD_AKS_CLUSTER: {
phase: "create-std-aks-cluster",
makeTarget: "aks-standard-cluster",
description: "Creating an aks cluster(azure)",
makeTarget: `aks-standard-cluster`,
description: `Creating an Azure AKS cluster(version: ${k8s_version})`,
},
STANDARD_EKS_CLUSTER: {
phase: "create-std-eks-cluster",
makeTarget: "eks-standard-cluster",
description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
makeTarget: `eks-standard-cluster`,
description: `Creating a AWS EKS cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
},
CERT_MANAGER: {
phase: "setup-cert-manager",
Expand All @@ -140,7 +142,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
},
GENERATE_KOTS_CONFIG: {
phase: "generate-kots-config",
makeTarget: `generate-kots-config storage=${randomize()} registry=${randomize()} db=${randomize()}`,
makeTarget: `generate-kots-config storage=${randDeps()} registry=${randDeps()} db=${randDeps()}`,
description: `Generate KOTS Config file`,
},
CLUSTER_ISSUER: {
Expand Down Expand Up @@ -335,7 +337,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
werft.log(phase, `Calling ${makeTarget}`);

// exporting cloud env var is important for the make targets
const response = exec(`export cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
const response = exec(`export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
slice: phase,
dontCheckRc: true,
});
Expand All @@ -347,24 +349,61 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
werft.fail(phase, "Operation failed");
return response.code;
}
werft.log(phase, `Phase failed`);
werft.log(phase, `'${description}' failed`);
} else {
werft.log(phase, `Phase succeeded`);
werft.log(phase, `'${description}' succeeded`);
werft.done(phase);
}

return response.code;
}

function randomize(): string {
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform
function randomize(options: string[]): string {
return options[Math.floor(Math.random() * options.length)];
}

function randDeps(): string {
var depOptions: string[] = ["incluster", "external"]

if(deps && depOptions.includes(deps)) {
return deps
}

return depOptions[Math.floor(Math.random() * depOptions.length)];
return randomize(depOptions)
}

function randK8sVersion(config: string): string {
var options: string[] = []
switch(config) {
case "STANDARD_GKE_TEST": {
options = ["1.21", "1.22", "1.23"]
break;
}
case "STANDARD_AKS_TEST": {
options = ["1.21", "1.22", "1.23"]
break;
}
case "STANDARD_EKS_TEST": {
options = ["1.20", "1.21", "1.22"]
break;
}
case "STANDARD_K3S_TEST": {
options = ["v1.22.12+k3s1", "v1.23.9+k3s1", "v1.24.3+k3s1"]
break;
}
}
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform

return randomize(options)
}

function randOsVersion(): string {
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform
var options: string[] = ["2204", "2004", "1804"]

return randomize(options)
}

function cleanup() {
Expand Down
2 changes: 1 addition & 1 deletion .werft/jobs/build/self-hosted-upgrade-tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern

werft.phase(upgradeConfig.phase, upgradeConfig.description);

annotation = `${annotation} -a cluster=${phase}`
annotation = `${annotation} -a cluster=${phase} -a updateGitHubStatus=gitpod-io/gitpod`

const testFile: string = ".werft/self-hosted-installer-tests.yaml";

Expand Down
33 changes: 14 additions & 19 deletions install/infra/terraform/aks/kubernetes.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
data "azurerm_kubernetes_service_versions" "k8s" {
location = azurerm_resource_group.gitpod.location
include_preview = false
}

resource "azurerm_role_assignment" "k8s" {
count = var.dns_enabled ? 1 : 0

Expand All @@ -20,37 +15,37 @@ resource "azurerm_role_assignment" "k8s_reader" {
}

resource "azurerm_kubernetes_cluster" "k8s" {
name = format(local.name_format, local.location, "primary")
location = azurerm_resource_group.gitpod.location
resource_group_name = azurerm_resource_group.gitpod.name
dns_prefix = "gitpod"
tags = {}
api_server_authorized_ip_ranges = []

kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
name = format(local.name_format, local.location, "primary")
location = azurerm_resource_group.gitpod.location
resource_group_name = azurerm_resource_group.gitpod.name
dns_prefix = "gitpod"
tags = {}
api_server_authorized_ip_ranges = []

kubernetes_version = var.cluster_version
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a feeling that this may break in time. Azure tend to remove versions of k8s at fairly short notice, which is why the data "azurerm_kubernetes_service_versions" is the recommended way of getting the versions. You can use a version_prefix as well

My suggestion is to keep this change as you've done it, but do so in the knowledge that we may have to change it in the future.

http_application_routing_enabled = false

default_node_pool {
name = local.nodes.0.name
vm_size = local.machine


node_taints = []
tags = {}
zones = []
node_taints = []
tags = {}
zones = []

enable_auto_scaling = true
min_count = 1
max_count = 10
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
orchestrator_version = var.cluster_version
node_labels = local.nodes.0.labels

type = "VirtualMachineScaleSets"
vnet_subnet_id = azurerm_subnet.network.id
}

identity {
type = "SystemAssigned"
type = "SystemAssigned"
identity_ids = []
}

Expand All @@ -74,7 +69,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" {
enable_auto_scaling = true
min_count = 1
max_count = 10
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
orchestrator_version = var.cluster_version
node_labels = local.nodes[count.index + 1].labels
vnet_subnet_id = azurerm_subnet.network.id
}
Expand Down
9 changes: 6 additions & 3 deletions install/infra/terraform/aks/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
// Common variables
variable "kubeconfig" {
default = "./kubeconfig"
default = "./kubeconfig"
}

variable "cluster_version" {
description = "kubernetes version of to create the cluster with"
}

variable "dns_enabled" {}
variable "domain_name" {}
variable "enable_airgapped" {}
Expand All @@ -14,6 +18,5 @@ variable "workspace_name" {

// Azure-specific variables
variable "location" {
default = "northeurope"

default = "northeurope"
}
36 changes: 18 additions & 18 deletions install/infra/terraform/eks/kubernetes.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,30 @@ module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "3.12.0"

name = "vpc-${var.cluster_name}"
cidr = var.vpc_cidr
azs = var.vpc_availability_zones
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
enable_nat_gateway = true
name = "vpc-${var.cluster_name}"
cidr = var.vpc_cidr
azs = var.vpc_availability_zones
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
enable_nat_gateway = true
enable_dns_hostnames = true
}

resource "aws_security_group" "nodes" {
name = "nodes-sg-${var.cluster_name}"
name = "nodes-sg-${var.cluster_name}"
vpc_id = module.vpc.vpc_id

ingress {
from_port = 0
to_port = 0
protocol = "-1"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}

egress {
from_port = 0
to_port = 0
protocol = "-1"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
Expand All @@ -34,10 +34,10 @@ module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "18.8.1"

cluster_name = var.cluster_name
cluster_version = "1.22"
cluster_name = var.cluster_name
cluster_version = var.cluster_version

cluster_endpoint_public_access = true
cluster_endpoint_public_access = true

vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.public_subnets
Expand All @@ -58,7 +58,7 @@ module "eks" {
iam_role_attach_cni_policy = true
ami_id = var.image_id
enable_bootstrap_user_data = true
vpc_security_group_ids = [aws_security_group.nodes.id]
vpc_security_group_ids = [aws_security_group.nodes.id]
}

eks_managed_node_groups = {
Expand Down Expand Up @@ -144,7 +144,7 @@ module "vpc_cni_irsa" {
}

resource "null_resource" "kubeconfig" {
depends_on = [ module.eks ]
depends_on = [module.eks]
provisioner "local-exec" {
command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
}
Expand Down
21 changes: 14 additions & 7 deletions install/infra/terraform/eks/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,45 @@ variable "cluster_name" {
type = string
description = "EKS cluster name."
}

variable "cluster_version" {
type = string
description = "Kubernetes version to create the cluster with"
default = "1.22"
}

variable "kubeconfig" {
type = string
description = "Path to the kubeconfig file"
default = "kubeconfig"
}

variable "image_id" {
type = string
type = string
description = "AMI Image ID specific to the region"
// latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/
default = "ami-0793b4124359a6ad7"
}

variable "service_machine_type" {
type = string
type = string
description = "Machine type for service workload node pool"
default = "m6i.xlarge"
default = "m6i.xlarge"
}

variable "workspace_machine_type" {
type = string
type = string
description = "Machine type for workspace workload node pool"
default = "m6i.2xlarge"
default = "m6i.2xlarge"
}

variable "region" {
type = string
type = string
default = "eu-west-1"
}

variable "vpc_availability_zones" {
type = list(string)
type = list(string)
default = ["eu-west-1c", "eu-west-1b"]
}

Expand Down
Loading