[terraform/aks] Implement separate nodepool types, node pool max sizes, machine types

install/infra/modules/aks/kubernetes.tf

@@ -26,8 +26,8 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   http_application_routing_enabled = false
 
   default_node_pool {
-    name    = local.nodes.0.name
-    vm_size = local.machine
+    name    = "services"
+    vm_size = var.services_machine_type
 
 
     node_taints = []
@@ -36,9 +36,13 @@ resource "azurerm_kubernetes_cluster" "k8s" {
 
     enable_auto_scaling  = true
     min_count            = 1
-    max_count            = 10
+    max_count            = var.max_node_count_services
     orchestrator_version = var.cluster_version
-    node_labels          = local.nodes.0.labels
+    node_labels = {
+      "gitpod.io/workload_meta"               = true
+      "gitpod.io/workload_ide"                = true
+      "gitpod.io/workload_workspace_services" = true
+    }
 
     type           = "VirtualMachineScaleSets"
     vnet_subnet_id = azurerm_subnet.network.id
@@ -59,18 +63,29 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   }
 }
 
-resource "azurerm_kubernetes_cluster_node_pool" "pools" {
-  count = length(local.nodes) - 1
+resource "azurerm_kubernetes_cluster_node_pool" "regularws" {
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id
+  name                  = "regularws"
+  vm_size               = var.workspaces_machine_type
+
+  enable_auto_scaling  = true
+  min_count            = 1
+  max_count            = var.max_node_count_regular_workspaces
+  orchestrator_version = var.cluster_version
+  node_labels          = { "gitpod.io/workload_workspace_regular" = true }
+  vnet_subnet_id       = azurerm_subnet.network.id
+}
 
+resource "azurerm_kubernetes_cluster_node_pool" "headlessws" {
   kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id
-  name                  = local.nodes[count.index + 1].name
-  vm_size               = local.machine
+  name                  = "headlessws"
+  vm_size               = var.workspaces_machine_type
 
   enable_auto_scaling  = true
   min_count            = 1
-  max_count            = 10
+  max_count            = var.max_node_count_headless_workspaces
   orchestrator_version = var.cluster_version
-  node_labels          = local.nodes[count.index + 1].labels
+  node_labels          = { "gitpod.io/workload_workspace_headless" = true }
   vnet_subnet_id       = azurerm_subnet.network.id
 }
 
@@ -82,7 +97,8 @@ data "azurerm_resources" "k8s" {
 
   depends_on = [
     azurerm_kubernetes_cluster.k8s,
-    azurerm_kubernetes_cluster_node_pool.pools
+    azurerm_kubernetes_cluster_node_pool.regularws,
+    azurerm_kubernetes_cluster_node_pool.headlessws,
   ]
 }
 
@@ -107,7 +123,8 @@ resource "azurerm_network_security_rule" "k8s" {
 
 resource "local_file" "kubeconfig" {
   depends_on = [
-    resource.azurerm_kubernetes_cluster_node_pool.pools,
+    resource.azurerm_kubernetes_cluster_node_pool.regularws,
+    resource.azurerm_kubernetes_cluster_node_pool.headlessws,
   ]
   filename = var.kubeconfig
   content  = azurerm_kubernetes_cluster.k8s.kube_config_raw

install/infra/modules/aks/local.tf

@@ -1,11 +1,4 @@
 locals {
-  labels = tomap({
-    workload_meta : "gitpod.io/workload_meta"
-    workload_ide : "gitpod.io/workload_ide"
-    workspace_services : "gitpod.io/workload_workspace_services"
-    workspace_regular : "gitpod.io/workload_workspace_regular"
-    workspace_headless : "gitpod.io/workload_workspace_headless"
-  })
   dns_enabled = var.domain_name != null
 
   name_format = join("-", [
@@ -75,16 +68,4 @@ locals {
       priority                   = 4096
     }
   ] : []
-  nodes = [
-    {
-      name = "services"
-      labels = {
-        lookup(local.labels, "workload_meta")      = true
-        lookup(local.labels, "workload_ide")       = true
-        lookup(local.labels, "workspace_services") = true
-        lookup(local.labels, "workspace_regular")  = true
-        lookup(local.labels, "workspace_headless") = true
-      }
-    }
-  ]
 }

install/infra/modules/aks/variables.tf

@@ -1,23 +1,62 @@
 // Common variables
 variable "kubeconfig" {
+  description = "Path to write the kubeconfig output to"
   default = "./kubeconfig"
 }
 
 variable "cluster_version" {
-  description = "kubernetes version of to create the cluster with"
+  description = "The AKS Kubernetes version"
+}
+
+variable "domain_name" {
+  description = "The domain name where Gitpod will create DNS records. Leave blank to disable DNS management."
+  type = string
 }
 
-variable "domain_name" {}
 variable "enable_airgapped" {
   default = false
 }
 
 variable "create_external_database" {}
 variable "create_external_registry" {}
 variable "create_external_storage" {}
-variable "resource_group_name" {}
+
+variable "resource_group_name" {
+  description = "The resource group where Gitpod resources will be created."
+  type = string
+}
 
 // Azure-specific variables
 variable "location" {
   default = "northeurope"
 }
+
+variable "max_node_count_regular_workspaces" {
+  type        = number
+  description = "Maximum number of nodes in the regular workspaces NodePool. Must be >= 1."
+  default     = 50
+}
+
+variable "max_node_count_headless_workspaces" {
+  type        = number
+  description = "Maximum number of nodes in the headless workspaces NodePool. Must be >= 1."
+  default     = 50
+}
+
+variable "max_node_count_services" {
+  type        = number
+  description = "Maximum number of nodes in the services NodePool. Must be >= 1."
+  default     = 4
+}
+
+variable "workspaces_machine_type" {
+  type        = string
+  description = "The regular and headless node machine type."
+  default     = "Standard_D8_v4"
+}
+
+variable "services_machine_type" {
+  type        = string
+  description = "The services node machine type."
+  default     = "Standard_D4_v4"
+}