Sync terraform EKS module with the reference architecture

.werft/installer-tests.ts

@@ -269,7 +269,7 @@ export async function installerTests(config: TestConfig) {
     console.log(config.DESCRIPTION);
     // these phases sets up or clean up the infrastructure
     // If the cloud variable is not set, we have a cleanup job in hand
-    const majorPhase: string = cloud == "" ? `create-${cloud}-infra` : "cleanup-infra"
+    const majorPhase: string = cloud == "" ? "cleanup-infra" :`create-${cloud}-infra`
 
     werft.phase(majorPhase, `Manage the infrastructure in ${cloud}`);
     for (let phase of config.PHASES) {
@@ -411,7 +411,7 @@ function randK8sVersion(config: string): string {
             break;
         }
         case "STANDARD_EKS_TEST": {
-            options = ["1.20", "1.21", "1.22"]
+            options = ["1.21", "1.22"]
             break;
         }
         case "STANDARD_K3S_TEST": {

install/infra/modules/eks/database.tf

@@ -0,0 +1,50 @@
+resource "random_password" "password" {
+  count = var.create_external_database ? 1 : 0
+
+  length           = 16
+  special          = true
+  override_special = "!#$%&*()-_=+[]{}<>:?"
+}
+
+resource "aws_db_subnet_group" "gitpod_subnets" {
+  count = var.create_external_database ? 1 : 0
+
+  name       = "db-sg-${var.cluster_name}"
+  subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]]
+}
+
+resource "aws_security_group" "rdssg" {
+  count = var.create_external_database ? 1 : 0
+
+  name   = "dh-sg-${var.cluster_name}"
+  vpc_id = module.vpc.vpc_id
+}
+
+resource "aws_security_group_rule" "db-ingress-nodes" {
+  description       = "Allow nodes to communicate with the db"
+  from_port         = 0
+  protocol          = "tcp"
+  cidr_blocks       = [var.vpc_cidr]
+  security_group_id = aws_security_group.rdssg[0].id
+  to_port           = 3306
+  type              = "ingress"
+}
+
+resource "aws_db_instance" "gitpod" {
+  count = var.create_external_database ? 1 : 0
+
+  allocated_storage      = 20
+  max_allocated_storage  = 120
+  engine                 = "mysql"
+  engine_version         = "5.7"
+  instance_class         = "db.m5.large"
+  vpc_security_group_ids = [aws_security_group.rdssg[0].id]
+  identifier             = "db-${var.cluster_name}"
+  name                   = "gitpod"
+  username               = "gitpod"
+  password               = random_password.password[0].result
+  parameter_group_name   = "default.mysql5.7"
+  db_subnet_group_name   = aws_db_subnet_group.gitpod_subnets[0].name
+  skip_final_snapshot    = true
+  publicly_accessible    = false
+}

install/infra/terraform/eks/kubernetes.tf → install/infra/modules/eks/kubernetes.tf

@@ -1,27 +1,61 @@
+locals {
+  private_primary_subnet_cidr   = cidrsubnet(var.vpc_cidr, 7, 0)
+  private_secondary_subnet_cidr = cidrsubnet(var.vpc_cidr, 7, 1)
+  public_primary_subnet_cidr    = cidrsubnet(var.vpc_cidr, 7, 2)
+  public_secondary_subnet_cidr  = cidrsubnet(var.vpc_cidr, 7, 3)
+  public_db_subnet_cidr_1       = cidrsubnet(var.vpc_cidr, 7, 4)
+  public_db_subnet_cidr_2       = cidrsubnet(var.vpc_cidr, 7, 5)
+}
+
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "3.12.0"
 
   name                 = "vpc-${var.cluster_name}"
   cidr                 = var.vpc_cidr
   azs                  = var.vpc_availability_zones
-  private_subnets      = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
-  public_subnets       = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
+  private_subnets      = [local.private_primary_subnet_cidr, local.private_secondary_subnet_cidr]
+  public_subnets       = [local.public_primary_subnet_cidr, local.public_secondary_subnet_cidr, local.public_db_subnet_cidr_1, local.public_db_subnet_cidr_2]
   enable_nat_gateway   = true
   enable_dns_hostnames = true
 }
 
+resource "aws_security_group_rule" "eks-worker-ingress-self" {
+  description              = "Allow node to communicate with each other"
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes.id
+  source_security_group_id = aws_security_group.nodes.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "eks-worker-ingress-cluster" {
+  description              = "Allow worker Kubelets and pods to receive communication from the cluster control plane"
+  from_port                = 1025
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.nodes.id
+  source_security_group_id = aws_security_group.nodes.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+### Worker Node Access to EKS Master
+resource "aws_security_group_rule" "eks-cluster-ingress-node-https" {
+  description              = "Allow pods to communicate with the cluster API Server"
+  from_port                = 443
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.nodes.id
+  source_security_group_id = aws_security_group.nodes.id
+  to_port                  = 443
+  type                     = "ingress"
+}
+
+
 resource "aws_security_group" "nodes" {
   name   = "nodes-sg-${var.cluster_name}"
   vpc_id = module.vpc.vpc_id
 
-  ingress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
   egress {
     from_port   = 0
     to_port     = 0
@@ -37,7 +71,8 @@ module "eks" {
   cluster_name    = var.cluster_name
   cluster_version = var.cluster_version
 
-  cluster_endpoint_public_access = true
+  cluster_endpoint_public_access  = true
+  cluster_endpoint_private_access = true
 
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.public_subnets
@@ -59,60 +94,77 @@ module "eks" {
     ami_id                     = var.image_id
     enable_bootstrap_user_data = true
     vpc_security_group_ids     = [aws_security_group.nodes.id]
+    ebs_optimized              = true
   }
 
   eks_managed_node_groups = {
     Services = {
       enable_bootstrap_user_data = true
-      instance_types = [var.service_machine_type]
-      name = "service-${var.cluster_name}"
-      subnet_ids   = module.vpc.public_subnets
-      min_size     = 1
-      max_size     = 10
-      desired_size = 1
-      block_device_mappings =  [{
+      instance_types             = [var.service_machine_type]
+      name                       = "service-${var.cluster_name}"
+      subnet_ids                 = module.vpc.public_subnets
+      min_size                   = 1
+      max_size                   = 4
+      desired_size               = 2
+      block_device_mappings = [{
         device_name = "/dev/sda1"
 
-        ebs =  [{
-          volume_size = 150
+        ebs = [{
+          volume_size           = 300
+          volume_type           = "gp3"
+          throughput            = 500
+          iops                  = 6000
+          delete_on_termination = true
         }]
       }]
       labels = {
         "gitpod.io/workload_meta" = true
         "gitpod.io/workload_ide"  = true
       }
+
+      tags = {
+        "k8s.io/cluster-autoscaler/enabled" = true
+        "k8s.io/cluster-autoscaler/gitpod"  = "owned"
+      }
+
       pre_bootstrap_user_data = <<-EOT
         #!/bin/bash
         set -ex
         cat <<-EOF > /etc/profile.d/bootstrap.sh
         export CONTAINER_RUNTIME="containerd"
         export USE_MAX_PODS=false
         EOF
-        # Source extra environment variables in bootstrap script
+        # Source extra environment 5ariables in bootstrap script
         sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
         EOT
     }
 
     Workspaces = {
       instance_types = [var.workspace_machine_type]
-      name = "ws-${var.cluster_name}"
-      subnet_ids   = module.vpc.public_subnets
-      min_size     = 1
-      max_size     = 10
-      block_device_mappings =  [{
+      name           = "ws-${var.cluster_name}"
+      subnet_ids     = module.vpc.public_subnets
+      min_size       = 1
+      max_size       = 50
+      block_device_mappings = [{
         device_name = "/dev/sda1"
 
-        ebs =  [{
-          volume_size = 150
+        ebs = [{
+          volume_size = 300
         }]
       }]
-      desired_size = 1
+      desired_size               = 2
       enable_bootstrap_user_data = true
       labels = {
         "gitpod.io/workload_workspace_services" = true
         "gitpod.io/workload_workspace_regular"  = true
         "gitpod.io/workload_workspace_headless" = true
       }
+
+      tags = {
+        "k8s.io/cluster-autoscaler/enabled" = true
+        "k8s.io/cluster-autoscaler/gitpod"  = "owned"
+      }
+
       pre_bootstrap_user_data = <<-EOT
         #!/bin/bash
         set -ex

install/infra/modules/eks/output.tf

@@ -0,0 +1,83 @@
+output "external_dns_settings" {
+  value = [
+    {
+      "name"  = "provider",
+      "value" = "aws"
+    },
+    {
+      "name"  = "aws.region",
+      "value" = var.region
+    },
+    {
+      "name"  = "aws.credentials.secretKey",
+      "value" = aws_iam_access_key.edns.secret
+    },
+    {
+      "name"  = "aws.credentials.accessKey",
+      "value" = aws_iam_access_key.edns.id
+    }
+  ]
+}
+
+output "secretAccessKey" {
+  sensitive = true
+  value     = try("${aws_iam_access_key.edns.secret}", "")
+}
+
+output "cert_manager_issuer" {
+  value = try({
+    region = var.region
+    secretAccessKeySecretRef = {
+      name = "route53-credentials"
+      key  = "secret-access-key"
+    }
+
+    hostedZoneID = aws_route53_zone.gitpod.zone_id
+    accessKeyID  = aws_iam_access_key.edns.id
+  }, {})
+}
+
+output "domain_nameservers" {
+  value = formatlist("%s.", resource.aws_route53_zone.gitpod.name_servers)
+}
+
+output "database" {
+  sensitive = true
+  value = try({
+    host     = "${aws_db_instance.gitpod[0].address}"
+    username = "${aws_db_instance.gitpod[0].username}"
+    password = random_password.password[0].result
+    port     = 3306
+  }, "No database created")
+}
+
+output "registry" {
+  sensitive = true
+  value = try({
+    server   = aws_ecr_repository.gitpod[0].repository_url
+    username = data.aws_ecr_authorization_token.gitpod[0].user_name
+    password = data.aws_ecr_authorization_token.gitpod[0].password
+  }, "No EKS registry created")
+}
+
+output "storage" {
+  sensitive = true
+  value = try({
+    region            = aws_s3_bucket.gitpod-storage[0].region
+    endpoint          = "s3.${aws_s3_bucket.gitpod-storage[0].region}.amazonaws.com"
+    bucket_name       = aws_s3_bucket.gitpod-storage[0].id
+    access_key_id     = aws_iam_access_key.bucket_storage_user[0].id
+    secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret
+  }, "No s3 bucket created for object storage")
+}
+
+output "registry_backend" {
+  sensitive = true
+  value = try({
+    region            = aws_s3_bucket.gitpod-registry-backend[0].region
+    endpoint          = "s3.${aws_s3_bucket.gitpod-registry-backend[0].region}.amazonaws.com"
+    bucket_name       = aws_s3_bucket.gitpod-registry-backend[0].id
+    access_key_id     = aws_iam_access_key.bucket_registry_user[0].id
+    secret_access_key = aws_iam_access_key.bucket_registry_user[0].secret
+  }, "No s3 bucket created for registry backend.")
+}

install/infra/terraform/eks/registry.tf → install/infra/modules/eks/registry.tf

@@ -1,4 +1,6 @@
 resource "aws_ecr_repository" "gitpod" {
+  count = var.create_external_registry ? 1 : 0
+
   name                 = "registry-${var.cluster_name}"
   image_tag_mutability = "MUTABLE"
 
@@ -8,5 +10,6 @@ resource "aws_ecr_repository" "gitpod" {
 }
 
 data "aws_ecr_authorization_token" "gitpod" {
-  registry_id = aws_ecr_repository.gitpod.registry_id
+  count       = var.create_external_registry ? 1 : 0
+  registry_id = aws_ecr_repository.gitpod[0].registry_id
 }