From 14a50db399b1998bd71a12d49d462f1eed23ee0a Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Fri, 22 Jul 2022 10:41:58 +0000 Subject: [PATCH 1/9] [installer-tests] Add tests for k8s versions and ubuntu version --- install/tests/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install/tests/Makefile b/install/tests/Makefile index 2e742992ce2a43..9f3520e5ef8500 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -74,6 +74,12 @@ ami_id_121 := "ami-0d57fb01036fac543" ami_id_122 := "ami-0b306cb7e98db98e4" +ami_id_120 := "ami-0ecb917eb4fbf4387" + +ami_id_121 := "ami-0d57fb01036fac543" + +ami_id_122 := "ami-0b306cb7e98db98e4" + .PHONY: ## eks-standard-cluster: Creates an EKS cluster eks-standard-cluster: ami_id = $(if $(ami_id_${TF_VAR_cluster_version//.}),$(ami_id_${TF_VAR_cluster_version//.}),$(ami_id_122)) From b848290ca6468f0c9e4160c6446a4e04197f1db0 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 26 Jul 2022 13:08:44 +0000 Subject: [PATCH 2/9] Add terraform configurations for single-cluster ref arch --- .../{terraform => modules}/aks/README.md | 0 .../{terraform => modules}/aks/database.tf | 0 .../{terraform => modules}/aks/kubernetes.tf | 0 .../infra/{terraform => modules}/aks/local.tf | 0 .../infra/{terraform => modules}/aks/main.tf | 0 .../{terraform => modules}/aks/monitoring.tf | 0 .../{terraform => modules}/aks/networks.tf | 0 .../{terraform => modules}/aks/output.tf | 0 .../{terraform => modules}/aks/registry.tf | 0 .../{terraform => modules}/aks/storage.tf | 0 .../{terraform => modules}/aks/variables.tf | 0 install/infra/modules/eks/database.tf | 54 +++++ .../infra/{terraform => modules}/eks/dns.tf | 0 .../{terraform => modules}/eks/kubernetes.tf | 43 ++-- install/infra/modules/eks/output.tf | 72 +++++++ .../{terraform => modules}/eks/providers.tf | 0 .../{terraform => modules}/eks/registry.tf | 5 +- install/infra/modules/eks/storage.tf | 44 +++++ .../{terraform => modules}/eks/variables.tf | 27 +-- .../{terraform => modules}/gke/README.md | 0 .../infra/{terraform => modules}/gke/main.tf | 0 .../{terraform => modules}/gke/output.tf | 0 .../{terraform => modules}/gke/variables.tf | 4 +- .../{terraform => modules}/k3s/README.md | 0 .../infra/{terraform => modules}/k3s/main.tf | 0 .../{terraform => modules}/k3s/variables.tf | 0 .../tools/azure-external-dns/main.tf | 0 .../tools/cert-manager/README.md | 0 .../tools/cert-manager/main.tf | 0 .../tools/cert-manager/variables.tf | 0 .../tools/cloud-dns-external-dns/README.md | 0 .../tools/cloud-dns-external-dns/main.tf | 0 .../tools/cloud-dns-external-dns/variables.tf | 0 .../tools/cloud-dns-ns/main.tf | 0 .../tools/external-dns/main.tf | 0 .../tools/issuer/main.tf | 0 .../tools/issuer/variables.tf | 0 install/infra/single-cluster/aws/.env_sample | 3 + install/infra/single-cluster/aws/Makefile | 67 +++++++ install/infra/single-cluster/aws/README.md | 184 ++++++++++++++++++ install/infra/single-cluster/aws/cluster.tf | 16 ++ install/infra/single-cluster/aws/main.tf | 6 + install/infra/single-cluster/aws/output.tf | 19 ++ .../infra/single-cluster/aws/terraform.tfvars | 18 ++ install/infra/single-cluster/aws/tools.tf | 22 +++ install/infra/single-cluster/aws/variables.tf | 59 ++++++ install/infra/terraform/eks/database.tf | 46 ----- install/infra/terraform/eks/output.tf | 72 ------- install/infra/terraform/eks/storage.tf | 37 ---- install/tests/main.tf | 26 +-- 50 files changed, 618 insertions(+), 206 deletions(-) rename install/infra/{terraform => modules}/aks/README.md (100%) rename install/infra/{terraform => modules}/aks/database.tf (100%) rename install/infra/{terraform => modules}/aks/kubernetes.tf (100%) rename install/infra/{terraform => modules}/aks/local.tf (100%) rename install/infra/{terraform => modules}/aks/main.tf (100%) rename install/infra/{terraform => modules}/aks/monitoring.tf (100%) rename install/infra/{terraform => modules}/aks/networks.tf (100%) rename install/infra/{terraform => modules}/aks/output.tf (100%) rename install/infra/{terraform => modules}/aks/registry.tf (100%) rename install/infra/{terraform => modules}/aks/storage.tf (100%) rename install/infra/{terraform => modules}/aks/variables.tf (100%) create mode 100644 install/infra/modules/eks/database.tf rename install/infra/{terraform => modules}/eks/dns.tf (100%) rename install/infra/{terraform => modules}/eks/kubernetes.tf (74%) create mode 100644 install/infra/modules/eks/output.tf rename install/infra/{terraform => modules}/eks/providers.tf (100%) rename install/infra/{terraform => modules}/eks/registry.tf (60%) create mode 100644 install/infra/modules/eks/storage.tf rename install/infra/{terraform => modules}/eks/variables.tf (69%) rename install/infra/{terraform => modules}/gke/README.md (100%) rename install/infra/{terraform => modules}/gke/main.tf (100%) rename install/infra/{terraform => modules}/gke/output.tf (100%) rename install/infra/{terraform => modules}/gke/variables.tf (95%) rename install/infra/{terraform => modules}/k3s/README.md (100%) rename install/infra/{terraform => modules}/k3s/main.tf (100%) rename install/infra/{terraform => modules}/k3s/variables.tf (100%) rename install/infra/{terraform => modules}/tools/azure-external-dns/main.tf (100%) rename install/infra/{terraform => modules}/tools/cert-manager/README.md (100%) rename install/infra/{terraform => modules}/tools/cert-manager/main.tf (100%) rename install/infra/{terraform => modules}/tools/cert-manager/variables.tf (100%) rename install/infra/{terraform => modules}/tools/cloud-dns-external-dns/README.md (100%) rename install/infra/{terraform => modules}/tools/cloud-dns-external-dns/main.tf (100%) rename install/infra/{terraform => modules}/tools/cloud-dns-external-dns/variables.tf (100%) rename install/infra/{terraform => modules}/tools/cloud-dns-ns/main.tf (100%) rename install/infra/{terraform => modules}/tools/external-dns/main.tf (100%) rename install/infra/{terraform => modules}/tools/issuer/main.tf (100%) rename install/infra/{terraform => modules}/tools/issuer/variables.tf (100%) create mode 100644 install/infra/single-cluster/aws/.env_sample create mode 100644 install/infra/single-cluster/aws/Makefile create mode 100644 install/infra/single-cluster/aws/README.md create mode 100644 install/infra/single-cluster/aws/cluster.tf create mode 100644 install/infra/single-cluster/aws/main.tf create mode 100644 install/infra/single-cluster/aws/output.tf create mode 100644 install/infra/single-cluster/aws/terraform.tfvars create mode 100644 install/infra/single-cluster/aws/tools.tf create mode 100644 install/infra/single-cluster/aws/variables.tf delete mode 100644 install/infra/terraform/eks/database.tf delete mode 100644 install/infra/terraform/eks/output.tf delete mode 100644 install/infra/terraform/eks/storage.tf diff --git a/install/infra/terraform/aks/README.md b/install/infra/modules/aks/README.md similarity index 100% rename from install/infra/terraform/aks/README.md rename to install/infra/modules/aks/README.md diff --git a/install/infra/terraform/aks/database.tf b/install/infra/modules/aks/database.tf similarity index 100% rename from install/infra/terraform/aks/database.tf rename to install/infra/modules/aks/database.tf diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/modules/aks/kubernetes.tf similarity index 100% rename from install/infra/terraform/aks/kubernetes.tf rename to install/infra/modules/aks/kubernetes.tf diff --git a/install/infra/terraform/aks/local.tf b/install/infra/modules/aks/local.tf similarity index 100% rename from install/infra/terraform/aks/local.tf rename to install/infra/modules/aks/local.tf diff --git a/install/infra/terraform/aks/main.tf b/install/infra/modules/aks/main.tf similarity index 100% rename from install/infra/terraform/aks/main.tf rename to install/infra/modules/aks/main.tf diff --git a/install/infra/terraform/aks/monitoring.tf b/install/infra/modules/aks/monitoring.tf similarity index 100% rename from install/infra/terraform/aks/monitoring.tf rename to install/infra/modules/aks/monitoring.tf diff --git a/install/infra/terraform/aks/networks.tf b/install/infra/modules/aks/networks.tf similarity index 100% rename from install/infra/terraform/aks/networks.tf rename to install/infra/modules/aks/networks.tf diff --git a/install/infra/terraform/aks/output.tf b/install/infra/modules/aks/output.tf similarity index 100% rename from install/infra/terraform/aks/output.tf rename to install/infra/modules/aks/output.tf diff --git a/install/infra/terraform/aks/registry.tf b/install/infra/modules/aks/registry.tf similarity index 100% rename from install/infra/terraform/aks/registry.tf rename to install/infra/modules/aks/registry.tf diff --git a/install/infra/terraform/aks/storage.tf b/install/infra/modules/aks/storage.tf similarity index 100% rename from install/infra/terraform/aks/storage.tf rename to install/infra/modules/aks/storage.tf diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/modules/aks/variables.tf similarity index 100% rename from install/infra/terraform/aks/variables.tf rename to install/infra/modules/aks/variables.tf diff --git a/install/infra/modules/eks/database.tf b/install/infra/modules/eks/database.tf new file mode 100644 index 00000000000000..82ea111adc5698 --- /dev/null +++ b/install/infra/modules/eks/database.tf @@ -0,0 +1,54 @@ +resource "random_password" "password" { + count = var.enable_external_database ? 1 : 0 + + length = 16 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + +resource "aws_db_subnet_group" "gitpod_subnets" { + count = var.enable_external_database ? 1 : 0 + + name = "db-sg-${var.cluster_name}" + subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] +} + +resource "aws_security_group" "rdssg" { + count = var.enable_external_database ? 1 : 0 + + name = "dh-sg-${var.cluster_name}" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 3306 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_db_instance" "gitpod" { + count = var.enable_external_database ? 1 : 0 + + allocated_storage = 10 + max_allocated_storage = 100 + engine = "mysql" + engine_version = "5.7" + instance_class = "db.t3.micro" + vpc_security_group_ids = [aws_security_group.rdssg[0].id] + identifier = "db-${var.cluster_name}" + name = "gitpod" + username = "gitpod" + password = random_password.password[0].result + parameter_group_name = "default.mysql5.7" + db_subnet_group_name = aws_db_subnet_group.gitpod_subnets[0].name + skip_final_snapshot = true + publicly_accessible = true +} diff --git a/install/infra/terraform/eks/dns.tf b/install/infra/modules/eks/dns.tf similarity index 100% rename from install/infra/terraform/eks/dns.tf rename to install/infra/modules/eks/dns.tf diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf similarity index 74% rename from install/infra/terraform/eks/kubernetes.tf rename to install/infra/modules/eks/kubernetes.tf index 5294e46a4ab2db..3bef25d99b283e 100644 --- a/install/infra/terraform/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -1,3 +1,12 @@ +locals { + private_primary_subnet_cidr = cidrsubnet(var.vpc_cidr, 7, 0) + private_secondary_subnet_cidr = cidrsubnet(var.vpc_cidr, 7, 1) + public_primary_subnet_cidr = cidrsubnet(var.vpc_cidr, 7, 2) + public_secondary_subnet_cidr = cidrsubnet(var.vpc_cidr, 7, 3) + public_db_subnet_cidr_1 = cidrsubnet(var.vpc_cidr, 7, 4) + public_db_subnet_cidr_2 = cidrsubnet(var.vpc_cidr, 7, 5) +} + module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "3.12.0" @@ -5,8 +14,8 @@ module "vpc" { name = "vpc-${var.cluster_name}" cidr = var.vpc_cidr azs = var.vpc_availability_zones - private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr] - public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2] + private_subnets = [local.private_primary_subnet_cidr, local.private_secondary_subnet_cidr] + public_subnets = [local.public_primary_subnet_cidr, local.public_secondary_subnet_cidr, local.public_db_subnet_cidr_1, local.public_db_subnet_cidr_2] enable_nat_gateway = true enable_dns_hostnames = true } @@ -64,16 +73,16 @@ module "eks" { eks_managed_node_groups = { Services = { enable_bootstrap_user_data = true - instance_types = [var.service_machine_type] - name = "service-${var.cluster_name}" - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 10 - desired_size = 1 - block_device_mappings = [{ + instance_types = [var.service_machine_type] + name = "service-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + desired_size = 1 + block_device_mappings = [{ device_name = "/dev/sda1" - ebs = [{ + ebs = [{ volume_size = 150 }] }] @@ -95,18 +104,18 @@ module "eks" { Workspaces = { instance_types = [var.workspace_machine_type] - name = "ws-${var.cluster_name}" - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 10 - block_device_mappings = [{ + name = "ws-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + block_device_mappings = [{ device_name = "/dev/sda1" - ebs = [{ + ebs = [{ volume_size = 150 }] }] - desired_size = 1 + desired_size = 1 enable_bootstrap_user_data = true labels = { "gitpod.io/workload_workspace_services" = true diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf new file mode 100644 index 00000000000000..b091b99b5a210e --- /dev/null +++ b/install/infra/modules/eks/output.tf @@ -0,0 +1,72 @@ +output "external_dns_settings" { + value = [ + { + "name" = "provider", + "value" = "aws" + }, + { + "name" = "aws.region", + "value" = var.region + }, + { + "name" = "aws.credentials.secretKey", + "value" = aws_iam_access_key.edns.secret + }, + { + "name" = "aws.credentials.accessKey", + "value" = aws_iam_access_key.edns.id + } + ] +} + +output "secretAccessKey" { + sensitive = true + value = try("${aws_iam_access_key.edns.secret}", "") +} + +output "cert_manager_issuer" { + value = try({ + region = var.region + secretAccessKeySecretRef = { + name = "route53-credentials" + key = "secret-access-key" + } + + hostedZoneID = aws_route53_zone.gitpod.zone_id + accessKeyID = aws_iam_access_key.edns.id + }, {}) +} + +output "domain_nameservers" { + value = formatlist("%s.", resource.aws_route53_zone.gitpod.name_servers) +} + +output "database" { + sensitive = true + value = try({ + host = "${aws_db_instance.gitpod[0].address}" + password = random_password.password[0].result + port = 3306 + username = "${aws_db_instance.gitpod[0].username}" + }, {}) +} + +output "registry" { + sensitive = true + value = try({ + server = aws_ecr_repository.gitpod[0].repository_url + username = data.aws_ecr_authorization_token.gitpod[0].user_name + password = data.aws_ecr_authorization_token.gitpod[0].password + }, {}) +} + +output "storage" { + sensitive = true + value = try({ + access_key_id = aws_iam_access_key.bucket_storage_user[0].id + secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret + region = aws_s3_bucket.gitpod-storage[0].region + bucket_name = aws_s3_bucket.gitpod-storage[0].id + endpoint = "s3.amazonaws.com" + }, {}) +} diff --git a/install/infra/terraform/eks/providers.tf b/install/infra/modules/eks/providers.tf similarity index 100% rename from install/infra/terraform/eks/providers.tf rename to install/infra/modules/eks/providers.tf diff --git a/install/infra/terraform/eks/registry.tf b/install/infra/modules/eks/registry.tf similarity index 60% rename from install/infra/terraform/eks/registry.tf rename to install/infra/modules/eks/registry.tf index 9756b6ed4b8858..87d07bbab50eb2 100644 --- a/install/infra/terraform/eks/registry.tf +++ b/install/infra/modules/eks/registry.tf @@ -1,4 +1,6 @@ resource "aws_ecr_repository" "gitpod" { + count = var.enable_external_registry ? 1 : 0 + name = "registry-${var.cluster_name}" image_tag_mutability = "MUTABLE" @@ -8,5 +10,6 @@ resource "aws_ecr_repository" "gitpod" { } data "aws_ecr_authorization_token" "gitpod" { - registry_id = aws_ecr_repository.gitpod.registry_id + count = var.enable_external_registry ? 1 : 0 + registry_id = aws_ecr_repository.gitpod[0].registry_id } diff --git a/install/infra/modules/eks/storage.tf b/install/infra/modules/eks/storage.tf new file mode 100644 index 00000000000000..ff2f0bd1e884e8 --- /dev/null +++ b/install/infra/modules/eks/storage.tf @@ -0,0 +1,44 @@ +resource "aws_s3_bucket" "gitpod-storage" { + count = var.enable_external_storage ? 1 : 0 + + force_destroy = true + bucket = "bucket-${var.cluster_name}" + acl = "private" + + versioning { + enabled = true + } +} + +data "aws_iam_policy_document" "s3_policy" { + count = var.enable_external_storage ? 1 : 0 + statement { + actions = ["s3:*"] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "policy" { + count = var.enable_external_storage ? 1 : 0 + name = "spolicy-${var.cluster_name}" + description = "s3 storage bucket policy" + policy = data.aws_iam_policy_document.s3_policy[0].json +} + +resource "aws_iam_user" "bucket_storage" { + count = var.enable_external_storage ? 1 : 0 + name = "suser-${var.cluster_name}" + +} + +resource "aws_iam_user_policy_attachment" "attachment" { + count = var.enable_external_storage ? 1 : 0 + user = aws_iam_user.bucket_storage[0].name + policy_arn = aws_iam_policy.policy[0].arn +} + +resource "aws_iam_access_key" "bucket_storage_user" { + count = var.enable_external_storage ? 1 : 0 + user = aws_iam_user.bucket_storage[0].name +} diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/modules/eks/variables.tf similarity index 69% rename from install/infra/terraform/eks/variables.tf rename to install/infra/modules/eks/variables.tf index 192ba4d3398ffc..341b2f81410bc7 100644 --- a/install/infra/terraform/eks/variables.tf +++ b/install/infra/modules/eks/variables.tf @@ -51,26 +51,17 @@ variable "vpc_cidr" { default = "10.100.0.0/16" } -variable "private_primary_subnet_cidr" { - default = "10.100.160.0/19" +variable "enable_external_database" { + default = true + description = "Set this to false to avoid creating an RDS database to use with Gitpod instead of inclsuter mysql" } -variable "private_secondary_subnet_cidr" { - default = "10.100.128.0/19" +variable "enable_external_storage" { + default = true + description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" } -variable "public_primary_subnet_cidr" { - default = "10.100.64.0/18" -} - -variable "public_secondary_subnet_cidr" { - default = "10.100.0.0/18" -} - -variable "public_db_subnet_cidr_1" { - default = "10.100.192.0/19" -} - -variable "public_db_subnet_cidr_2" { - default = "10.100.224.0/19" +variable "enable_external_registry" { + default = true + description = "Set this to false to avoid creating an AWS registry to use with Gitpod instead of incluster registry" } diff --git a/install/infra/terraform/gke/README.md b/install/infra/modules/gke/README.md similarity index 100% rename from install/infra/terraform/gke/README.md rename to install/infra/modules/gke/README.md diff --git a/install/infra/terraform/gke/main.tf b/install/infra/modules/gke/main.tf similarity index 100% rename from install/infra/terraform/gke/main.tf rename to install/infra/modules/gke/main.tf diff --git a/install/infra/terraform/gke/output.tf b/install/infra/modules/gke/output.tf similarity index 100% rename from install/infra/terraform/gke/output.tf rename to install/infra/modules/gke/output.tf diff --git a/install/infra/terraform/gke/variables.tf b/install/infra/modules/gke/variables.tf similarity index 95% rename from install/infra/terraform/gke/variables.tf rename to install/infra/modules/gke/variables.tf index c07d3588a514b0..d7ee6fe554de81 100644 --- a/install/infra/terraform/gke/variables.tf +++ b/install/infra/modules/gke/variables.tf @@ -35,13 +35,13 @@ variable "name" { variable "workspaces_machine_type" { type = string description = "Type of the node compute engines for workspace nodepool." - default = "n2-standard-8" + default = "n2d-standard-8" } variable "services_machine_type" { type = string description = "Type of the node compute engines for services nodepool." - default = "n2-standard-4" + default = "n2d-standard-4" } variable "max_count" { diff --git a/install/infra/terraform/k3s/README.md b/install/infra/modules/k3s/README.md similarity index 100% rename from install/infra/terraform/k3s/README.md rename to install/infra/modules/k3s/README.md diff --git a/install/infra/terraform/k3s/main.tf b/install/infra/modules/k3s/main.tf similarity index 100% rename from install/infra/terraform/k3s/main.tf rename to install/infra/modules/k3s/main.tf diff --git a/install/infra/terraform/k3s/variables.tf b/install/infra/modules/k3s/variables.tf similarity index 100% rename from install/infra/terraform/k3s/variables.tf rename to install/infra/modules/k3s/variables.tf diff --git a/install/infra/terraform/tools/azure-external-dns/main.tf b/install/infra/modules/tools/azure-external-dns/main.tf similarity index 100% rename from install/infra/terraform/tools/azure-external-dns/main.tf rename to install/infra/modules/tools/azure-external-dns/main.tf diff --git a/install/infra/terraform/tools/cert-manager/README.md b/install/infra/modules/tools/cert-manager/README.md similarity index 100% rename from install/infra/terraform/tools/cert-manager/README.md rename to install/infra/modules/tools/cert-manager/README.md diff --git a/install/infra/terraform/tools/cert-manager/main.tf b/install/infra/modules/tools/cert-manager/main.tf similarity index 100% rename from install/infra/terraform/tools/cert-manager/main.tf rename to install/infra/modules/tools/cert-manager/main.tf diff --git a/install/infra/terraform/tools/cert-manager/variables.tf b/install/infra/modules/tools/cert-manager/variables.tf similarity index 100% rename from install/infra/terraform/tools/cert-manager/variables.tf rename to install/infra/modules/tools/cert-manager/variables.tf diff --git a/install/infra/terraform/tools/cloud-dns-external-dns/README.md b/install/infra/modules/tools/cloud-dns-external-dns/README.md similarity index 100% rename from install/infra/terraform/tools/cloud-dns-external-dns/README.md rename to install/infra/modules/tools/cloud-dns-external-dns/README.md diff --git a/install/infra/terraform/tools/cloud-dns-external-dns/main.tf b/install/infra/modules/tools/cloud-dns-external-dns/main.tf similarity index 100% rename from install/infra/terraform/tools/cloud-dns-external-dns/main.tf rename to install/infra/modules/tools/cloud-dns-external-dns/main.tf diff --git a/install/infra/terraform/tools/cloud-dns-external-dns/variables.tf b/install/infra/modules/tools/cloud-dns-external-dns/variables.tf similarity index 100% rename from install/infra/terraform/tools/cloud-dns-external-dns/variables.tf rename to install/infra/modules/tools/cloud-dns-external-dns/variables.tf diff --git a/install/infra/terraform/tools/cloud-dns-ns/main.tf b/install/infra/modules/tools/cloud-dns-ns/main.tf similarity index 100% rename from install/infra/terraform/tools/cloud-dns-ns/main.tf rename to install/infra/modules/tools/cloud-dns-ns/main.tf diff --git a/install/infra/terraform/tools/external-dns/main.tf b/install/infra/modules/tools/external-dns/main.tf similarity index 100% rename from install/infra/terraform/tools/external-dns/main.tf rename to install/infra/modules/tools/external-dns/main.tf diff --git a/install/infra/terraform/tools/issuer/main.tf b/install/infra/modules/tools/issuer/main.tf similarity index 100% rename from install/infra/terraform/tools/issuer/main.tf rename to install/infra/modules/tools/issuer/main.tf diff --git a/install/infra/terraform/tools/issuer/variables.tf b/install/infra/modules/tools/issuer/variables.tf similarity index 100% rename from install/infra/terraform/tools/issuer/variables.tf rename to install/infra/modules/tools/issuer/variables.tf diff --git a/install/infra/single-cluster/aws/.env_sample b/install/infra/single-cluster/aws/.env_sample new file mode 100644 index 00000000000000..8bf5edf5fd9ead --- /dev/null +++ b/install/infra/single-cluster/aws/.env_sample @@ -0,0 +1,3 @@ +export AWS_REGION=eu-west-1 +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile new file mode 100644 index 00000000000000..903d235c3a3b44 --- /dev/null +++ b/install/infra/single-cluster/aws/Makefile @@ -0,0 +1,67 @@ +## +# Terraform AWS reference architecture +# + +.PHONY: init +init: + @terraform init + +.PHONY: plan +plan: plan-cluster plan-cm-edns + +.PHONY: apply +apply: apply-cluster apply-tools + +.PHONY: destroy +destroy: destroy-tools destroy-cluster + +.PHONY: output +output: + @terraform output -json + +.PHONY: plan-cluster +plan-cluster: + @terraform plan -target=module.eks + +.PHONY: plan-tools +plan-tools: plan-cm-edns plan-cluster-issuer + +.PHONY: plan-cm-edns +plan-cm-edns: + @terraform plan -target=module.certmanager -target=module.externaldns + +.PHONY: plan-cluster-issuer +plan-cluster-issuer: + @terraform plan -target=module.cluster-issuer + +.PHONY: apply-cluster +apply-cluster: + @terraform apply -target=module.eks --auto-approve + +.PHONY: apply-tools +apply-tools: install-cm-edns install-cluster-issuer + +.PHONY: install-cm-edns +install-cm-edns: + @terraform apply -target=module.certmanager -target=module.externaldns --auto-approve + +.PHONY: install-cluster-issuer +install-cluster-issuer: + @terraform apply -target=module.externaldns --auto-approve + +.PHONY: destroy-cluster +destroy-cluster: + @terraform destroy -target=module.eks --auto-approve + +.PHONY: destroy-tools +destroy-tools: destroy-cluster-issuer destroy-cm-edns + +.PHONY: destroy-cm-edns +destroy-cm-edns: + @terraform destroy -target=module.certmanager -target=module.externaldns --auto-approve + +.PHONY: destroy-cluster-issuer +destroy-cluster-issuer: + @terraform destroy -target=module.cluster-issuer --auto-approve + +# end diff --git a/install/infra/single-cluster/aws/README.md b/install/infra/single-cluster/aws/README.md new file mode 100644 index 00000000000000..34bc278fbf7e15 --- /dev/null +++ b/install/infra/single-cluster/aws/README.md @@ -0,0 +1,184 @@ +# Terraform setup for AWS Single-cluster Gitpod reference architecture + +This directory has terraform configuration necessary to achieve a infrastructure +corresponding to the [Single-cluster reference architecture for Gitpod on AWS](https://www.gitpod.io/docs/self-hosted/latest/reference-architecture/single-cluster-ref-arch). + +This module will do the following steps: +- Create an EKS managed cluster +- Creates external dependencies like database, storage and registry (if chosen) +- Sets up route53 entries for the domain name (if chosen) +- Set up cert-manager, external-dns and cluster issuer on the created cluster + +Since the entire setup requires more than one terraform target to be run due to +dependencies (eg: helm provider depends on kubernetes cluster config, which is +not available until the `eks` module finishes), this directory has a `Makefile` +with targets binding together targeted terraform calls. This document will +explain the execution of the terraform module in terms of these `make` targets. + +## Requirements + +* `terraform` >= `v1.1.0` +* `kubectl` >= `v1.20.0` + +## Setup AWS authentication and s3 backend storage + +create IAM, set env vars, create backend bucket + +Before starting the installation process, you need: + +* An AWS account with Administrator access + * [Create one now by clicking here](https://aws.amazon.com/getting-started/) +* Setup credentials to be usable in one of the following ways: + * [As environmental variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) + * Copy the file `.env_sample` to `.env` and update the values corresponding + to your AWS user. Run: + ```sh + source .env + ``` + * [As credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) +* Create and configure s3 bucket for terraform backend + * Create an [AWS S3 bucket](https://aws.amazon.com/s3/) to store the terraform backend state + * Replace the name of the bucket in [`main.tf`](./main.tf) - currently it is set as `gitpod-tf` + +## Update the `terraform.tfvars` file with appropriate values + +The file [`terraform.tfvars`](./terraform.tfvars) with values that will be used +by terraform to create the cluster. While some of them are fairly +straightforward like the name of the cluster(`cluster_name`), others need a bit +more attention: + +* VPC CIDR IP + +It is necessary to ensure that the `VPC` setup will not have conflicts with +existing VPCs or has sufficiently large enough IP range so as to not run out of +them. Please check under the region's VPCs if the IP range you are choosing is +already in use. The CIDR will be split among 5 subnets and hence we recommend +`/16` as prefix to allow sufficient IP ranges. The default value is: `10.100.0.0/16` + +* External database, storage and registry + +If you wish to create cloud specific database, storage and registry to be used +with `Gitpod`, leave the following 3 booleans set: + +``` sh +enable_external_database = true +enable_external_storage = true +enable_external_registry = true +``` + +The corresponding resources will be created by the terraform script which +inclustes an `RDS` mysql database, an `S3` bucket and an AWS container registry setup. The +expectation is that you can use the credentials to these setups(provided later +as terraform outputs) during the setup of Gitpod via UI later in the process. +Alternatively, one can choose to use incluster dependencies or separately +created resources of choice. + +* AMI Image ID and Kubernetes version + +We officially support Ubuntu images for Gitpod setup. In EKS cluster, AMI images +are kubernetes version and region specific. You can find a list of AMI IDs +[here](https://cloud-images.ubuntu.com/docs/aws/eks/). + +Make sure you provide the corresponding kubernetes version as a value to the +variable `cluster_version`. We officially support kubernetes versions >= `1.20`. + +* Domain name configuration + +If you are already sure of the domain name under which you want to setup Gitpod, +we recommend highly to provide the value as `domain_name`. This will save a lot +of hassle in setting up `route53` records to point to the cluster and +corresponding TLS certificate requests. + +## Initialize terraform backend and confirm the plan + +* Initialize the terraform backend with: + + ``` sh + make init + ``` + +* Get the plan of the execution to verify the resources that are going to get + created: + + ```sh + make plan + ``` + +## Apply terraform setup + +If the plan looks good, now you can go ahead and create the resources: + +``` sh +make apply +``` + +The `apply` target calls the `terraform` apply on `eks` module, `cert-manager` +module, `external-dns` module and `cluster-issuer` module in that exact order. +The entire operation would take around *30 minutes* to complete. + +Upon completion, you will find a creation `kubeconfig` file in the local +directory. You can try accessing the cluster using this file as follows: + +``` sh +export KUBECONFIG=/path/to/kubeconfig +kubectl get pods -A +``` + +You can list all the other outputs with the following command: + +``` sh +make output +``` + +## Note the NS records from terraform output + +Once the apply process has exited successfully, we can go ahead and prepare to +setup Gitpod. If you specified the `domain_name` in the `terraform.tfvars` file, +the terraform module registers the module with `route53` to point to the +cluster. Now you have to configure whichever provider you use to host your +domain name to route traffic to the AWS name servers. You can find these name +servers in the `make output` command from above. It would be of the format: + +``` json + "nameservers": { + "sensitive": false, + "type": [ + "list", + "string" + ], + "value": [ + "ns-100.awsdns-12.com.", + "ns-1434.awsdns-51.org.", + "ns-1765.awsdns-28.co.uk.", + "ns-786.awsdns-34.net." + ] +``` + +Add the `ns` records similar to the above 4 URIs as NS records under your domain +DNS management setup. Check with your domain hosting service for specific information. + +## Note the dependency credentials from terraform output + +If you enabled the creation of external database, storage and registry, the +above `make output` command would list the credentials to connect to the +corresponding resource. Make a note of this, so as to provide the same when +setting up Gitpod via KOTS UI. + +## Install Gitpod + +You can install `KOTS` CLI to install Gitpod: + +``` sh +curl https://kots.io/install | bash +``` + +Run the following to get started with Gitpod installation: + +``` sh +export KUBECONFIG=kubeconfig +kubectl kots install gitpod/stable # you can replace `stable` with `unstable` or `beta` as per the requirement +``` + +Upon completion, you can access `KOTS` UI in `localhost:8800`. Here you can +proceed to configuring and intalling Gitpod. Please follow the [official +documentaion](https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod) to complete the Gitpod setup. diff --git a/install/infra/single-cluster/aws/cluster.tf b/install/infra/single-cluster/aws/cluster.tf new file mode 100644 index 00000000000000..ae4c22c3f78dd7 --- /dev/null +++ b/install/infra/single-cluster/aws/cluster.tf @@ -0,0 +1,16 @@ +module "eks" { + source = "../../modules/eks" + domain_name = var.domain_name + cluster_name = var.cluster_name + region = var.region + vpc_cidr = var.vpc_cidr + vpc_availability_zones = var.vpc_availability_zones + image_id = var.image_id + cluster_version = var.cluster_version + kubeconfig = var.kubeconfig + enable_external_database = var.enable_external_database + enable_external_storage = var.enable_external_storage + enable_external_registry = var.enable_external_registry + service_machine_type = "m6i.xlarge" + workspace_machine_type = "m6i.2xlarge" +} diff --git a/install/infra/single-cluster/aws/main.tf b/install/infra/single-cluster/aws/main.tf new file mode 100644 index 00000000000000..9ae7aee2fbf265 --- /dev/null +++ b/install/infra/single-cluster/aws/main.tf @@ -0,0 +1,6 @@ +terraform { + backend "s3" { + bucket = "gitpod-tf" + key = "aws/terraform.state" + } +} diff --git a/install/infra/single-cluster/aws/output.tf b/install/infra/single-cluster/aws/output.tf new file mode 100644 index 00000000000000..896cb50b3c7274 --- /dev/null +++ b/install/infra/single-cluster/aws/output.tf @@ -0,0 +1,19 @@ +output "aws_storage" { + sensitive = true + value = module.eks.storage +} + +output "aws_registry" { + sensitive = true + value = module.eks.registry +} + +output "aws_database" { + sensitive = true + value = module.eks.database +} + +output "nameservers" { + sensitive = false + value = module.eks.domain_nameservers +} diff --git a/install/infra/single-cluster/aws/terraform.tfvars b/install/infra/single-cluster/aws/terraform.tfvars new file mode 100644 index 00000000000000..c7b935c0df2571 --- /dev/null +++ b/install/infra/single-cluster/aws/terraform.tfvars @@ -0,0 +1,18 @@ + +cluster_name = "new-cluster" + +image_id = "ami-0793b4124359a6ad7" + +region = "eu-west-1" + +vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] + +vpc_cidr = "10.100.0.0/16" + +domain_name = "new-cluster-gitpoid.com" + +cluster_version = "1.22" + +enable_external_database = true +enable_external_storage = true +enable_external_registry = true diff --git a/install/infra/single-cluster/aws/tools.tf b/install/infra/single-cluster/aws/tools.tf new file mode 100644 index 00000000000000..400eb943674765 --- /dev/null +++ b/install/infra/single-cluster/aws/tools.tf @@ -0,0 +1,22 @@ +module "certmanager" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" + source = "../../modules/tools/cert-manager" + + kubeconfig = var.kubeconfig +} + +module "externaldns" { + source = "../../modules/tools/external-dns" + kubeconfig = var.kubeconfig + settings = module.eks.external_dns_settings + domain_name = var.domain_name + txt_owner_id = var.cluster_name +} + +module "cluster-issuer" { + source = "../../modules/tools/issuer" + kubeconfig = var.kubeconfig + cert_manager_issuer = module.eks.cert_manager_issuer + secretAccessKey = module.eks.secretAccessKey + issuer_name = "route53" +} diff --git a/install/infra/single-cluster/aws/variables.tf b/install/infra/single-cluster/aws/variables.tf new file mode 100644 index 00000000000000..79083f381971b7 --- /dev/null +++ b/install/infra/single-cluster/aws/variables.tf @@ -0,0 +1,59 @@ +variable "cluster_name" { + type = string + description = "EKS cluster name." +} +variable "kubeconfig" { + type = string + description = "Path to the kubeconfig file to write the KUBECONFIG output in" + default = "kubeconfig" +} + +variable "image_id" { + type = string + description = "AMI Image ID specific to the region" + // latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/ + default = "ami-0793b4124359a6ad7" +} + +variable "region" { + type = string + description = "Region to create the resources in" + default = "eu-west-1" +} + +variable "vpc_availability_zones" { + type = list(string) + description = "Availibiliy zones under the provided region, should be atleast two" + default = ["eu-west-1c", "eu-west-1b"] +} + +variable "vpc_cidr" { + type = string + description = "CIDR block IP to create the VPC. The CIDR will be divided into 5 subnets" + default = "10.100.0.0/16" +} + +variable "domain_name" { + description = "Domain name to associate with the Gitpod installation" +} + +variable "enable_external_database" { + default = true + description = "Set this to false to avoid creating an RDS database to use with Gitpod instead of inclsuter mysql" +} + +variable "enable_external_storage" { + default = true + description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" +} + +variable "enable_external_registry" { + default = true + description = "Set this to false to avoid creating an AWS registry to use with Gitpod instead of incluster registry" +} + +variable "cluster_version" { + type = string + description = "Kubernetes version to create the cluster with" + default = "1.22" +} diff --git a/install/infra/terraform/eks/database.tf b/install/infra/terraform/eks/database.tf deleted file mode 100644 index 97265f2f5ea00e..00000000000000 --- a/install/infra/terraform/eks/database.tf +++ /dev/null @@ -1,46 +0,0 @@ -resource "random_password" "password" { - length = 16 - special = true - override_special = "!#$%&*()-_=+[]{}<>:?" -} - -resource "aws_db_subnet_group" "gitpod_subnets" { - name = "db-sg-${var.cluster_name}" - subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] -} - -resource "aws_security_group" "rdssg" { - name = "dh-sg-${var.cluster_name}" - vpc_id = module.vpc.vpc_id - - ingress { - from_port = 0 - to_port = 3306 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_db_instance" "gitpod" { - allocated_storage = 10 - max_allocated_storage = 100 - engine = "mysql" - engine_version = "5.7" - instance_class = "db.t3.micro" - vpc_security_group_ids = [ aws_security_group.rdssg.id ] - identifier = "db-${var.cluster_name}" - name = "gitpod" - username = "gitpod" - password = random_password.password.result - parameter_group_name = "default.mysql5.7" - db_subnet_group_name = aws_db_subnet_group.gitpod_subnets.name - skip_final_snapshot = true - publicly_accessible = true -} diff --git a/install/infra/terraform/eks/output.tf b/install/infra/terraform/eks/output.tf deleted file mode 100644 index 4e34598e907e12..00000000000000 --- a/install/infra/terraform/eks/output.tf +++ /dev/null @@ -1,72 +0,0 @@ -output "external_dns_settings" { - value = [ - { - "name" = "provider", - "value" = "aws" - }, - { - "name" = "aws.region", - "value" = var.region - }, - { - "name" = "aws.credentials.secretKey", - "value" = aws_iam_access_key.edns.secret - }, - { - "name" = "aws.credentials.accessKey", - "value" = aws_iam_access_key.edns.id - } - ] -} - -output "secretAccessKey" { - sensitive = true - value = try("${aws_iam_access_key.edns.secret}", "") -} - -output "cert_manager_issuer" { - value = try({ - region = var.region - secretAccessKeySecretRef = { - name = "route53-credentials" - key = "secret-access-key" - } - - hostedZoneID = aws_route53_zone.gitpod.zone_id - accessKeyID = aws_iam_access_key.edns.id - }, {}) -} - -output "domain_nameservers" { - value = formatlist("%s.", resource.aws_route53_zone.gitpod.name_servers) -} - -output "database" { - sensitive = true - value = try({ - host = "${aws_db_instance.gitpod.address}" - password = random_password.password.result - port = 3306 - username = "${aws_db_instance.gitpod.username}" - }, {}) -} - -output "registry" { - sensitive = true - value = try({ - server = aws_ecr_repository.gitpod.repository_url - username = data.aws_ecr_authorization_token.gitpod.user_name - password = data.aws_ecr_authorization_token.gitpod.password - }, {}) -} - -output "storage" { - sensitive = true - value = try({ - access_key_id = aws_iam_access_key.bucket_storage_user.id - secret_access_key = aws_iam_access_key.bucket_storage_user.secret - region = aws_s3_bucket.gitpod-storage.region - bucket_name = aws_s3_bucket.gitpod-storage.id - endpoint = "s3.amazonaws.com" - }, {}) -} diff --git a/install/infra/terraform/eks/storage.tf b/install/infra/terraform/eks/storage.tf deleted file mode 100644 index 772187015ab190..00000000000000 --- a/install/infra/terraform/eks/storage.tf +++ /dev/null @@ -1,37 +0,0 @@ -resource "aws_s3_bucket" "gitpod-storage" { - force_destroy = true - bucket = "bucket-${var.cluster_name}" - acl = "private" - - versioning { - enabled = true - } -} - -data "aws_iam_policy_document" "s3_policy" { - statement { - actions = ["s3:*"] - resources = ["*"] - effect = "Allow" - } -} - -resource "aws_iam_policy" "policy" { - name = "spolicy-${var.cluster_name}" - description = "s3 storage bucket policy" - policy = data.aws_iam_policy_document.s3_policy.json -} - -resource "aws_iam_user" "bucket_storage" { - name = "suser-${var.cluster_name}" - -} - -resource "aws_iam_user_policy_attachment" "attachment" { - user = aws_iam_user.bucket_storage.name - policy_arn = aws_iam_policy.policy.arn -} - -resource "aws_iam_access_key" "bucket_storage_user" { - user = aws_iam_user.bucket_storage.name -} diff --git a/install/tests/main.tf b/install/tests/main.tf index d4196b3af4b7a0..3333b2f967d3c4 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -27,7 +27,7 @@ variable "cluster_version" { module "gke" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=main" # we can later use tags here - source = "../infra/terraform/gke" # we can later use tags here + source = "../infra/modules/gke" # we can later use tags here name = var.TEST_ID project = var.project @@ -40,7 +40,7 @@ module "gke" { module "k3s" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/k3s?ref=main" # we can later use tags here - source = "../infra/terraform/k3s" # we can later use tags here + source = "../infra/modules/k3s" # we can later use tags here name = var.TEST_ID gcp_project = var.project @@ -55,7 +55,7 @@ module "k3s" { } module "gcp-issuer" { - source = "../infra/terraform/tools/issuer" + source = "../infra/modules/tools/issuer" kubeconfig = var.kubeconfig issuer_name = "cloudDNS" cert_manager_issuer = { @@ -70,7 +70,7 @@ module "gcp-issuer" { module "aks" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here - source = "../infra/terraform/aks" + source = "../infra/modules/aks" domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" enable_airgapped = false @@ -84,7 +84,7 @@ module "aks" { } module "eks" { - source = "../infra/terraform/eks" + source = "../infra/modules/eks" domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" cluster_name = var.TEST_ID region = "eu-west-1" @@ -96,7 +96,7 @@ module "eks" { module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" - source = "../infra/terraform/tools/cert-manager" + source = "../infra/modules/tools/cert-manager" kubeconfig = var.kubeconfig credentials = var.dns_sa_creds @@ -104,13 +104,13 @@ module "certmanager" { module "clouddns-externaldns" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/external-dns?ref=main" - source = "../infra/terraform/tools/cloud-dns-external-dns" + source = "../infra/modules/tools/cloud-dns-external-dns" kubeconfig = var.kubeconfig credentials = var.dns_sa_creds } module "azure-externaldns" { - source = "../infra/terraform/tools/external-dns" + source = "../infra/modules/tools/external-dns" kubeconfig = var.kubeconfig settings = module.aks.external_dns_settings domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" @@ -118,7 +118,7 @@ module "azure-externaldns" { } module "aws-externaldns" { - source = "../infra/terraform/tools/external-dns" + source = "../infra/modules/tools/external-dns" kubeconfig = var.kubeconfig settings = module.eks.external_dns_settings domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" @@ -126,14 +126,14 @@ module "aws-externaldns" { } module "azure-issuer" { - source = "../infra/terraform/tools/issuer" + source = "../infra/modules/tools/issuer" kubeconfig = var.kubeconfig cert_manager_issuer = module.aks.cert_manager_issuer issuer_name = "azureDNS" } module "aws-issuer" { - source = "../infra/terraform/tools/issuer" + source = "../infra/modules/tools/issuer" kubeconfig = var.kubeconfig cert_manager_issuer = module.eks.cert_manager_issuer secretAccessKey = module.eks.secretAccessKey @@ -141,7 +141,7 @@ module "aws-issuer" { } module "azure-add-dns-record" { - source = "../infra/terraform/tools/cloud-dns-ns" + source = "../infra/modules/tools/cloud-dns-ns" credentials = var.dns_sa_creds nameservers = module.aks.domain_nameservers dns_project = "dns-for-playgrounds" @@ -150,7 +150,7 @@ module "azure-add-dns-record" { } module "aws-add-dns-record" { - source = "../infra/terraform/tools/cloud-dns-ns" + source = "../infra/modules/tools/cloud-dns-ns" credentials = var.dns_sa_creds nameservers = module.eks.domain_nameservers dns_project = "dns-for-playgrounds" From c6c0de2f6b114d5feda4a83c0c85b0e7681138c6 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Fri, 5 Aug 2022 10:35:40 +0000 Subject: [PATCH 3/9] Improve the tf module documentation --- install/infra/modules/eks/output.tf | 19 +++- install/infra/modules/eks/storage.tf | 12 +++ install/infra/modules/eks/variables.tf | 9 +- install/infra/modules/tools/issuer/main.tf | 2 +- .../infra/modules/tools/issuer/variables.tf | 4 + install/infra/single-cluster/aws/Makefile | 47 +++++++- install/infra/single-cluster/aws/README.md | 101 +++++++++++++----- install/infra/single-cluster/aws/cluster.tf | 3 +- install/infra/single-cluster/aws/output.tf | 23 +++- .../infra/single-cluster/aws/terraform.tfvars | 20 ++-- install/infra/single-cluster/aws/variables.tf | 6 +- 11 files changed, 193 insertions(+), 53 deletions(-) diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf index b091b99b5a210e..8ee3e11d34fced 100644 --- a/install/infra/modules/eks/output.tf +++ b/install/infra/modules/eks/output.tf @@ -48,7 +48,7 @@ output "database" { password = random_password.password[0].result port = 3306 username = "${aws_db_instance.gitpod[0].username}" - }, {}) + }, "No database created") } output "registry" { @@ -57,7 +57,7 @@ output "registry" { server = aws_ecr_repository.gitpod[0].repository_url username = data.aws_ecr_authorization_token.gitpod[0].user_name password = data.aws_ecr_authorization_token.gitpod[0].password - }, {}) + }, "No EKS registry created") } output "storage" { @@ -67,6 +67,17 @@ output "storage" { secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret region = aws_s3_bucket.gitpod-storage[0].region bucket_name = aws_s3_bucket.gitpod-storage[0].id - endpoint = "s3.amazonaws.com" - }, {}) + endpoint = "s3.${aws_s3_bucket.gitpod-storage[0].region}.amazonaws.com" + }, "No s3 bucket created for object storage") +} + +output "registry_backend" { + sensitive = true + value = try({ + access_key_id = aws_iam_access_key.bucket_storage_user[0].id + secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret + region = aws_s3_bucket.gitpod-registry-backend[0].region + bucket_name = aws_s3_bucket.gitpod-registry-backend[0].id + endpoint = "s3.${aws_s3_bucket.gitpod-registry-backend[0].region}.amazonaws.com" + }, "No s3 bucket created for registry backend.") } diff --git a/install/infra/modules/eks/storage.tf b/install/infra/modules/eks/storage.tf index ff2f0bd1e884e8..bb6facd01c5449 100644 --- a/install/infra/modules/eks/storage.tf +++ b/install/infra/modules/eks/storage.tf @@ -10,6 +10,18 @@ resource "aws_s3_bucket" "gitpod-storage" { } } +resource "aws_s3_bucket" "gitpod-registry-backend" { + count = var.enable_external_storage_for_registry_backend ? 1 : 0 + + force_destroy = true + bucket = "reg-bucket-${var.cluster_name}" + acl = "private" + + versioning { + enabled = true + } +} + data "aws_iam_policy_document" "s3_policy" { count = var.enable_external_storage ? 1 : 0 statement { diff --git a/install/infra/modules/eks/variables.tf b/install/infra/modules/eks/variables.tf index 341b2f81410bc7..2f13bb57e31bc3 100644 --- a/install/infra/modules/eks/variables.tf +++ b/install/infra/modules/eks/variables.tf @@ -61,7 +61,12 @@ variable "enable_external_storage" { description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" } +variable "enable_external_storage_for_registry_backend" { + default = false + description = "Set this to true to create a separate s3 storage to use with Gitpod as registry backend" +} + variable "enable_external_registry" { - default = true - description = "Set this to false to avoid creating an AWS registry to use with Gitpod instead of incluster registry" + default = false + description = "Set this to true to create an AWS ECR registry to use with Gitpod(Not officially supported)" } diff --git a/install/infra/modules/tools/issuer/main.tf b/install/infra/modules/tools/issuer/main.tf index a00036c6019f9f..5f12d9eec0f6f7 100644 --- a/install/infra/modules/tools/issuer/main.tf +++ b/install/infra/modules/tools/issuer/main.tf @@ -19,7 +19,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" { "apiVersion" = "cert-manager.io/v1" "kind" = "ClusterIssuer" "metadata" = { - "name" = "gitpod-issuer" + "name" = var.cluster_issuer_name } "spec" = { "acme" = { diff --git a/install/infra/modules/tools/issuer/variables.tf b/install/infra/modules/tools/issuer/variables.tf index c9622796efa386..eaa5a4346c2fc5 100644 --- a/install/infra/modules/tools/issuer/variables.tf +++ b/install/infra/modules/tools/issuer/variables.tf @@ -14,3 +14,7 @@ variable "secretAccessKey" { variable "issuer_name" { default = "route53" } + +variable "cluster_issuer_name" { + default = "gitpod-issuer" +} diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile index 903d235c3a3b44..a57249af1c77e7 100644 --- a/install/infra/single-cluster/aws/Makefile +++ b/install/infra/single-cluster/aws/Makefile @@ -16,8 +16,47 @@ apply: apply-cluster apply-tools destroy: destroy-tools destroy-cluster .PHONY: output -output: - @terraform output -json +output: output-done-msg output-url output-nameservers output-registry output-database output-storage output-issuer + +output-done-msg: + @echo "==========================" + @echo "๐ŸŽ‰๐Ÿฅณ๐Ÿ”ฅ๐Ÿงก๐Ÿš€" + @echo "Your cloud infrastructure is ready to install Gitpod. Please visit" + @echo "https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod" + @echo "for your next steps." + @echo "=================" + @echo "Config Parameters" + @echo "=================" + +output-url: + @echo "\nGitpod domain name:" + @echo "=================" + @terraform output -json url | jq + +output-nameservers: + @echo "\nNameservers for the domain(to be added as NS records in your domain provider):" + @echo "=================" + @terraform output -json nameservers | jq + +output-storage: + @echo "\nObject storage:" + @echo "==============" + @terraform output -json storage | jq + +output-registry: + @echo "\nS3 registry backend:" + @echo "==================" + @terraform output -json registry | jq + +output-database: + @echo "\nDatabase:" + @echo "========" + @terraform output -json database | jq + +output-issuer: + @echo "\nClusterIssuer name:" + @echo "=================" + @terraform output -json cluster_issuer | jq .PHONY: plan-cluster plan-cluster: @@ -45,9 +84,9 @@ apply-tools: install-cm-edns install-cluster-issuer install-cm-edns: @terraform apply -target=module.certmanager -target=module.externaldns --auto-approve -.PHONY: install-cluster-issuer +PHONY: install-cluster-issuer install-cluster-issuer: - @terraform apply -target=module.externaldns --auto-approve + @terraform apply -target=module.cluster-issuer --auto-approve .PHONY: destroy-cluster destroy-cluster: diff --git a/install/infra/single-cluster/aws/README.md b/install/infra/single-cluster/aws/README.md index 34bc278fbf7e15..cfaa01278816f9 100644 --- a/install/infra/single-cluster/aws/README.md +++ b/install/infra/single-cluster/aws/README.md @@ -4,10 +4,13 @@ This directory has terraform configuration necessary to achieve a infrastructure corresponding to the [Single-cluster reference architecture for Gitpod on AWS](https://www.gitpod.io/docs/self-hosted/latest/reference-architecture/single-cluster-ref-arch). This module will do the following steps: -- Create an EKS managed cluster -- Creates external dependencies like database, storage and registry (if chosen) -- Sets up route53 entries for the domain name (if chosen) -- Set up cert-manager, external-dns and cluster issuer on the created cluster +- Creates the infrastructure using our [`eks` terraform module](../../modules/eks/), which does: + - Setup an EKS managed cluster, along with external dependencies like database, storage and registry (if chosen) + - Sets up route53 entries for the domain name (if chosen) +- Provisioning the cluster: + - Set up cert-manager using our [`cert-manager` module](../../modules/tools/cert-manager/) + - Set up external-dns using our [`external-dns` module](../../modules/tools/external-dns/) + - Creates a cluster-issuer using our [`issuer` module](../../modules/tools/issuer/) Since the entire setup requires more than one terraform target to be run due to dependencies (eg: helm provider depends on kubernetes cluster config, which is @@ -19,6 +22,7 @@ explain the execution of the terraform module in terms of these `make` targets. * `terraform` >= `v1.1.0` * `kubectl` >= `v1.20.0` +* [`jq`](https://stedolan.github.io/jq/download/) ## Setup AWS authentication and s3 backend storage @@ -47,7 +51,7 @@ by terraform to create the cluster. While some of them are fairly straightforward like the name of the cluster(`cluster_name`), others need a bit more attention: -* VPC CIDR IP +### VPC CIDR IP It is necessary to ensure that the `VPC` setup will not have conflicts with existing VPCs or has sufficiently large enough IP range so as to not run out of @@ -55,25 +59,28 @@ them. Please check under the region's VPCs if the IP range you are choosing is already in use. The CIDR will be split among 5 subnets and hence we recommend `/16` as prefix to allow sufficient IP ranges. The default value is: `10.100.0.0/16` -* External database, storage and registry +### External database, storage and registry backend -If you wish to create cloud specific database, storage and registry to be used +If you wish to create cloud specific database, storage and registry backend to be used with `Gitpod`, leave the following 3 booleans set: ``` sh -enable_external_database = true -enable_external_storage = true -enable_external_registry = true +enable_external_database = true +enable_external_storage = true +enable_external_storage_for_registry_backend = true ``` The corresponding resources will be created by the terraform script which -inclustes an `RDS` mysql database, an `S3` bucket and an AWS container registry setup. The -expectation is that you can use the credentials to these setups(provided later +inclustes an `RDS` mysql database, an `S3` bucket and another `S3` bucket to +be used as registry backend. By default `enable_external_storage_for_registry_backend` +is set to `false`. One can re-use the same `S3` bucket for both object storage and registry backend. + +The expectation is that you can use the credentials to these setups(provided later as terraform outputs) during the setup of Gitpod via UI later in the process. Alternatively, one can choose to use incluster dependencies or separately created resources of choice. -* AMI Image ID and Kubernetes version +### AMI Image ID and Kubernetes version We officially support Ubuntu images for Gitpod setup. In EKS cluster, AMI images are kubernetes version and region specific. You can find a list of AMI IDs @@ -82,7 +89,7 @@ are kubernetes version and region specific. You can find a list of AMI IDs Make sure you provide the corresponding kubernetes version as a value to the variable `cluster_version`. We officially support kubernetes versions >= `1.20`. -* Domain name configuration +### Domain name configuration If you are already sure of the domain name under which you want to setup Gitpod, we recommend highly to provide the value as `domain_name`. This will save a lot @@ -139,19 +146,15 @@ cluster. Now you have to configure whichever provider you use to host your domain name to route traffic to the AWS name servers. You can find these name servers in the `make output` command from above. It would be of the format: -``` json - "nameservers": { - "sensitive": false, - "type": [ - "list", - "string" - ], - "value": [ - "ns-100.awsdns-12.com.", - "ns-1434.awsdns-51.org.", - "ns-1765.awsdns-28.co.uk.", - "ns-786.awsdns-34.net." - ] +```json +Nameservers for the domain(to be added as NS records in your domain provider): +================= +[ + "ns-1444.awsdns-52.org.", + "ns-1559.awsdns-02.co.uk.", + "ns-209.awsdns-26.com.", + "ns-969.awsdns-57.net." +] ``` Add the `ns` records similar to the above 4 URIs as NS records under your domain @@ -182,3 +185,47 @@ kubectl kots install gitpod/stable # you can replace `stable` with `unstable` or Upon completion, you can access `KOTS` UI in `localhost:8800`. Here you can proceed to configuring and intalling Gitpod. Please follow the [official documentaion](https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod) to complete the Gitpod setup. + +## Troubleshooting + +### Some pods never start (Init state) + +```sh +kubectl get pods -l component=proxy +NAME READY STATUS RESTARTS AGE +proxy-5998488f4c-t8vkh 0/1 Init 0/1 0 5m +``` + +The most likely reason is that the DNS01 challenge has yet to resolve. To fix this, make sure you have added the NS records corresponding to the `route53` zone of the `domain_name` added to your domain provider. + +Once the DNS record has been updated, you will need to delete all Cert Manager pods to retrigger the certificate request + +``` +kubectl delete pods -n cert-manager --all +``` + +After a few minutes, you should see the https-certificate become ready. + +``` +kubectl get certificate +NAME READY SECRET AGE +https-certificates True https-certificates 5m +``` + +## Cleanup + +Make sure you first delete the `gitpod` resources in the cluster so things like load balancer created by the k8s `service` gets deleted. Otherwise terraform will not be able to delete the VPC. + +```sh +kubectl delete --now namespace gitpod +``` + +> It is okay to ignore any dangling workspaces that are not deleted + +Now run the terraform destroy step to cleanup all the cloud resources: + +```sh +make destroy +``` + +The destroy should take around 20 minutes. diff --git a/install/infra/single-cluster/aws/cluster.tf b/install/infra/single-cluster/aws/cluster.tf index ae4c22c3f78dd7..8119e59eb297ed 100644 --- a/install/infra/single-cluster/aws/cluster.tf +++ b/install/infra/single-cluster/aws/cluster.tf @@ -10,7 +10,8 @@ module "eks" { kubeconfig = var.kubeconfig enable_external_database = var.enable_external_database enable_external_storage = var.enable_external_storage - enable_external_registry = var.enable_external_registry service_machine_type = "m6i.xlarge" workspace_machine_type = "m6i.2xlarge" + + enable_external_storage_for_registry_backend = var.enable_external_storage_for_registry_backend } diff --git a/install/infra/single-cluster/aws/output.tf b/install/infra/single-cluster/aws/output.tf index 896cb50b3c7274..b4c4942b4551d2 100644 --- a/install/infra/single-cluster/aws/output.tf +++ b/install/infra/single-cluster/aws/output.tf @@ -1,14 +1,22 @@ -output "aws_storage" { +output "url" { + value = var.domain_name +} + +output "cluster_name" { + value = var.cluster_name +} + +output "registry_backend" { sensitive = true - value = module.eks.storage + value = module.eks.registry_backend } -output "aws_registry" { +output "storage" { sensitive = true - value = module.eks.registry + value = module.eks.storage } -output "aws_database" { +output "database" { sensitive = true value = module.eks.database } @@ -17,3 +25,8 @@ output "nameservers" { sensitive = false value = module.eks.domain_nameservers } + +output "cluster_issuer" { + sensitive = false + value = module.cluster-issuer.cluster_issuer +} diff --git a/install/infra/single-cluster/aws/terraform.tfvars b/install/infra/single-cluster/aws/terraform.tfvars index c7b935c0df2571..e37b8ba3027eb1 100644 --- a/install/infra/single-cluster/aws/terraform.tfvars +++ b/install/infra/single-cluster/aws/terraform.tfvars @@ -1,18 +1,26 @@ -cluster_name = "new-cluster" +# the cluster_name should be of length less than 16 characters +cluster_name = "nan-cluster" -image_id = "ami-0793b4124359a6ad7" +# a route53 zone and certificate request will be created for this domain +domain_name = "nan-cluster.gitpod-self-hosted.com" region = "eu-west-1" -vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] - +# make sure the cidr do not have any conflicts and will have IP ranges enough to split into 5 subnets vpc_cidr = "10.100.0.0/16" -domain_name = "new-cluster-gitpoid.com" +# should be atleast 2 zones +vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] +# you can find the list of UBUNTU AMIs here corresponding to the k8s version and your region +# https://cloud-images.ubuntu.com/docs/aws/eks/ cluster_version = "1.22" +image_id = "ami-0793b4124359a6ad7" enable_external_database = true enable_external_storage = true -enable_external_registry = true + +# if you want to create a separate s3 bucket to use as registry backend, +# set the following to true. You can re-use the above bucket or incluster registry otherwise. +enable_external_storage_for_registry_backend = false diff --git a/install/infra/single-cluster/aws/variables.tf b/install/infra/single-cluster/aws/variables.tf index 79083f381971b7..a2f921a729d26a 100644 --- a/install/infra/single-cluster/aws/variables.tf +++ b/install/infra/single-cluster/aws/variables.tf @@ -47,9 +47,9 @@ variable "enable_external_storage" { description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" } -variable "enable_external_registry" { - default = true - description = "Set this to false to avoid creating an AWS registry to use with Gitpod instead of incluster registry" +variable "enable_external_storage_for_registry_backend" { + default = false + description = "Set this to true to create a separate S3 bucket to use as registry backend(if not, you can use the same bucket as above or the incluster registry)" } variable "cluster_version" { From 166c412958daf2749aeff0809beffa0d8b879945 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Fri, 5 Aug 2022 13:03:42 +0000 Subject: [PATCH 4/9] adding output for cluster issuer --- .werft/installer-tests.ts | 4 ++-- install/infra/modules/eks/output.tf | 14 ++++++------- install/infra/modules/eks/storage.tf | 22 ++++++++++++++------ install/infra/modules/tools/issuer/output.tf | 3 +++ install/infra/single-cluster/aws/Makefile | 2 +- install/infra/single-cluster/aws/README.md | 13 ++++++++++++ install/tests/Makefile | 2 +- 7 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 install/infra/modules/tools/issuer/output.tf diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 93a71a4406b31f..5d8cde05ff12c8 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -269,7 +269,7 @@ export async function installerTests(config: TestConfig) { console.log(config.DESCRIPTION); // these phases sets up or clean up the infrastructure // If the cloud variable is not set, we have a cleanup job in hand - const majorPhase: string = cloud == "" ? `create-${cloud}-infra` : "cleanup-infra" + const majorPhase: string = cloud == "" ? "cleanup-infra" :`create-${cloud}-infra` werft.phase(majorPhase, `Manage the infrastructure in ${cloud}`); for (let phase of config.PHASES) { @@ -411,7 +411,7 @@ function randK8sVersion(config: string): string { break; } case "STANDARD_EKS_TEST": { - options = ["1.20", "1.21", "1.22"] + options = ["1.21", "1.22"] break; } case "STANDARD_K3S_TEST": { diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf index 8ee3e11d34fced..ba730e699e06a1 100644 --- a/install/infra/modules/eks/output.tf +++ b/install/infra/modules/eks/output.tf @@ -45,9 +45,9 @@ output "database" { sensitive = true value = try({ host = "${aws_db_instance.gitpod[0].address}" + username = "${aws_db_instance.gitpod[0].username}" password = random_password.password[0].result port = 3306 - username = "${aws_db_instance.gitpod[0].username}" }, "No database created") } @@ -63,21 +63,21 @@ output "registry" { output "storage" { sensitive = true value = try({ - access_key_id = aws_iam_access_key.bucket_storage_user[0].id - secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret region = aws_s3_bucket.gitpod-storage[0].region - bucket_name = aws_s3_bucket.gitpod-storage[0].id endpoint = "s3.${aws_s3_bucket.gitpod-storage[0].region}.amazonaws.com" + bucket_name = aws_s3_bucket.gitpod-storage[0].id + access_key_id = aws_iam_access_key.bucket_storage_user[0].id + secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret }, "No s3 bucket created for object storage") } output "registry_backend" { sensitive = true value = try({ - access_key_id = aws_iam_access_key.bucket_storage_user[0].id - secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret region = aws_s3_bucket.gitpod-registry-backend[0].region - bucket_name = aws_s3_bucket.gitpod-registry-backend[0].id endpoint = "s3.${aws_s3_bucket.gitpod-registry-backend[0].region}.amazonaws.com" + bucket_name = aws_s3_bucket.gitpod-registry-backend[0].id + access_key_id = aws_iam_access_key.bucket_storage_user[0].id + secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret }, "No s3 bucket created for registry backend.") } diff --git a/install/infra/modules/eks/storage.tf b/install/infra/modules/eks/storage.tf index bb6facd01c5449..d53269d8814101 100644 --- a/install/infra/modules/eks/storage.tf +++ b/install/infra/modules/eks/storage.tf @@ -4,10 +4,6 @@ resource "aws_s3_bucket" "gitpod-storage" { force_destroy = true bucket = "bucket-${var.cluster_name}" acl = "private" - - versioning { - enabled = true - } } resource "aws_s3_bucket" "gitpod-registry-backend" { @@ -16,9 +12,23 @@ resource "aws_s3_bucket" "gitpod-registry-backend" { force_destroy = true bucket = "reg-bucket-${var.cluster_name}" acl = "private" +} + +resource "aws_s3_bucket_versioning" "storage" { + count = var.enable_external_storage ? 1 : 0 + + bucket = aws_s3_bucket.gitpod-storage[0].id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_versioning" "registry" { + count = var.enable_external_storage_for_registry_backend ? 1 : 0 - versioning { - enabled = true + bucket = aws_s3_bucket.gitpod-registry-backend[0].id + versioning_configuration { + status = "Enabled" } } diff --git a/install/infra/modules/tools/issuer/output.tf b/install/infra/modules/tools/issuer/output.tf new file mode 100644 index 00000000000000..a82529cfb25888 --- /dev/null +++ b/install/infra/modules/tools/issuer/output.tf @@ -0,0 +1,3 @@ +output "cluster_issuer" { + value = var.cluster_issuer_name +} diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile index a57249af1c77e7..63d56077a631cb 100644 --- a/install/infra/single-cluster/aws/Makefile +++ b/install/infra/single-cluster/aws/Makefile @@ -46,7 +46,7 @@ output-storage: output-registry: @echo "\nS3 registry backend:" @echo "==================" - @terraform output -json registry | jq + @terraform output -json registry_backend | jq output-database: @echo "\nDatabase:" diff --git a/install/infra/single-cluster/aws/README.md b/install/infra/single-cluster/aws/README.md index cfaa01278816f9..5136c8b99ebcdb 100644 --- a/install/infra/single-cluster/aws/README.md +++ b/install/infra/single-cluster/aws/README.md @@ -12,6 +12,9 @@ This module will do the following steps: - Set up external-dns using our [`external-dns` module](../../modules/tools/external-dns/) - Creates a cluster-issuer using our [`issuer` module](../../modules/tools/issuer/) +> ๐Ÿ’ก If you would like to create the infrastructure orchestrating the terraform modules by yourself, you can find all the modules we support [here](../../modules/). + + Since the entire setup requires more than one terraform target to be run due to dependencies (eg: helm provider depends on kubernetes cluster config, which is not available until the `eks` module finishes), this directory has a `Makefile` @@ -137,6 +140,8 @@ You can list all the other outputs with the following command: make output ``` +> ๐Ÿ’ก Alternatively, you can get the simple JSON output with a `terraform output` command + ## Note the NS records from terraform output Once the apply process has exited successfully, we can go ahead and prepare to @@ -212,6 +217,14 @@ NAME READY SECRET AGE https-certificates True https-certificates 5m ``` +### Cannot connect to the created cluster after a while + +There is a chance that your kubeconfig has gotten expired after a specific amount of time. You can reconnect to the cluster by using: + +``` sh +aws eks --region update-kubeconfig --name +``` + ## Cleanup Make sure you first delete the `gitpod` resources in the cluster so things like load balancer created by the k8s `service` gets deleted. Otherwise terraform will not be able to delete the VPC. diff --git a/install/tests/Makefile b/install/tests/Makefile index 9f3520e5ef8500..dadec305911a42 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -250,7 +250,7 @@ generate-kots-config: cloud_storage = $(if $(findstring external,$(storage)),$(c generate-kots-config: cloud_registry = $(if $(findstring external,$(registry)),$(cloud),incluster) generate-kots-config: cloud_db = $(if $(findstring external,$(db)),$(cloud),incluster) ## generate-kots-config: Generate the kots config based on test config -generate-kots-config: check-env-cloud get-base-config +generate-kots-config: select-workspace check-env-cloud get-base-config $(MAKE) storage-config-${cloud_storage} $(MAKE) db-config-${cloud_db} $(MAKE) registry-config-${cloud_registry} From a43bd6cd52777792b42f3f6873d820fabf956ad8 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 9 Aug 2022 10:36:04 +0000 Subject: [PATCH 5/9] Adding refresh by default --- install/infra/modules/eks/database.tf | 23 ++++---- install/infra/modules/eks/kubernetes.tf | 70 +++++++++++++++++++---- install/infra/single-cluster/aws/Makefile | 21 ++++++- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/install/infra/modules/eks/database.tf b/install/infra/modules/eks/database.tf index 82ea111adc5698..78154438b3c541 100644 --- a/install/infra/modules/eks/database.tf +++ b/install/infra/modules/eks/database.tf @@ -19,13 +19,6 @@ resource "aws_security_group" "rdssg" { name = "dh-sg-${var.cluster_name}" vpc_id = module.vpc.vpc_id - ingress { - from_port = 0 - to_port = 3306 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - egress { from_port = 0 to_port = 0 @@ -34,14 +27,24 @@ resource "aws_security_group" "rdssg" { } } +resource "aws_security_group_rule" "db-ingress-nodes" { + description = "Allow nodes to communicate with the db" + from_port = 0 + protocol = "tcp" + cidr_blocks = [var.vpc_cidr] + security_group_id = aws_security_group.rdssg[0].id + to_port = 3306 + type = "ingress" +} + resource "aws_db_instance" "gitpod" { count = var.enable_external_database ? 1 : 0 - allocated_storage = 10 - max_allocated_storage = 100 + allocated_storage = 20 + max_allocated_storage = 120 engine = "mysql" engine_version = "5.7" - instance_class = "db.t3.micro" + instance_class = "db.m5.large" vpc_security_group_ids = [aws_security_group.rdssg[0].id] identifier = "db-${var.cluster_name}" name = "gitpod" diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index 3bef25d99b283e..b80bcaf05aa9a6 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -20,14 +20,46 @@ module "vpc" { enable_dns_hostnames = true } +resource "aws_security_group_rule" "eks-worker-ingress-self" { + description = "Allow node to communicate with each other" + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes.id + source_security_group_id = aws_security_group.nodes.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "eks-worker-ingress-cluster" { + description = "Allow worker Kubelets and pods to receive communication from the cluster control plane" + from_port = 1025 + protocol = "tcp" + security_group_id = aws_security_group.nodes.id + source_security_group_id = aws_security_group.nodes.id + to_port = 65535 + type = "ingress" +} + +### Worker Node Access to EKS Master +resource "aws_security_group_rule" "eks-cluster-ingress-node-https" { + description = "Allow pods to communicate with the cluster API Server" + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.nodes.id + source_security_group_id = aws_security_group.nodes.id + to_port = 443 + type = "ingress" +} + + resource "aws_security_group" "nodes" { name = "nodes-sg-${var.cluster_name}" vpc_id = module.vpc.vpc_id ingress { from_port = 0 - to_port = 0 - protocol = "-1" + to_port = 6443 + protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } @@ -46,7 +78,8 @@ module "eks" { cluster_name = var.cluster_name cluster_version = var.cluster_version - cluster_endpoint_public_access = true + cluster_endpoint_public_access = true + cluster_endpoint_private_access = true vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.public_subnets @@ -68,6 +101,7 @@ module "eks" { ami_id = var.image_id enable_bootstrap_user_data = true vpc_security_group_ids = [aws_security_group.nodes.id] + ebs_optimized = true } eks_managed_node_groups = { @@ -77,19 +111,29 @@ module "eks" { name = "service-${var.cluster_name}" subnet_ids = module.vpc.public_subnets min_size = 1 - max_size = 10 - desired_size = 1 + max_size = 4 + desired_size = 2 block_device_mappings = [{ device_name = "/dev/sda1" ebs = [{ - volume_size = 150 + volume_size = 300 + volume_type = "gp3" + throughput = 500 + iops = 6000 + delete_on_termination = true }] }] labels = { "gitpod.io/workload_meta" = true "gitpod.io/workload_ide" = true } + + tags = { + "k8s.io/cluster-autoscaler/enabled" = true + "k8s.io/cluster-autoscaler/gitpod" = "owned" + } + pre_bootstrap_user_data = <<-EOT #!/bin/bash set -ex @@ -97,7 +141,7 @@ module "eks" { export CONTAINER_RUNTIME="containerd" export USE_MAX_PODS=false EOF - # Source extra environment variables in bootstrap script + # Source extra environment 5ariables in bootstrap script sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT } @@ -107,21 +151,27 @@ module "eks" { name = "ws-${var.cluster_name}" subnet_ids = module.vpc.public_subnets min_size = 1 - max_size = 10 + max_size = 50 block_device_mappings = [{ device_name = "/dev/sda1" ebs = [{ - volume_size = 150 + volume_size = 300 }] }] - desired_size = 1 + desired_size = 2 enable_bootstrap_user_data = true labels = { "gitpod.io/workload_workspace_services" = true "gitpod.io/workload_workspace_regular" = true "gitpod.io/workload_workspace_headless" = true } + + tags = { + "k8s.io/cluster-autoscaler/enabled" = true + "k8s.io/cluster-autoscaler/gitpod" = "owned" + } + pre_bootstrap_user_data = <<-EOT #!/bin/bash set -ex diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile index 63d56077a631cb..abf80fcce98ece 100644 --- a/install/infra/single-cluster/aws/Makefile +++ b/install/infra/single-cluster/aws/Makefile @@ -6,8 +6,14 @@ init: @terraform init +touch-kubeconfig: + @touch kubeconfig + +cleanup-kubeconfig: + @rm kubeconfig + .PHONY: plan -plan: plan-cluster plan-cm-edns +plan: touch-kubeconfig plan-cluster plan-cm-edns cleanup-kubeconfig .PHONY: apply apply: apply-cluster apply-tools @@ -15,13 +21,22 @@ apply: apply-cluster apply-tools .PHONY: destroy destroy: destroy-tools destroy-cluster +.PHONY: refresh +refresh: + @echo "Refreshing terraform state" + @terraform refresh + @echo "" + @echo "Done!" + .PHONY: output -output: output-done-msg output-url output-nameservers output-registry output-database output-storage output-issuer +output: refresh output-done-msg output-url output-nameservers output-registry output-database output-storage output-issuer output-done-msg: + @echo "" + @echo "" @echo "==========================" @echo "๐ŸŽ‰๐Ÿฅณ๐Ÿ”ฅ๐Ÿงก๐Ÿš€" - @echo "Your cloud infrastructure is ready to install Gitpod. Please visit" + @echo "Your AWS cloud infrastructure is ready to install Gitpod. Please visit" @echo "https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod" @echo "for your next steps." @echo "=================" From 05cc5edcd154ffa4dd0d8b39c46237cd8ca66222 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 11 Aug 2022 05:07:39 +0200 Subject: [PATCH 6/9] Update install/infra/modules/eks/storage.tf Co-authored-by: Adrien Thebo --- install/infra/modules/eks/storage.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/infra/modules/eks/storage.tf b/install/infra/modules/eks/storage.tf index d53269d8814101..ab17933de395a1 100644 --- a/install/infra/modules/eks/storage.tf +++ b/install/infra/modules/eks/storage.tf @@ -44,7 +44,7 @@ data "aws_iam_policy_document" "s3_policy" { resource "aws_iam_policy" "policy" { count = var.enable_external_storage ? 1 : 0 name = "spolicy-${var.cluster_name}" - description = "s3 storage bucket policy" + description = "Gitpod ${var.cluster_name} object storage bucket policy" policy = data.aws_iam_policy_document.s3_policy[0].json } From 3646ecc7fbefc7657b6e993f3fdcbc7594b478cd Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 11 Aug 2022 14:29:15 +0000 Subject: [PATCH 7/9] addressing the review comments --- install/infra/modules/eks/database.tf | 17 +-- install/infra/modules/eks/kubernetes.tf | 7 - install/infra/modules/eks/output.tf | 4 +- install/infra/modules/eks/registry.tf | 4 +- install/infra/modules/eks/storage.tf | 123 ++++++++++++++---- install/infra/modules/eks/variables.tf | 16 +-- install/infra/single-cluster/aws/cluster.tf | 6 +- install/infra/single-cluster/aws/main.tf | 2 +- .../infra/single-cluster/aws/terraform.tfvars | 10 +- install/infra/single-cluster/aws/variables.tf | 12 +- 10 files changed, 131 insertions(+), 70 deletions(-) diff --git a/install/infra/modules/eks/database.tf b/install/infra/modules/eks/database.tf index 78154438b3c541..cbd26c7be1419f 100644 --- a/install/infra/modules/eks/database.tf +++ b/install/infra/modules/eks/database.tf @@ -1,5 +1,5 @@ resource "random_password" "password" { - count = var.enable_external_database ? 1 : 0 + count = var.create_external_database ? 1 : 0 length = 16 special = true @@ -7,24 +7,17 @@ resource "random_password" "password" { } resource "aws_db_subnet_group" "gitpod_subnets" { - count = var.enable_external_database ? 1 : 0 + count = var.create_external_database ? 1 : 0 name = "db-sg-${var.cluster_name}" subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] } resource "aws_security_group" "rdssg" { - count = var.enable_external_database ? 1 : 0 + count = var.create_external_database ? 1 : 0 name = "dh-sg-${var.cluster_name}" vpc_id = module.vpc.vpc_id - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } } resource "aws_security_group_rule" "db-ingress-nodes" { @@ -38,7 +31,7 @@ resource "aws_security_group_rule" "db-ingress-nodes" { } resource "aws_db_instance" "gitpod" { - count = var.enable_external_database ? 1 : 0 + count = var.create_external_database ? 1 : 0 allocated_storage = 20 max_allocated_storage = 120 @@ -53,5 +46,5 @@ resource "aws_db_instance" "gitpod" { parameter_group_name = "default.mysql5.7" db_subnet_group_name = aws_db_subnet_group.gitpod_subnets[0].name skip_final_snapshot = true - publicly_accessible = true + publicly_accessible = false } diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index b80bcaf05aa9a6..af7a1279bfa8bb 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -56,13 +56,6 @@ resource "aws_security_group" "nodes" { name = "nodes-sg-${var.cluster_name}" vpc_id = module.vpc.vpc_id - ingress { - from_port = 0 - to_port = 6443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - egress { from_port = 0 to_port = 0 diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf index ba730e699e06a1..6acad0d24ccebc 100644 --- a/install/infra/modules/eks/output.tf +++ b/install/infra/modules/eks/output.tf @@ -77,7 +77,7 @@ output "registry_backend" { region = aws_s3_bucket.gitpod-registry-backend[0].region endpoint = "s3.${aws_s3_bucket.gitpod-registry-backend[0].region}.amazonaws.com" bucket_name = aws_s3_bucket.gitpod-registry-backend[0].id - access_key_id = aws_iam_access_key.bucket_storage_user[0].id - secret_access_key = aws_iam_access_key.bucket_storage_user[0].secret + access_key_id = aws_iam_access_key.bucket_registry_user[0].id + secret_access_key = aws_iam_access_key.bucket_registry_user[0].secret }, "No s3 bucket created for registry backend.") } diff --git a/install/infra/modules/eks/registry.tf b/install/infra/modules/eks/registry.tf index 87d07bbab50eb2..e99bd739d19a2b 100644 --- a/install/infra/modules/eks/registry.tf +++ b/install/infra/modules/eks/registry.tf @@ -1,5 +1,5 @@ resource "aws_ecr_repository" "gitpod" { - count = var.enable_external_registry ? 1 : 0 + count = var.create_external_registry ? 1 : 0 name = "registry-${var.cluster_name}" image_tag_mutability = "MUTABLE" @@ -10,6 +10,6 @@ resource "aws_ecr_repository" "gitpod" { } data "aws_ecr_authorization_token" "gitpod" { - count = var.enable_external_registry ? 1 : 0 + count = var.create_external_registry ? 1 : 0 registry_id = aws_ecr_repository.gitpod[0].registry_id } diff --git a/install/infra/modules/eks/storage.tf b/install/infra/modules/eks/storage.tf index ab17933de395a1..5ba009d9b48273 100644 --- a/install/infra/modules/eks/storage.tf +++ b/install/infra/modules/eks/storage.tf @@ -1,21 +1,19 @@ resource "aws_s3_bucket" "gitpod-storage" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 force_destroy = true bucket = "bucket-${var.cluster_name}" - acl = "private" } -resource "aws_s3_bucket" "gitpod-registry-backend" { - count = var.enable_external_storage_for_registry_backend ? 1 : 0 +resource "aws_s3_bucket_acl" "gitpod-storage" { + count = var.create_external_storage ? 1 : 0 - force_destroy = true - bucket = "reg-bucket-${var.cluster_name}" - acl = "private" + bucket = aws_s3_bucket.gitpod-storage[count.index].id + acl = "private" } resource "aws_s3_bucket_versioning" "storage" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 bucket = aws_s3_bucket.gitpod-storage[0].id versioning_configuration { @@ -23,44 +21,121 @@ resource "aws_s3_bucket_versioning" "storage" { } } -resource "aws_s3_bucket_versioning" "registry" { - count = var.enable_external_storage_for_registry_backend ? 1 : 0 - - bucket = aws_s3_bucket.gitpod-registry-backend[0].id - versioning_configuration { - status = "Enabled" - } -} - data "aws_iam_policy_document" "s3_policy" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 statement { - actions = ["s3:*"] - resources = ["*"] + actions = [ + "s3:PutObject", + "s3:ListMultipartUploadParts", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload" + ] + resources = [ + "arn:aws:s3:::${aws_s3_bucket.gitpod-storage[count.index].id}", + ] effect = "Allow" } } resource "aws_iam_policy" "policy" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 name = "spolicy-${var.cluster_name}" description = "Gitpod ${var.cluster_name} object storage bucket policy" policy = data.aws_iam_policy_document.s3_policy[0].json } resource "aws_iam_user" "bucket_storage" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 name = "suser-${var.cluster_name}" } resource "aws_iam_user_policy_attachment" "attachment" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 user = aws_iam_user.bucket_storage[0].name policy_arn = aws_iam_policy.policy[0].arn } +resource "aws_iam_user_policy_attachment" "full_access_attachment" { + count = var.create_external_storage ? 1 : 0 + user = aws_iam_user.bucket_storage[0].name + policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess" +} + resource "aws_iam_access_key" "bucket_storage_user" { - count = var.enable_external_storage ? 1 : 0 + count = var.create_external_storage ? 1 : 0 user = aws_iam_user.bucket_storage[0].name } + +// s3 bucket for registry backend + +resource "aws_s3_bucket" "gitpod-registry-backend" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + + force_destroy = true + bucket = "reg-bucket-${var.cluster_name}" +} + +resource "aws_s3_bucket_acl" "gitpod-registry-storage" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + + bucket = aws_s3_bucket.gitpod-registry-backend[count.index].id + acl = "private" +} + +resource "aws_s3_bucket_versioning" "registry" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + + bucket = aws_s3_bucket.gitpod-registry-backend[0].id + versioning_configuration { + status = "Enabled" + } +} + +data "aws_iam_policy_document" "s3_policy_registry" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + statement { + actions = [ + "s3:PutObject", + "s3:ListMultipartUploadParts", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload" + ] + resources = [ + "arn:aws:s3:::${aws_s3_bucket.gitpod-registry-backend[count.index].id}", + ] + effect = "Allow" + } +} + +resource "aws_iam_policy" "policy_registry" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + name = "registry-policy-${var.cluster_name}" + description = "Gitpod ${var.cluster_name} registry backend storage bucket policy" + policy = data.aws_iam_policy_document.s3_policy_registry[count.index].json +} + +resource "aws_iam_user" "bucket_registry" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + name = "registry-user-${var.cluster_name}" + +} + +resource "aws_iam_user_policy_attachment" "registry_attachment" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + user = aws_iam_user.bucket_registry[count.index].name + policy_arn = aws_iam_policy.policy_registry[count.index].arn +} + +resource "aws_iam_user_policy_attachment" "full_access_registry_attachment" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + user = aws_iam_user.bucket_registry[count.index].name + policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess" +} + +resource "aws_iam_access_key" "bucket_registry_user" { + count = var.create_external_storage_for_registry_backend ? 1 : 0 + user = aws_iam_user.bucket_registry[count.index].name +} diff --git a/install/infra/modules/eks/variables.tf b/install/infra/modules/eks/variables.tf index 2f13bb57e31bc3..c1ff5f4bfb6ec2 100644 --- a/install/infra/modules/eks/variables.tf +++ b/install/infra/modules/eks/variables.tf @@ -51,22 +51,22 @@ variable "vpc_cidr" { default = "10.100.0.0/16" } -variable "enable_external_database" { +variable "create_external_database" { default = true - description = "Set this to false to avoid creating an RDS database to use with Gitpod instead of inclsuter mysql" + description = "Create a mysql RDS database" } -variable "enable_external_storage" { +variable "create_external_storage" { default = true - description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" + description = "Create an S3 bucket" } -variable "enable_external_storage_for_registry_backend" { +variable "create_external_storage_for_registry_backend" { default = false - description = "Set this to true to create a separate s3 storage to use with Gitpod as registry backend" + description = "Create an S3 bucket for registry backend" } -variable "enable_external_registry" { +variable "create_external_registry" { default = false - description = "Set this to true to create an AWS ECR registry to use with Gitpod(Not officially supported)" + description = "Create an EKS registry(Not officially supported)" } diff --git a/install/infra/single-cluster/aws/cluster.tf b/install/infra/single-cluster/aws/cluster.tf index 8119e59eb297ed..e8631c7ee266f5 100644 --- a/install/infra/single-cluster/aws/cluster.tf +++ b/install/infra/single-cluster/aws/cluster.tf @@ -8,10 +8,10 @@ module "eks" { image_id = var.image_id cluster_version = var.cluster_version kubeconfig = var.kubeconfig - enable_external_database = var.enable_external_database - enable_external_storage = var.enable_external_storage + create_external_database = var.create_external_database + create_external_storage = var.create_external_storage service_machine_type = "m6i.xlarge" workspace_machine_type = "m6i.2xlarge" - enable_external_storage_for_registry_backend = var.enable_external_storage_for_registry_backend + create_external_storage_for_registry_backend = var.create_external_storage_for_registry_backend } diff --git a/install/infra/single-cluster/aws/main.tf b/install/infra/single-cluster/aws/main.tf index 9ae7aee2fbf265..2cd2e408fcf459 100644 --- a/install/infra/single-cluster/aws/main.tf +++ b/install/infra/single-cluster/aws/main.tf @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "gitpod-tf" + bucket = "nan-tf-bucket" key = "aws/terraform.state" } } diff --git a/install/infra/single-cluster/aws/terraform.tfvars b/install/infra/single-cluster/aws/terraform.tfvars index e37b8ba3027eb1..12618b02599ce0 100644 --- a/install/infra/single-cluster/aws/terraform.tfvars +++ b/install/infra/single-cluster/aws/terraform.tfvars @@ -1,9 +1,9 @@ # the cluster_name should be of length less than 16 characters -cluster_name = "nan-cluster" +cluster_name = "nan" # a route53 zone and certificate request will be created for this domain -domain_name = "nan-cluster.gitpod-self-hosted.com" +domain_name = "nan-cluster.doptig.com" region = "eu-west-1" @@ -18,9 +18,9 @@ vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] cluster_version = "1.22" image_id = "ami-0793b4124359a6ad7" -enable_external_database = true -enable_external_storage = true +create_external_database = true +create_external_storage = true # if you want to create a separate s3 bucket to use as registry backend, # set the following to true. You can re-use the above bucket or incluster registry otherwise. -enable_external_storage_for_registry_backend = false +create_external_storage_for_registry_backend = false diff --git a/install/infra/single-cluster/aws/variables.tf b/install/infra/single-cluster/aws/variables.tf index a2f921a729d26a..6aba467c201fa0 100644 --- a/install/infra/single-cluster/aws/variables.tf +++ b/install/infra/single-cluster/aws/variables.tf @@ -37,19 +37,19 @@ variable "domain_name" { description = "Domain name to associate with the Gitpod installation" } -variable "enable_external_database" { +variable "create_external_database" { default = true - description = "Set this to false to avoid creating an RDS database to use with Gitpod instead of inclsuter mysql" + description = "Create a mysql RDS database" } -variable "enable_external_storage" { +variable "create_external_storage" { default = true - description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" + description = "Create an S3 bucket" } -variable "enable_external_storage_for_registry_backend" { +variable "create_external_storage_for_registry_backend" { default = false - description = "Set this to true to create a separate S3 bucket to use as registry backend(if not, you can use the same bucket as above or the incluster registry)" + description = "Create an S3 bucket for registry backend" } variable "cluster_version" { From 66e24cf935f185a0daa0638576083b5304dd7a22 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 26 Jul 2022 13:08:44 +0000 Subject: [PATCH 8/9] Add terraform configurations for single-cluster ref arch --- install/infra/modules/gke/main.tf | 47 +++- install/infra/modules/gke/output.tf | 33 +++ install/infra/modules/gke/variables.tf | 22 +- .../tools/cloud-dns-external-dns/main.tf | 8 +- .../tools/cloud-dns-external-dns/variables.tf | 6 +- install/infra/single-cluster/aws/README.md | 4 - .../infra/single-cluster/aws/terraform.tfvars | 1 - .../single-cluster/gcp/.terraform.lock.hcl | 97 +++++++ install/infra/single-cluster/gcp/Makefile | 121 +++++++++ install/infra/single-cluster/gcp/README.md | 238 ++++++++++++++++++ install/infra/single-cluster/gcp/cluster.tf | 16 ++ install/infra/single-cluster/gcp/main.tf | 20 ++ install/infra/single-cluster/gcp/output.tf | 36 +++ .../infra/single-cluster/gcp/terraform.tfvars | 16 ++ install/infra/single-cluster/gcp/tools.tf | 28 +++ install/infra/single-cluster/gcp/variables.tf | 54 ++++ install/tests/Makefile | 2 +- install/tests/main.tf | 2 +- 18 files changed, 726 insertions(+), 25 deletions(-) create mode 100644 install/infra/single-cluster/gcp/.terraform.lock.hcl create mode 100644 install/infra/single-cluster/gcp/Makefile create mode 100644 install/infra/single-cluster/gcp/README.md create mode 100644 install/infra/single-cluster/gcp/cluster.tf create mode 100644 install/infra/single-cluster/gcp/main.tf create mode 100644 install/infra/single-cluster/gcp/output.tf create mode 100644 install/infra/single-cluster/gcp/terraform.tfvars create mode 100644 install/infra/single-cluster/gcp/tools.tf create mode 100644 install/infra/single-cluster/gcp/variables.tf diff --git a/install/infra/modules/gke/main.tf b/install/infra/modules/gke/main.tf index cb9b7afc561688..b78de6366674f9 100644 --- a/install/infra/modules/gke/main.tf +++ b/install/infra/modules/gke/main.tf @@ -17,12 +17,12 @@ provider "google" { } resource "google_compute_network" "vpc" { - name = "vpc-${var.name}" + name = "vpc-${var.cluster_name}" auto_create_subnetworks = "false" } resource "google_compute_subnetwork" "subnet" { - name = "subnet-${var.name}" + name = "subnet-${var.cluster_name}" region = var.region network = google_compute_network.vpc.name ip_cidr_range = "10.255.0.0/16" @@ -39,7 +39,7 @@ resource "google_compute_subnetwork" "subnet" { } resource "google_container_cluster" "gitpod-cluster" { - name = "gitpod-${var.name}" + name = var.cluster_name location = var.zone == null ? var.region : var.zone cluster_autoscaling { @@ -112,7 +112,7 @@ resource "google_container_cluster" "gitpod-cluster" { } resource "google_container_node_pool" "workspaces" { - name = "workspaces-${var.name}" + name = "workspaces-${var.cluster_name}" location = google_container_cluster.gitpod-cluster.location cluster = google_container_cluster.gitpod-cluster.name version = var.cluster_version // kubernetes version @@ -153,7 +153,8 @@ resource "google_container_node_pool" "workspaces" { } resource "google_sql_database_instance" "gitpod" { - name = "sql-${var.name}" + count = var.enable_external_database ? 1 : 0 + name = "sql-${var.cluster_name}" database_version = "MYSQL_5_7" region = var.region settings { @@ -162,17 +163,43 @@ resource "google_sql_database_instance" "gitpod" { deletion_protection = false } +resource "random_password" "password" { + count = var.enable_external_database ? 1 : 0 + + length = 16 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + resource "google_sql_database" "database" { + count = var.enable_external_database ? 1 : 0 name = "gitpod" - instance = google_sql_database_instance.gitpod.name + instance = google_sql_database_instance.gitpod[count.index].name charset = "utf8" collation = "utf8_general_ci" } resource "google_sql_user" "users" { - name = "gitpod" - instance = google_sql_database_instance.gitpod.name - password = "gitpod" + count = var.enable_external_database ? 1 : 0 + name = "dbuser-${var.cluster_name}-${count.index}" + instance = google_sql_database_instance.gitpod[count.index].name + password = random_password.password[count.index].result +} + +resource "google_dns_managed_zone" "gitpod-dns-zone" { + count = var.domain_name == null ? 0 : 1 + + name = "zone-${var.cluster_name}" + dns_name = "${var.domain_name}." + description = "Terraform managed DNS zone for ${var.cluster_name}" + force_destroy = true + labels = { + app = "gitpod" + } +} + +data "google_container_registry_repository" "gitpod" { + count = var.enable_external_registry ? 1 : 0 } module "gke_auth" { @@ -182,7 +209,7 @@ module "gke_auth" { project_id = var.project location = google_container_cluster.gitpod-cluster.location - cluster_name = "gitpod-${var.name}" + cluster_name = var.cluster_name } resource "local_file" "kubeconfig" { diff --git a/install/infra/modules/gke/output.tf b/install/infra/modules/gke/output.tf index 470911d8a97705..1b47e86ac098d4 100644 --- a/install/infra/modules/gke/output.tf +++ b/install/infra/modules/gke/output.tf @@ -3,6 +3,10 @@ output "kubernetes_endpoint" { value = module.gke_auth.host } +output "name_servers" { + value = google_dns_managed_zone.gitpod-dns-zone[0].name_servers +} + output "client_token" { sensitive = true value = module.gke_auth.token @@ -17,3 +21,32 @@ output "kubeconfig" { sensitive = true value = module.gke_auth.kubeconfig_raw } + +output "database" { + sensitive = true + value = try({ + instance = "${var.project}:${var.region}:${google_sql_database_instance.gitpod[0].name}" + username = "${google_sql_user.users[0].name}" + password = random_password.password[0].result + service_account_key = "Upload the JSON file corresponding the service account credentials" + }, "No database created") +} + +output "registry" { + sensitive = true + value = try({ + url = data.google_container_registry_repository.gitpod[0].repository_url + server = regex("[^/?#]*", data.google_container_registry_repository.gitpod[0].repository_url) + username = "_json_key" + password = "Copy paste the content of the service account credentials JSON file" + }, "No container registry created") +} + +output "storage" { + sensitive = true + value = try({ + region = var.region + project = var.project + credentials = "Upload the JSON file corresponding the service account credentials" + }, "No GCS bucket created for object storage") +} diff --git a/install/infra/modules/gke/variables.tf b/install/infra/modules/gke/variables.tf index d7ee6fe554de81..6fc65bc0ea50dc 100644 --- a/install/infra/modules/gke/variables.tf +++ b/install/infra/modules/gke/variables.tf @@ -26,7 +26,7 @@ variable "cluster_version" { default = "1.22.8-gke.201" } -variable "name" { +variable "cluster_name" { type = string description = "The name of the cluster." default = "gitpod" @@ -60,3 +60,23 @@ variable "credentials" { description = "Path to the JSON file storing Google service account credentials" default = "" } + +variable "domain_name" { + description = "Domain name register with Cloud DNS, leave empty if you want to manage it yourself" + default = null +} + +variable "enable_external_database" { + default = true + description = "Set this to false to avoid creating an RDS database to use with Gitpod instead of incluster mysql" +} + +variable "enable_external_storage" { + default = true + description = "Set this to false to avoid creating an s3 storage to use with Gitpod instead of incluster minio" +} + +variable "enable_external_registry" { + default = true + description = "Set this to false to create an AWS ECR registry to use with Gitpod(Not officially supported)" +} diff --git a/install/infra/modules/tools/cloud-dns-external-dns/main.tf b/install/infra/modules/tools/cloud-dns-external-dns/main.tf index bb719f259abc5e..75c2d79bf6f38c 100644 --- a/install/infra/modules/tools/cloud-dns-external-dns/main.tf +++ b/install/infra/modules/tools/cloud-dns-external-dns/main.tf @@ -8,9 +8,9 @@ data local_file "gcp_credentials" { provider "google" { credentials = var.credentials - project = var.gcp_project - region = var.gcp_region - zone = var.gcp_zone + project = var.project + region = var.region + zone = var.zone } provider "helm" { @@ -57,7 +57,7 @@ resource "helm_release" "external-dns" { } set { name = "google.project" - value = var.gcp_project + value = var.project } set { name = "logFormat" diff --git a/install/infra/modules/tools/cloud-dns-external-dns/variables.tf b/install/infra/modules/tools/cloud-dns-external-dns/variables.tf index 86046e5029bc2f..ba90701864c37d 100644 --- a/install/infra/modules/tools/cloud-dns-external-dns/variables.tf +++ b/install/infra/modules/tools/cloud-dns-external-dns/variables.tf @@ -3,17 +3,17 @@ variable "kubeconfig" { default = "./kubeconfig" } -variable "gcp_project" { +variable "project" { description = "Google cloud Region to perform operations in" default = "dns-for-playgrounds" } -variable "gcp_region" { +variable "region" { description = "Google cloud Region to perform operations in" default = "europe-west1" } -variable "gcp_zone" { +variable "zone" { description = "Google cloud Zone to perform operations in" default = "europe-west1-d" } diff --git a/install/infra/single-cluster/aws/README.md b/install/infra/single-cluster/aws/README.md index 5136c8b99ebcdb..cbc1a129f5fa8f 100644 --- a/install/infra/single-cluster/aws/README.md +++ b/install/infra/single-cluster/aws/README.md @@ -14,7 +14,6 @@ This module will do the following steps: > ๐Ÿ’ก If you would like to create the infrastructure orchestrating the terraform modules by yourself, you can find all the modules we support [here](../../modules/). - Since the entire setup requires more than one terraform target to be run due to dependencies (eg: helm provider depends on kubernetes cluster config, which is not available until the `eks` module finishes), this directory has a `Makefile` @@ -79,9 +78,6 @@ be used as registry backend. By default `enable_external_storage_for_registry_ba is set to `false`. One can re-use the same `S3` bucket for both object storage and registry backend. The expectation is that you can use the credentials to these setups(provided later -as terraform outputs) during the setup of Gitpod via UI later in the process. -Alternatively, one can choose to use incluster dependencies or separately -created resources of choice. ### AMI Image ID and Kubernetes version diff --git a/install/infra/single-cluster/aws/terraform.tfvars b/install/infra/single-cluster/aws/terraform.tfvars index 12618b02599ce0..6a85059273f28a 100644 --- a/install/infra/single-cluster/aws/terraform.tfvars +++ b/install/infra/single-cluster/aws/terraform.tfvars @@ -1,4 +1,3 @@ - # the cluster_name should be of length less than 16 characters cluster_name = "nan" diff --git a/install/infra/single-cluster/gcp/.terraform.lock.hcl b/install/infra/single-cluster/gcp/.terraform.lock.hcl new file mode 100644 index 00000000000000..a6c73d89e769ba --- /dev/null +++ b/install/infra/single-cluster/gcp/.terraform.lock.hcl @@ -0,0 +1,97 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "4.31.0" + hashes = [ + "h1:+X1KG9mnYGMkYD1c/O+uUi72d4kB3kPSDei4yMEDVAQ=", + "zh:02a19ed46c2007f6aadfb6ff90aa6063be063194d1f0dd02dc839adc212f7cae", + "zh:1046de7e13e81a8f86461f99e9d5ff25d5dabe8465f51efe72084ded426ba771", + "zh:209b054685f7364f3f5e8b570ceb62701e5b466d37cce8b7108385fc1feb3683", + "zh:717773619a1102748204699974c30aba39dc727baf389b874afcab6e17b63ffa", + "zh:7d5f4885cda2ca0ec8cb8bac36ea156aeca7787c01c17e65f7226742b60369d8", + "zh:82df57f2df5708441c57045b3e1a9a91ed55abe67d0d2f00705c7a1f512ec6ec", + "zh:a0191b194e68dd3c0ac5a26712f95d435839ff20d2b2ad53670374c64946042d", + "zh:a95b8358469d6347a5bcf4462ad18efaf80014f07f36bd26019ca039c523ff48", + "zh:b62c968f50d3afa8300c9267388d273a90a5be1a4e9a218205a358e6954e7844", + "zh:bc11cc9b8defec24831bbd6a73a2fa940659c7c610ea7aa0d8b38c2b1af6689b", + "zh:e6ac4c46c3e5a32635fcd27784c189b6cbc6aa9cbf7a3b09e999ec3aa3e2004a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.6.0" + hashes = [ + "h1:rGVucCeYAqklKupwoLVG5VPQTIkUhO7WGcw3WuHYrm8=", + "zh:0ac248c28acc1a4fd11bd26a85e48ab78dd6abf0f7ac842bf1cd7edd05ac6cf8", + "zh:3d32c8deae3740d8c5310136cc11c8afeffc350fbf88afaca0c34a223a5246f5", + "zh:4055a27489733d19ca7fa2dfce14d323fe99ae9dede7d0fea21ee6db0b9ca74b", + "zh:58a8ed39653fd4c874a2ecb128eccfa24c94266a00e349fd7fb13e22ad81f381", + "zh:6c81508044913f25083de132d0ff81d083732aba07c506cc2db05aa0cefcde2c", + "zh:7db5d18093047bfc4fe597f79610c0a281b21db0d61b0bacb3800585e976f814", + "zh:8269207b7422db99e7be80a5352d111966c3dfc7eb98511f11c8ff7b2e813456", + "zh:b1d7ababfb2374e72532308ff442cc906b79256b66b3fe7a98d42c68c4ddf9c5", + "zh:ca63e226cbdc964a5d63ef21189f059ce45c3fa4a5e972204d6916a9177d2b44", + "zh:d205a72d60e8cc362943d66f5bcdd6b6aaaa9aab2b89fd83bf6f1978ac0b1e4c", + "zh:db47dc579a0e68e5bfe3a61f2e950e6e2af82b1f388d1069de014a937962b56a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.12.1" + hashes = [ + "h1:6ZgqegUao9WcfVzYg7taxCQOQldTmMVw0HqjG5S46OY=", + "zh:1ecb2adff52754fb4680c7cfe6143d1d8c264b00bb0c44f07f5583b1c7f978b8", + "zh:1fbd155088cd5818ad5874e4d59ccf1801e4e1961ac0711442b963315f1967ab", + "zh:29e927c7c8f112ee0e8ab70e71b498f2f2ae6f47df1a14e6fd0fdb6f14b57c00", + "zh:42c2f421da6b5b7c997e42aa04ca1457fceb13dd66099a057057a0812b680836", + "zh:522a7bccd5cd7acbb4ec3ef077d47f4888df7e59ff9f3d598b717ad3ee4fe9c9", + "zh:b45d8dc5dcbc5e30ae570d0c2e198505f47d09098dfd5f004871be8262e6ec1e", + "zh:c3ea0943f2050001c7d6a7115b9b990f148b082ebfc4ff3c2ff3463a8affcc4a", + "zh:f111833a64e06659d2e21864de39b7b7dec462615294d02f04c777956742a930", + "zh:f182dba5707b90b0952d5984c23f7a2da3baa62b4d71e78df7759f16cc88d957", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f76655a68680887daceabd947b2f68e2103f5bbec49a2bc29530f82ab8e3bca3", + "zh:fadb77352caa570bd3259dfb59c31db614d55bc96df0ff15a3c0cd2e685678b9", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.2.3" + hashes = [ + "h1:aWp5iSUxBGgPv1UnV5yag9Pb0N+U1I0sZb38AXBFO8A=", + "zh:04f0978bb3e052707b8e82e46780c371ac1c66b689b4a23bbc2f58865ab7d5c0", + "zh:6484f1b3e9e3771eb7cc8e8bab8b35f939a55d550b3f4fb2ab141a24269ee6aa", + "zh:78a56d59a013cb0f7eb1c92815d6eb5cf07f8b5f0ae20b96d049e73db915b238", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8aa9950f4c4db37239bcb62e19910c49e47043f6c8587e5b0396619923657797", + "zh:996beea85f9084a725ff0e6473a4594deb5266727c5f56e9c1c7c62ded6addbb", + "zh:9a7ef7a21f48fabfd145b2e2a4240ca57517ad155017e86a30860d7c0c109de3", + "zh:a63e70ac052aa25120113bcddd50c1f3cfe61f681a93a50cea5595a4b2cc3e1c", + "zh:a6e8d46f94108e049ad85dbed60354236dc0b9b5ec8eabe01c4580280a43d3b8", + "zh:bb112ce7efbfcfa0e65ed97fa245ef348e0fd5bfa5a7e4ab2091a9bd469f0a9e", + "zh:d7bec0da5c094c6955efed100f3fe22fca8866859f87c025be1760feb174d6d9", + "zh:fb9f271b72094d07cef8154cd3d50e9aa818a0ea39130bc193132ad7b23076fd", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.3.2" + hashes = [ + "h1:H5V+7iXol/EHB2+BUMzGlpIiCOdV74H8YjzCxnSAWcg=", + "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", + "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", + "zh:427611a4ce9d856b1c73bea986d841a969e4c2799c8ac7c18798d0cc42b78d32", + "zh:49718d2da653c06a70ba81fd055e2b99dfd52dcb86820a6aeea620df22cd3b30", + "zh:5574828d90b19ab762604c6306337e6cd430e65868e13ef6ddb4e25ddb9ad4c0", + "zh:7222e16f7833199dabf1bc5401c56d708ec052b2a5870988bc89ff85b68a5388", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b1b2d7d934784d2aee98b0f8f07a8ccfc0410de63493ae2bf2222c165becf938", + "zh:b8f85b6a20bd264fcd0814866f415f0a368d1123cd7879c8ebbf905d370babc8", + "zh:c3813133acc02bbebddf046d9942e8ba5c35fc99191e3eb057957dafc2929912", + "zh:e7a41dbc919d1de800689a81c240c27eec6b9395564630764ebb323ea82ac8a9", + "zh:ee6d23208449a8eaa6c4f203e33f5176fa795b4b9ecf32903dffe6e2574732c2", + ] +} diff --git a/install/infra/single-cluster/gcp/Makefile b/install/infra/single-cluster/gcp/Makefile new file mode 100644 index 00000000000000..0bd6ad1287c634 --- /dev/null +++ b/install/infra/single-cluster/gcp/Makefile @@ -0,0 +1,121 @@ +## +# Terraform AWS reference architecture +# + +.PHONY: init +init: + @terraform init + +touch-kubeconfig: + @touch kubeconfig + +cleanup-kubeconfig: + @rm kubeconfig + +.PHONY: plan +plan: touch-kubeconfig plan-cluster plan-cm-edns cleanup-kubeconfig + +.PHONY: apply +apply: apply-cluster apply-tools + +.PHONY: destroy +destroy: destroy-tools destroy-cluster + +.PHONY: refresh +refresh: + @echo "Refreshing terraform state" + @terraform refresh + @echo "" + @echo "Done!" + +.PHONY: output +output: refresh output-done-msg output-url output-nameservers output-registry output-database output-storage output-issuer + +output-done-msg: + @echo "" + @echo "" + @echo "==========================" + @echo "๐ŸŽ‰๐Ÿฅณ๐Ÿ”ฅ๐Ÿงก๐Ÿš€" + @echo "Your cloud infrastructure is ready to install Gitpod. Please visit" + @echo "https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod" + @echo "for your next steps." + @echo "=================" + @echo "Config Parameters" + @echo "=================" + +output-url: + @echo "\nGitpod domain name:" + @echo "=================" + @terraform output -json url | jq + +output-nameservers: + @echo "\nNameservers for the domain(to be added as NS records in your domain provider):" + @echo "=================" + @terraform output -json nameservers | jq + +output-storage: + @echo "\nObject storage:" + @echo "==============" + @terraform output -json storage | jq + +output-registry: + @echo "\nGCR registry:" + @echo "==================" + @terraform output -json registry | jq + +output-database: + @echo "\nDatabase:" + @echo "========" + @terraform output -json database | jq + +output-issuer: + @echo "\nClusterIssuer name:" + @echo "=================" + @terraform output -json cluster_issuer | jq + +.PHONY: plan-cluster +plan-cluster: + @terraform plan -target=module.gke + +.PHONY: plan-tools +plan-tools: plan-cm-edns plan-cluster-issuer + +.PHONY: plan-cm-edns +plan-cm-edns: + @terraform plan -target=module.certmanager -target=module.externaldns + +.PHONY: plan-cluster-issuer +plan-cluster-issuer: + @terraform plan -target=module.cluster-issuer + +.PHONY: apply-cluster +apply-cluster: + @terraform apply -target=module.gke --auto-approve + +.PHONY: apply-tools +apply-tools: install-cm-edns install-cluster-issuer + +.PHONY: install-cm-edns +install-cm-edns: + @terraform apply -target=module.certmanager -target=module.externaldns --auto-approve + +.PHONY: install-cluster-issuer +install-cluster-issuer: + @terraform apply -target=module.cluster-issuer --auto-approve + +.PHONY: destroy-cluster +destroy-cluster: + @terraform destroy -target=module.gke --auto-approve + +.PHONY: destroy-tools +destroy-tools: destroy-cluster-issuer destroy-cm-edns + +.PHONY: destroy-cm-edns +destroy-cm-edns: + @terraform destroy -target=module.certmanager -target=module.externaldns --auto-approve + +.PHONY: destroy-cluster-issuer +destroy-cluster-issuer: + @terraform destroy -target=module.cluster-issuer --auto-approve + +# end diff --git a/install/infra/single-cluster/gcp/README.md b/install/infra/single-cluster/gcp/README.md new file mode 100644 index 00000000000000..665cca58959682 --- /dev/null +++ b/install/infra/single-cluster/gcp/README.md @@ -0,0 +1,238 @@ +# Terraform setup for GCP Single-cluster Gitpod reference architecture + +This directory has terraform configuration necessary to achieve a infrastructure +corresponding to the [Single-cluster reference architecture for Gitpod on GCP](https://www.gitpod.io/docs/self-hosted/latest/reference-architecture/single-cluster-ref-arch). + +This module will do the following steps: +- Creates the infrastructure using our [`gke` terraform module](../../modules/gke/), which does: + - Setup an GKE managed cluster, along with external dependencies like database, storage and registry (if chosen) + - Sets up `CloudDNS` entries for the domain name (if provided) +- Provisioning the cluster: + - Set up cert-manager using our [`cert-manager` module](../../modules/tools/cert-manager/) + - Set up external-dns using our [`external-dns` module](../../modules/tools/external-dns/) + - Creates a cluster-issuer using our [`issuer` module](../../modules/tools/issuer/) + +> ๐Ÿ’ก If you would like to create the infrastructure orchestrating the terraform modules by yourself, you can find all the modules we support [here](../../modules/). + + +Since the entire setup requires more than one terraform target to be run due to +dependencies (eg: helm provider depends on kubernetes cluster config, which is +not available until the `gke` module finishes), this directory has a `Makefile` +with targets binding together targeted terraform calls. This document will +explain the execution of the terraform module in terms of these `make` targets. + +## Requirements + +* `terraform` >= `v1.1.0` +* `kubectl` >= `v1.20.0` +* [`jq`](https://stedolan.github.io/jq/download/) +* [`gcloud`](https://cloud.google.com/sdk/docs/install) + +## Setup GCP authentication and GCS backend storage + +Before starting the installation process, you need: + +* A GCP account with administative access + * [Create one now by clicking here](https://console.cloud.google.com/freetrial) +* Store the JSON credentials corresponding to the service account locally in a file +* Create and configure GCS bucket for terraform backend + * Create a [GCS bucket](https://cloud.google.com/storage) to store the terraform backend state + * Replace the name of the bucket in [`main.tf`](./main.tf) - currently it is set as `gitpod-tf` + +## Update the `terraform.tfvars` file with appropriate values + +The file [`terraform.tfvars`](./terraform.tfvars) with values that will be used +by terraform to create the cluster. While some of them are fairly +straightforward like the name of the cluster(`cluster_name`), others need a bit +more attention: + +### Credentials and project configuration + +To configure against a standing GCP account, we expect the the key corresponding +to the service account stored as a JSON file. The path to the JSON file is +expected to be provided as a value to the `credentials` field. Alongside, one is +expected to provide the name of the project(`project` field) corresponding to +this service account and region in with the cluster to be created(`region` +field). If you want your cluster to be zonal(only existing in one zone), you can +provide a zone corresponding to the project(`zone` field), else the cluster will +be regional. + +### External database, storage and registry + +If you wish to create cloud specific database, storage and registry backend to be used +with `Gitpod`, leave the following 3 booleans set: + +``` sh +enable_external_database = true +enable_external_storage = true +enable_external_registry = true +``` + +The corresponding resources will be created by the terraform script which +creates an `CloudSQL` mysql database, and access credentials to the GCS storage and GCR registry. + +The expectation is that you can use the credentials to these setups(provided later +as terraform outputs) during the setup of Gitpod via UI later in the process. +Alternatively, one can choose to use incluster dependencies or separately +created resources of choice. + +### Kubernetes version + +Make sure you provide the corresponding kubernetes version as a value to the +variable `cluster_version`. We officially support kubernetes versions >= `1.20`. + +### Domain name configuration + +If you do not yet have a DNS zone created for Gitpod or plan on using cert-manager +to generate TLS certificates for gitpod, we strongly recommend setting `domain_name` +to a domain for use with Gitpod. This will save a lot +of hassle in setting up `Cloud DNS` records to point to the cluster and +corresponding TLS certificate requests. + +## Initialize terraform backend and confirm the plan + +* Initialize the terraform backend with: + + ``` sh + make init + ``` + +* Get the plan of the execution to verify the resources that are going to get + created: + + ```sh + make plan + ``` + +## Apply terraform setup + +If the plan looks good, now you can go ahead and create the resources: + +``` sh +make apply +``` + +The `apply` target calls the `terraform` apply on `eks` module, `cert-manager` +module, `external-dns` module and `cluster-issuer` module in that exact order. +The entire operation would take around *30 minutes* to complete. + +Upon completion, you will find a creation `kubeconfig` file in the local +directory. You can try accessing the cluster using this file as follows: + +``` sh +export KUBECONFIG=/path/to/kubeconfig +kubectl get pods -A +``` + +You can list all the other outputs with the following command: + +``` sh +make output +``` + +> ๐Ÿ’ก Alternatively, you can get the simple JSON output with a `terraform output` command + +## Note the NS records from terraform output + +Once the apply process has exited successfully, we can go ahead and prepare to +setup Gitpod. If you specified the `domain_name` in the `terraform.tfvars` file, +the terraform module registers the module with `cloudDNS` to point to the +cluster. Now you have to configure whichever provider you use to host your +domain name to route traffic to the GCP name servers. You can find these name +servers in the `make output` command from above. It would be of the format: + +```json +Nameservers for the domain(to be added as NS records in your domain provider): +================= +[ + "ns-cloud-c1.googledomains.com.", + "ns-cloud-c2.googledomains.com.", + "ns-cloud-c3.googledomains.com.", + "ns-cloud-c4.googledomains.com." +] + + +``` + +Add the `ns` records similar to the above 4 URIs as NS records under your domain +DNS management setup. Check with your domain hosting service for specific information. + +## Note the dependency credentials from terraform output + +If you enabled the creation of external database, storage and registry, the +above `make output` command would list the credentials to connect to the +corresponding resource. Make a note of this, so as to provide the same when +setting up Gitpod via KOTS UI. + +## Install Gitpod + +You can install `KOTS` CLI to install Gitpod: + +``` sh +curl https://kots.io/install | bash +``` + +Run the following to get started with Gitpod installation: + +``` sh +export KUBECONFIG=kubeconfig +kubectl kots install gitpod/stable # you can replace `stable` with `unstable` or `beta` as per the requirement +``` + +Upon completion, you can access `KOTS` UI in `localhost:8800`. Here you can +proceed to configuring and intalling Gitpod. Please follow the [official +documentaion](https://www.gitpod.io/docs/self-hosted/latest/getting-started#step-4-install-gitpod) to complete the Gitpod setup. + +## Troubleshooting + +### Some pods never start (Init state) + +```sh +kubectl get pods -l component=proxy +NAME READY STATUS RESTARTS AGE +proxy-5998488f4c-t8vkh 0/1 Init 0/1 0 5m +``` + +The most likely reason is that the DNS01 challenge cannot be completed, typically because DNS zone delegation hasn't been set up from the parent domain to the subdomain that Gitpod is managing (specified by the `domain_name` variable). To fix this, make sure that NS records for `domain_name` in the parent zone are created and point to the nameservers of the Gitpod managed zone. See the [Google Cloud DNS documentation](https://cloud.google.com/dns/docs/dns-overview#delegated_subzone) for more information on zone delegation. + +Once the DNS record has been updated, you will need to delete all Cert Manager pods to retrigger the certificate request + +``` +kubectl delete pods -n cert-manager --all +``` + +After a few minutes, you should see the https-certificate become ready. + +``` +kubectl get certificate +NAME READY SECRET AGE +https-certificates True https-certificates 5m +``` + +### Cannot connect to the created cluster after a while + +There is a chance that your kubeconfig has gotten expired after a specific amount of time. You can reconnect to the cluster by using: + +``` sh +# make sure you are authenticated using the service account you used to create the cluster +gcloud auth activate-service-account --key-file=/path/to/account/key.json +gcloud container clusters get-credentials --region --zone --project +``` + +## Cleanup + +Make sure you first delete the `gitpod` resources in the cluster so things like load balancer created by the k8s `service` gets deleted. Otherwise terraform will not be able to delete the VPC. + +```sh +kubectl delete --now namespace gitpod +``` + +> It is okay to ignore any dangling workspaces that are not deleted + +Now run the terraform destroy step to cleanup all the cloud resources: + +```sh +make destroy +``` + +The destroy should take around 20 minutes. diff --git a/install/infra/single-cluster/gcp/cluster.tf b/install/infra/single-cluster/gcp/cluster.tf new file mode 100644 index 00000000000000..136cc795198dd5 --- /dev/null +++ b/install/infra/single-cluster/gcp/cluster.tf @@ -0,0 +1,16 @@ +module "gke" { + source = "../../modules/gke" + + credentials = var.credentials + cluster_name = var.cluster_name + kubeconfig = var.kubeconfig + cluster_version = var.cluster_version + project = var.project + region = var.region + zone = var.zone + + domain_name = var.domain_name + enable_external_database = var.enable_external_database + enable_external_storage = var.enable_external_storage + enable_external_registry = var.enable_external_registry +} diff --git a/install/infra/single-cluster/gcp/main.tf b/install/infra/single-cluster/gcp/main.tf new file mode 100644 index 00000000000000..4cfce62d2aacfb --- /dev/null +++ b/install/infra/single-cluster/gcp/main.tf @@ -0,0 +1,20 @@ +terraform { + backend "gcs" { + bucket = "gitpod-tf" + prefix = "gcp/terraform.state" + } + + required_providers { + google = { + source = "hashicorp/google" + } + + kubernetes = { + source = "hashicorp/kubernetes" + } + + helm = { + source = "hashicorp/helm" + } + } +} diff --git a/install/infra/single-cluster/gcp/output.tf b/install/infra/single-cluster/gcp/output.tf new file mode 100644 index 00000000000000..d1a04ee3cee52b --- /dev/null +++ b/install/infra/single-cluster/gcp/output.tf @@ -0,0 +1,36 @@ +output "url" { + value = var.domain_name +} + +output "cluster_name" { + value = var.cluster_name +} + +output "region" { + value = var.region +} + +output "nameservers" { + sensitive = false + value = module.gke.name_servers +} + +output "database" { + sensitive = true + value = module.gke.database +} + +output "cluster_issuer" { + sensitive = false + value = module.cluster-issuer.cluster_issuer +} + +output "registry" { + sensitive = true + value = module.gke.registry +} + +output "storage" { + sensitive = true + value = module.gke.storage +} diff --git a/install/infra/single-cluster/gcp/terraform.tfvars b/install/infra/single-cluster/gcp/terraform.tfvars new file mode 100644 index 00000000000000..71856db8419736 --- /dev/null +++ b/install/infra/single-cluster/gcp/terraform.tfvars @@ -0,0 +1,16 @@ +# the cluster_name should be of length less than 16 characters +cluster_name = "nan-cluster" + +# a cloudDNS zone and certificate request will be created for this domain +domain_name = "nan-cluster.gitpod-self-hosted.com" + +region = "europe-west1" +zone = "europe-west1-d" +project = "my-gcp-project" +credentials = "/path/to/account/key.json" + +cluster_version = "1.22" + +enable_external_database = true +enable_external_storage = true +enable_external_registry = true diff --git a/install/infra/single-cluster/gcp/tools.tf b/install/infra/single-cluster/gcp/tools.tf new file mode 100644 index 00000000000000..bcfa6b1cd8ccb2 --- /dev/null +++ b/install/infra/single-cluster/gcp/tools.tf @@ -0,0 +1,28 @@ +module "certmanager" { + source = "../../modules/tools/cert-manager" + + kubeconfig = var.kubeconfig + credentials = var.credentials +} + +module "externaldns" { + source = "../../modules/tools/cloud-dns-external-dns" + kubeconfig = var.kubeconfig + credentials = var.credentials + project = var.project + region = var.region + zone = var.zone +} + +module "cluster-issuer" { + source = "../../modules/tools/issuer" + kubeconfig = var.kubeconfig + issuer_name = "cloudDNS" + cert_manager_issuer = { + project = var.project + serviceAccountSecretRef = { + name = "clouddns-dns01-solver" + key = "keys.json" + } + } +} diff --git a/install/infra/single-cluster/gcp/variables.tf b/install/infra/single-cluster/gcp/variables.tf new file mode 100644 index 00000000000000..fa0e0b981fb817 --- /dev/null +++ b/install/infra/single-cluster/gcp/variables.tf @@ -0,0 +1,54 @@ +variable "credentials" { + description = "Path to JSON file authenticating to GCP service account" +} + +variable "project" { + description = "GCP project to create the resources in" +} + +variable "cluster_name" { + type = string + description = "GKE cluster name." +} +variable "kubeconfig" { + type = string + description = "Path to the kubeconfig file to write the KUBECONFIG output in" + default = "kubeconfig" +} + +variable "region" { + type = string + description = "Region to create the resources in" + default = "europe-west1" +} + +variable "zone" { + type = string + description = "Zones under the provided region, if left empty a regional cluster will be created" + default = "europe-west1-b" +} + +variable "domain_name" { + description = "Domain name to associate with the Gitpod installation" +} + +variable "cluster_version" { + type = string + description = "Kubernetes version to create the cluster with" + default = "1.22" +} + +variable "enable_external_database" { + default = true + description = "Create an external Cloud DNS instance for Gitpod" +} + +variable "enable_external_storage" { + default = true + description = "Create a GCS bucket for Gitpod object storage" +} + +variable "enable_external_registry" { + default = true + description = "Create a Google Container Registry for Gitpod workspace images" +} diff --git a/install/tests/Makefile b/install/tests/Makefile index dadec305911a42..869e93327426f1 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -46,7 +46,7 @@ k3s-kubeconfig: sync-kubeconfig gcp-kubeconfig: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests export KUBECONFIG=${KUBECONFIG} && \ - gcloud container clusters get-credentials gitpod-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present" + gcloud container clusters get-credentials gp-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present" ## azure-kubeconfig: Get the kubeconfig configuration for Azure AKS azure-kubeconfig: diff --git a/install/tests/main.tf b/install/tests/main.tf index 3333b2f967d3c4..9457142c698e01 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -29,7 +29,7 @@ module "gke" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=main" # we can later use tags here source = "../infra/modules/gke" # we can later use tags here - name = var.TEST_ID + cluster_name = "gp-${var.TEST_ID}" project = var.project credentials = var.sa_creds kubeconfig = var.kubeconfig From c3e6b78cc3c2171ccbb7f14bd47783e6a04ff2ed Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Tue, 26 Jul 2022 13:08:44 +0000 Subject: [PATCH 9/9] Add terraform configurations for single-cluster ref arch --- install/infra/single-cluster/aws/Makefile | 92 ++++++++++--------- install/infra/single-cluster/aws/README.md | 3 + install/infra/single-cluster/aws/main.tf | 2 +- .../infra/single-cluster/aws/terraform.tfvars | 8 +- install/tests/main.tf | 3 + 5 files changed, 58 insertions(+), 50 deletions(-) diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile index abf80fcce98ece..edab2a20de3d36 100644 --- a/install/infra/single-cluster/aws/Makefile +++ b/install/infra/single-cluster/aws/Makefile @@ -21,6 +21,53 @@ apply: apply-cluster apply-tools .PHONY: destroy destroy: destroy-tools destroy-cluster +.PHONY: plan-cluster +plan-cluster: + @terraform plan -target=module.eks + +.PHONY: plan-tools +plan-tools: plan-cm-edns plan-cluster-issuer + +.PHONY: plan-cm-edns +plan-cm-edns: + @terraform plan -target=module.certmanager -target=module.externaldns + +.PHONY: plan-cluster-issuer +plan-cluster-issuer: + @terraform plan -target=module.cluster-issuer + +.PHONY: apply-cluster +apply-cluster: + @terraform apply -target=module.eks --auto-approve + +.PHONY: apply-tools +apply-tools: install-cm-edns install-cluster-issuer + +.PHONY: install-cm-edns +install-cm-edns: + @terraform apply -target=module.certmanager -target=module.externaldns --auto-approve + +PHONY: install-cluster-issuer +install-cluster-issuer: + @terraform apply -target=module.cluster-issuer --auto-approve + +.PHONY: destroy-cluster +destroy-cluster: + @terraform destroy -target=module.eks --auto-approve + +.PHONY: destroy-tools +destroy-tools: destroy-cluster-issuer destroy-cm-edns + +.PHONY: destroy-cm-edns +destroy-cm-edns: + @terraform destroy -target=module.certmanager -target=module.externaldns --auto-approve + +.PHONY: destroy-cluster-issuer +destroy-cluster-issuer: + @terraform destroy -target=module.cluster-issuer --auto-approve + +## Output targets + .PHONY: refresh refresh: @echo "Refreshing terraform state" @@ -73,49 +120,4 @@ output-issuer: @echo "=================" @terraform output -json cluster_issuer | jq -.PHONY: plan-cluster -plan-cluster: - @terraform plan -target=module.eks - -.PHONY: plan-tools -plan-tools: plan-cm-edns plan-cluster-issuer - -.PHONY: plan-cm-edns -plan-cm-edns: - @terraform plan -target=module.certmanager -target=module.externaldns - -.PHONY: plan-cluster-issuer -plan-cluster-issuer: - @terraform plan -target=module.cluster-issuer - -.PHONY: apply-cluster -apply-cluster: - @terraform apply -target=module.eks --auto-approve - -.PHONY: apply-tools -apply-tools: install-cm-edns install-cluster-issuer - -.PHONY: install-cm-edns -install-cm-edns: - @terraform apply -target=module.certmanager -target=module.externaldns --auto-approve - -PHONY: install-cluster-issuer -install-cluster-issuer: - @terraform apply -target=module.cluster-issuer --auto-approve - -.PHONY: destroy-cluster -destroy-cluster: - @terraform destroy -target=module.eks --auto-approve - -.PHONY: destroy-tools -destroy-tools: destroy-cluster-issuer destroy-cm-edns - -.PHONY: destroy-cm-edns -destroy-cm-edns: - @terraform destroy -target=module.certmanager -target=module.externaldns --auto-approve - -.PHONY: destroy-cluster-issuer -destroy-cluster-issuer: - @terraform destroy -target=module.cluster-issuer --auto-approve - # end diff --git a/install/infra/single-cluster/aws/README.md b/install/infra/single-cluster/aws/README.md index cbc1a129f5fa8f..af66be92332259 100644 --- a/install/infra/single-cluster/aws/README.md +++ b/install/infra/single-cluster/aws/README.md @@ -97,6 +97,9 @@ corresponding TLS certificate requests. ## Initialize terraform backend and confirm the plan +> โš ๏ธ We ship 4 terraform modules here and some of them have dependencies among each other (eg: `cert-manager` module depends on `eks` module for `kubeconfig`, or the `cluster-issuer` module depends on `cert-manager` for the CRD). Hence a simple run of `terraform plan` or `terraform apply` may lead to errors. Hence we wrap [targeted `terraform` operations](https://learn.hashicorp.com/tutorials/terraform/resource-targeting) in the following make targets. If you wish you use `terraform` commands instead, please make sure you look into the Makefile to understand the target order. + + * Initialize the terraform backend with: ``` sh diff --git a/install/infra/single-cluster/aws/main.tf b/install/infra/single-cluster/aws/main.tf index 2cd2e408fcf459..9ae7aee2fbf265 100644 --- a/install/infra/single-cluster/aws/main.tf +++ b/install/infra/single-cluster/aws/main.tf @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "nan-tf-bucket" + bucket = "gitpod-tf" key = "aws/terraform.state" } } diff --git a/install/infra/single-cluster/aws/terraform.tfvars b/install/infra/single-cluster/aws/terraform.tfvars index 6a85059273f28a..0712a2b6febe6a 100644 --- a/install/infra/single-cluster/aws/terraform.tfvars +++ b/install/infra/single-cluster/aws/terraform.tfvars @@ -1,8 +1,8 @@ -# the cluster_name should be of length less than 16 characters -cluster_name = "nan" +# the cluster_name should be of length less than 15 characters and surrounded by double quotes +cluster_name = -# a route53 zone and certificate request will be created for this domain -domain_name = "nan-cluster.doptig.com" +# a route53 zone and certificate request will be created for this domain and surrounded by double quotes +domain_name = region = "eu-west-1" diff --git a/install/tests/main.tf b/install/tests/main.tf index 9457142c698e01..de06587001d7d0 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -92,6 +92,9 @@ module "eks" { image_id = var.eks_node_image_id kubeconfig = var.kubeconfig cluster_version = var.cluster_version + create_external_registry = true + create_external_database = true + create_external_storage = true } module "certmanager" {