removing one level of indirection

nandajavarma · nandajavarma · commit dc4aae799ffd · 2022-09-22T20:35:22.000Z
diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf
@@ -179,6 +179,8 @@ module "eks" {
         "k8s.io/cluster-autoscaler/gitpod"  = "owned"
       }
 
+
+
       pre_bootstrap_user_data = <<-EOT
         #!/bin/bash
         set -ex
@@ -219,3 +221,35 @@ resource "null_resource" "kubeconfig" {
     create_before_destroy = true
   }
 }
+
+data "aws_iam_policy_document" "eks_policy" {
+  statement {
+    actions   = [
+      "eks:DescribeCluster",
+      "eks:ListClusters"
+    ]
+    resources = [
+      "*",
+    ]
+    effect    = "Allow"
+  }
+}
+
+resource "aws_iam_policy" "eks_policy" {
+  name        = "eks-policy-${var.cluster_name}"
+  description = "Gitpod ${var.cluster_name} EKS cluster access bucket policy"
+  policy      = data.aws_iam_policy_document.eks_policy.json
+}
+
+resource "aws_iam_user" "eks_user" {
+  name  = "eks-user-${var.cluster_name}"
+}
+
+resource "aws_iam_user_policy_attachment" "eks_attachment" {
+  user       = aws_iam_user.eks_user.name
+  policy_arn = aws_iam_policy.eks_policy.arn
+}
+
+resource "aws_iam_access_key" "eks_user_key" {
+  user  = aws_iam_user.eks_user.name
+}
diff --git a/install/infra/modules/eks/output.tf b/install/infra/modules/eks/output.tf
@@ -91,3 +91,13 @@ output "registry_backend" {
     secret_access_key = aws_iam_access_key.bucket_registry_user[0].secret
   }, "No s3 bucket created for registry backend.")
 }
+
+output "cluster_user" {
+  sensitive = true
+  value = {
+    userarn           = aws_iam_user.eks_user.arn
+    name              = aws_iam_user.eks_user.name
+    access_key_id     = aws_iam_access_key.eks_user_key.id
+    secret_access_key = aws_iam_access_key.eks_user_key.secret
+  }
+}
diff --git a/install/infra/modules/gke/cluster.tf b/install/infra/modules/gke/cluster.tf
@@ -158,7 +158,7 @@ resource "google_service_account" "cluster_user_sa" {
 resource "google_project_iam_member" "gke-user-sa-iam" {
   project = var.project
   role    = "roles/container.developer"
-  member  = "serviceAccount:${google_service_account.cluster_sa.email}"
+  member  = "serviceAccount:${google_service_account.cluster_user_sa.email}"
 }
 
 resource "google_service_account_key" "gke_sa_key" {
diff --git a/install/infra/modules/gke/database.tf b/install/infra/modules/gke/database.tf
@@ -41,8 +41,7 @@ resource "random_password" "password" {
   count = var.enable_external_database ? 1 : 0
 
   length           = 16
-  special          = true
-  override_special = "!#$%&*()-_=+[]{}<>:?"
+  special          = false
 }
 
 resource "google_sql_database" "database" {
diff --git a/install/tests/Makefile b/install/tests/Makefile
@@ -35,53 +35,49 @@ help: Makefile
 
 upload-gcp-cluster-creds:
 	export GKE_CREDS=$$(terraform output -json gke_user_key) && \
-	echo ${GKE_CREDS} > gcp-creds
+	echo $$GKE_CREDS > gcp-creds
 	gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests
-	gsutil cp gcp-creds gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-gcp-creds
+	gsutil cp gcp-creds gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds
 
-key_file ?= ${GOOGLE_APPLICATION_CREDENTIALS}
-download-gcp-cluster-creds:
-	gcloud auth activate-service-account --key-file=${key_file} --project=sh-automated-tests
-	gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-gcp-creds gs-creds && cat gs-creds | base64 -d > ${TF_VAR_TEST_ID}-key.json || echo "No GCP credentials"
-
-upload-aws-cluster-creds:
-
-upload-azure-cluster-creds:
+download-cluster-creds:
+	gcloud config set project sh-automated-tests
+	[[ -n $$TF_VAR_sa_creds ]] || gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds gcs-creds && cat gcs-creds | tr -d '"' | base64 -d > ${TF_VAR_TEST_ID}-key.json || echo "No GCP credentials"
+	rm -f gcs-creds
+	[[ -f ${TF_VAR_TEST_ID}-key.json ]] || cp ${GOOGLE_APPLICATION_CREDENTIALS} ${TF_VAR_TEST_ID}-key.json
 
 upload-kubeconfig-to-gcp:
 	gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests
 	gsutil cp ${KUBECONFIG} gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig
 
-key_file ?= ${GOOGLE_APPLICATION_CREDENTIALS}
 sync-kubeconfig:
-	gcloud auth activate-service-account --key-file=key_file --project=sh-automated-tests
+	gcloud config set project sh-automated-tests
 	gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig ${KUBECONFIG} || echo "No kubeconfig"
 
 ## k3s-kubeconfig: Get the kubeconfig configuration for GCP K3s
 k3s-kubeconfig: sync-kubeconfig
 
 ## gcp-kubeconfig: Get the kubeconfig configuration for GCP GKE
 gcp-kubeconfig:
-	kubectl get secret -n sh-gcs-secret -o -o jsonpath='{.data.key}' | base64 -d > gcs-key.json || echo "No access to core-dev cluster"
-	[[ -f gcs-key.json ]] || echo ${GOOGLE_APPLICATION_CREDENTIALS} > gcs-key.json
-	$(MAKE) download-gcp-cluster-creds key_file=${key_file}
-	[[ -f ${TF_VAR_TEST_ID}-key.json ]] &&  gcloud auth activate-service-account --key-file=${TF_VAR_TEST_ID}-key.json --project=sh-automated-tests
+	$(MAKE) download-cluster-creds
+	gcloud auth activate-service-account --key-file=${TF_VAR_TEST_ID}-key.json --project=sh-automated-tests || { echo "Count not authenicate the service account"; exit 1; }
 	export KUBECONFIG=${KUBECONFIG} && \
-	gcloud container clusters get-credentials gp-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || $(MAKE) sync-kubeconfig || echo "No cluster present"
+	gcloud container clusters get-credentials gp-${TF_VAR_TEST_ID} --zone europe-west1-d --project sh-automated-tests || echo "No cluster present"
 	rm -f ${TF_VAR_TEST_ID}-key.json
 
 ## azure-kubeconfig: Get the kubeconfig configuration for Azure AKS
 azure-kubeconfig:
-	az login --service-principal -u $$ARM_CLIENT_ID -p $$ARM_CLIENT_SECRET  --tenant $$ARM_TENANT_ID
+	[[ -n "$$ARM_CLIENT_SECRET" ]] && az login --service-principal -u $$ARM_CLIENT_ID -p $$ARM_CLIENT_SECRET  --tenant $$ARM_TENANT_ID || { echo "Please login to azure using az login command"; exit 1; }
 	export KUBECONFIG=${KUBECONFIG} && \
 	az aks get-credentials --name p$$TF_VAR_TEST_ID-cluster --resource-group p$$TF_VAR_TEST_ID --file ${KUBECONFIG} || echo "No cluster present"
 
 ## aws-kubeconfig: Get the kubeconfig configuration for AWS EKS
 aws-kubeconfig:
-	export KUBECONFIG=${KUBECONFIG} && \
+	gcloud config set project sh-automated-tests
+	[[ -n $$TF_VAR_sa_creds ]] || gsutil cp gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds ${TF_VAR_TEST_ID}-creds
+	[[ -f ${TF_VAR_TEST_ID}-creds ]] || touch ${TF_VAR_TEST_ID}-creds
+	source ${TF_VAR_TEST_ID}-creds; \
 	aws eks update-kubeconfig --name ${TF_VAR_TEST_ID} --region eu-west-1 --kubeconfig ${KUBECONFIG} || echo "No cluster present"
 
-
 .PHONY:
 ## gke-standard-cluster: Creates a zonal GKE cluster
 gke-standard-cluster: check-env-cluster-version
@@ -90,8 +86,22 @@ gke-standard-cluster: check-env-cluster-version
 	rm -f ${KUBECONFIG} && \
 	$(MAKE) get-kubeconfig && \
 	[[ -f ${KUBECONFIG} ]] || terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
+	$(MAKE) upload-gcp-cluster-creds
 	@echo "Done creating GKE cluster"
 
+upload-eks-user:
+	export AWS_CLUSTER_USER=$$(terraform output -json aws_cluster_user) && \
+	export USERARN=$$(echo $$AWS_CLUSTER_USER | yq r - 'userarn') && \
+	export NAME=$$(echo $$AWS_CLUSTER_USER | yq r - 'name') && \
+	envsubst < ./manifests/aws-auth.yaml > tmp-aws-auth.yaml && \
+	echo "export AWS_SECRET_ACCESS_KEY=$$(echo $$AWS_CLUSTER_USER | yq r - 'secret_access_key')" > ${TF_VAR_TEST_ID}-creds && \
+	echo "export AWS_ACCESS_KEY_ID=$$(echo $$AWS_CLUSTER_USER | yq r - 'access_key_id')" >> ${TF_VAR_TEST_ID}-creds && \
+	kubectl --kubeconfig=${KUBECONFIG} get configmap -n kube-system aws-auth -o yaml | grep -v "creationTimestamp\|resourceVersion\|selfLink\|uid" | sed '/^  annotations:/,+2 d' > /tmp/aws-auth.yaml
+	yq m --inplace /tmp/aws-auth.yaml tmp-aws-auth.yaml
+	gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests
+	gsutil cp ${TF_VAR_TEST_ID}-creds gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds
+	kubectl --kubeconfig=${KUBECONFIG} replace -f /tmp/aws-auth.yaml
+
 ami_id_121 := "ami-060637af2651bc8bb"
 
 ami_id_122 := "ami-0733d755ed2c97a4d"
@@ -107,6 +117,7 @@ eks-standard-cluster: check-env-cluster-version
 	rm -f ${KUBECONFIG} && \
 	$(MAKE) get-kubeconfig && \
 	[[ -f ${KUBECONFIG} ]] || terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} -var eks_node_image_id=${ami_id} --auto-approve
+	$(MAKE) upload-eks-user
 	@echo "Done creating EKS cluster"
 
 .PHONY:
@@ -156,7 +167,6 @@ k3s-standard-cluster: check-env-cluster-version
 	$(MAKE) get-kubeconfig && \
 	[[ -f ${KUBECONFIG} ]] || terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} -var k3s_node_image_id=${image_id} --auto-approve && \
 	$(MAKE) upload-kubeconfig-to-gcp # we upload the file to GCP since we cannot retrieve the file against without SSHing to the master
-	$(MAKE) upload-gcp-cluster-creds
 	@echo "Done creating k3s cluster"
 
 .PHONY:
@@ -180,8 +190,10 @@ external-dns: check-env-cloud select-workspace
 
 .PHONY:
 ## get-kubeconfig: Returns KUBECONFIG of a just created cluster
-get-kubeconfig: ${cloud}-kubeconfig
-
+get-kubeconfig:
+	echo "Getting kubeconfig for $$TF_VAR_TEST_ID terraform state" && \
+    export provider=$$(echo "$$TF_VAR_TEST_ID" | sed 's/\(.*\)-/\1 /' | xargs | awk '{print $$2}') && \
+	$(MAKE) $$provider-kubeconfig && echo "kubeconfig written to ${KUBECONFIG}"
 
 get-github-config:
 ifneq ($(GITHUB_SCM_OAUTH),)
@@ -233,8 +245,8 @@ registry-config-azure:
 	yq m -i tmp_config.yml tmp_2_config.yml
 
 storage-config-azure:
-	export PASSWORD=$$(terraform output -json azure_storage | yq r - 'account_name') && \
-	export USERNAME=$$(terraform output -json azure_storage | yq r - 'account_key') && \
+	export USERNAME=$$(terraform output -json azure_storage | yq r - 'account_name') && \
+	export PASSWORD=$$(terraform output -json azure_storage | yq r - 'account_key') && \
 	export REGION=$$(terraform output -json azure_storage | yq r - 'storage_region') && \
 	envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml
 	yq m -i tmp_config.yml tmp_2_config.yml
@@ -409,7 +421,7 @@ kots-upgrade:
 	kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy
 
 cloud ?= cluster
-cleanup: $(cloud)-kubeconfig destroy-gitpod tf-init destroy-$(cloud) destroy-workspace destroy-kubeconfig
+cleanup: get-kubeconfig destroy-gitpod tf-init destroy-$(cloud) destroy-workspace destroy-kubeconfig
 
 cluster-kubeconfig: azure-kubeconfig aws-kubeconfig k3s-kubeconfig gcp-kubeconfig
 
@@ -421,6 +433,7 @@ destroy-cluster: destroy-gcp destroy-aws destroy-azure
 destroy-kubeconfig:
 	gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} --project=sh-automated-tests
 	gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-kubeconfig || echo "No kubeconfig"
+	gsutil rm gs://nightly-tests/tf-state/${TF_VAR_TEST_ID}-creds || echo "No credentials file"
 	rm ${KUBECONFIG} || echo "No kubeconfig"
 
 select-workspace:
diff --git a/install/tests/cleanup.sh b/install/tests/cleanup.sh
@@ -19,6 +19,7 @@ for i in $(gsutil ls gs://nightly-tests/tf-state); do
     [ -z "$filename" ] && continue
 
     if [[ "$filename" == *-kubeconfig ]]; then continue; fi
+    if [[ "$filename" == *-creds ]]; then continue; fi
 
     TF_VAR_TEST_ID=$(basename "$filename" .tfstate)
 
diff --git a/install/tests/manifests/aws-auth.yaml b/install/tests/manifests/aws-auth.yaml
@@ -0,0 +1,6 @@
+data:
+  mapUsers: |
+    - userarn: ${USERARN}
+      username: ${NAME}
+      groups:
+        - system:masters
diff --git a/install/tests/output.tf b/install/tests/output.tf
@@ -13,6 +13,11 @@ output "k3s_database" {
     value = try(module.k3s.database, null)
 }
 
+output "aws_cluster_user" {
+    sensitive = true
+    value = try(module.eks.cluster_user, null)
+}
+
 output "aws_storage" {
     sensitive = true
     value = try(module.eks.storage, null)

Original file line number	Diff line number	Diff line change
`@@ -158,7 +158,7 @@ resource "google_service_account" "cluster_user_sa" {`
`158`	`158`	`resource "google_project_iam_member" "gke-user-sa-iam" {`
`159`	`159`	`project = var.project`
`160`	`160`	`role = "roles/container.developer"`
`161`		`- member = "serviceAccount:${google_service_account.cluster_sa.email}"`
	`161`	`+ member = "serviceAccount:${google_service_account.cluster_user_sa.email}"`
`162`	`162`	`}`
`163`	`163`
`164`	`164`	`resource "google_service_account_key" "gke_sa_key" {`
Original file line number	Diff line number	Diff line change
`@@ -41,8 +41,7 @@ resource "random_password" "password" {`
`41`	`41`	`count = var.enable_external_database ? 1 : 0`
`42`	`42`
`43`	`43`	`length = 16`
`44`		`- special = true`
`45`		`- override_special = "!#$%&*()-_=+[]{}<>:?"`
	`44`	`+ special = false`
`46`	`45`	`}`
`47`	`46`
`48`	`47`	`resource "google_sql_database" "database" {`