From 47a6f15a22169af83541aa9c44d83b0f2a00b72a Mon Sep 17 00:00:00 2001
From: Aleksandar Aleksandrov <a.g.aleksandrov@gmail.com>
Date: Tue, 20 Sep 2022 12:17:27 +0300
Subject: [PATCH] Terraform workflow for preview envs

---
 .github/CODEOWNERS                            |   3 +
 dev/preview/.gitignore                        |   5 +
 .../infrastructure/harvester/namespace.tf     |   6 +
 .../infrastructure/harvester/provider.tf      |  34 +++++
 .../infrastructure/harvester/variables.tf     |  16 +++
 dev/preview/workflow/lib/common.sh            |  65 +++++++++
 dev/preview/workflow/lib/terraform.sh         | 123 ++++++++++++++++++
 .../workflow/preview/deploy-harvester.sh      |  52 ++++++++
 .../workflow/terraform/terraform-apply.sh     |  13 ++
 .../workflow/terraform/terraform-init.sh      |  13 ++
 .../workflow/terraform/terraform-plan.sh      |  13 ++
 .../workflow/terraform/terraform-workspace.sh |  28 ++++
 12 files changed, 371 insertions(+)
 create mode 100644 dev/preview/.gitignore
 create mode 100644 dev/preview/infrastructure/harvester/namespace.tf
 create mode 100644 dev/preview/infrastructure/harvester/provider.tf
 create mode 100644 dev/preview/infrastructure/harvester/variables.tf
 create mode 100755 dev/preview/workflow/lib/common.sh
 create mode 100755 dev/preview/workflow/lib/terraform.sh
 create mode 100755 dev/preview/workflow/preview/deploy-harvester.sh
 create mode 100755 dev/preview/workflow/terraform/terraform-apply.sh
 create mode 100755 dev/preview/workflow/terraform/terraform-init.sh
 create mode 100755 dev/preview/workflow/terraform/terraform-plan.sh
 create mode 100755 dev/preview/workflow/terraform/terraform-workspace.sh

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index d1cd5c1ce86b4b..cc28b9e7eba20a 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -100,6 +100,9 @@
 /.werft/*installer-tests* @gitpod-io/engineering-self-hosted
 /.werft/jobs/build/self-hosted-* @gitpod-io/engineering-self-hosted
 
+/dev/preview/infrastructure/harvester @gitpod-io/platform
+/dev/preview/workflow @gitpod-io/platform
+
 #
 # Automation
 # The following files are updated automatically so we don't want to have a specific code-owner
diff --git a/dev/preview/.gitignore b/dev/preview/.gitignore
new file mode 100644
index 00000000000000..34fbc7edd22c0d
--- /dev/null
+++ b/dev/preview/.gitignore
@@ -0,0 +1,5 @@
+# terraform
+*.hcl
+*.tfstate
+*.tfstate.backup
+*.plan
diff --git a/dev/preview/infrastructure/harvester/namespace.tf b/dev/preview/infrastructure/harvester/namespace.tf
new file mode 100644
index 00000000000000..186f7503d02f74
--- /dev/null
+++ b/dev/preview/infrastructure/harvester/namespace.tf
@@ -0,0 +1,6 @@
+resource "kubernetes_namespace" "example" {
+  provider = k8s.harvester
+  metadata {
+    name = "preview-${var.preview_name}"
+  }
+}
diff --git a/dev/preview/infrastructure/harvester/provider.tf b/dev/preview/infrastructure/harvester/provider.tf
new file mode 100644
index 00000000000000..aa34de6fc058e4
--- /dev/null
+++ b/dev/preview/infrastructure/harvester/provider.tf
@@ -0,0 +1,34 @@
+terraform {
+
+  backend "gcs" {
+    bucket = "3f4745df-preview-tf-state"
+    prefix = "preview"
+  }
+
+  required_version = ">= 1.2"
+  required_providers {
+    harvester = {
+      source  = "harvester/harvester"
+      version = ">=0.5.1"
+    }
+    k8s = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.0"
+    }
+  }
+}
+
+provider "harvester" {
+  alias      = "harvester"
+  kubeconfig = file(var.harvester_kube_path)
+}
+
+provider "k8s" {
+  alias       = "dev"
+  config_path = var.dev_kube_path
+}
+
+provider "k8s" {
+  alias       = "harvester"
+  config_path = var.harvester_kube_path
+}
diff --git a/dev/preview/infrastructure/harvester/variables.tf b/dev/preview/infrastructure/harvester/variables.tf
new file mode 100644
index 00000000000000..5214223eb196fe
--- /dev/null
+++ b/dev/preview/infrastructure/harvester/variables.tf
@@ -0,0 +1,16 @@
+variable "preview_name" {
+  type        = string
+  description = "The preview environment's name"
+}
+
+variable "harvester_kube_path" {
+  type        = string
+  description = "The path to the Harvester Cluster kubeconfig"
+  default     = "~/.kube/harvester"
+}
+
+variable "dev_kube_path" {
+  type        = string
+  description = "The path to the Dev Cluster kubeconfig"
+  default     = "~/.kube/dev"
+}
diff --git a/dev/preview/workflow/lib/common.sh b/dev/preview/workflow/lib/common.sh
new file mode 100755
index 00000000000000..e4f7caa1c5ea2e
--- /dev/null
+++ b/dev/preview/workflow/lib/common.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# this script is meant to be sourced
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+
+# predefined exit codes for checks
+export ERROR_WRONG_WORKSPACE=30
+export ERROR_CHANGE_DIR=31
+export ERROR_NO_WORKSPACE=32
+export ERROR_NO_DIR=33
+export ERROR_NO_PLAN=34
+
+function import() {
+  local file="${SCRIPT_PATH}/${1}"
+  if [ -f "${file}" ]; then
+    # shellcheck disable=SC1090
+    source "${file}"
+  else
+    echo "Error: Cannot find library at: ${file}"
+    exit 1
+  fi
+}
+
+# define some colors for our helper log function
+BLUE='\033[0;34m'
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+# NC=no color
+NC='\033[0m'
+
+function log_error() {
+  local text=$1
+  echo -e "${RED}ERROR: ${NC}${text}" 1>&2
+}
+
+function log_success() {
+  local text=$1
+  echo -e "${GREEN}SUCCESS: ${NC}${text}"
+}
+
+function log_info() {
+  local text=$1
+  echo -e "${BLUE}INFO: ${NC}${text}"
+}
+
+# Checks if we have the correct context or exits with an error
+function check_kubectx() {
+  local expected=$1
+  if [[ $(kubectl config current-context) != "${expected}" ]]; then
+    log_error "Wrong context. Wanted [${expected}], got [$(kubectl config current-context)]"
+    exit "${ERROR_WRONG_KUBECTX}"
+  fi
+}
+
+function ask() {
+    while true; do
+        # shellcheck disable=SC2162
+        read -p "$* [y/n]: " yn
+        case $yn in
+            [Yy]*) return 0  ;;
+            [Nn]*) echo "Aborted" ; return  1 ;;
+        esac
+    done
+}
diff --git a/dev/preview/workflow/lib/terraform.sh b/dev/preview/workflow/lib/terraform.sh
new file mode 100755
index 00000000000000..5cd15b868b4fb0
--- /dev/null
+++ b/dev/preview/workflow/lib/terraform.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# this script is meant to be sourced
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=./common.sh
+source "${SCRIPT_PATH}/common.sh"
+
+if [ -n "${DESTROY-}" ]; then
+  export TF_CLI_ARGS_plan="-destroy"
+fi
+
+function check_workspace() {
+  local workspace=$1
+  if [[ $(terraform workspace show) != "${workspace}" ]]; then
+    log_error "Expected to be in [${workspace}]. We are in [$(terraform workspace show)]"
+    return "${ERROR_WRONG_WORKSPACE}"
+  fi
+}
+
+function set_workspace() {
+  local workspace=$1
+  if terraform workspace list | grep -q "${workspace}"; then
+    terraform workspace select "${workspace}"
+  else
+    terraform workspace new "${workspace}"
+  fi
+}
+
+function delete_workspace() {
+  local workspace=$1
+  if [[ $(terraform workspace show) == "${workspace}" ]]; then
+    terraform workspace select default
+  fi
+
+  exists=0
+  terraform workspace list | grep -q "${workspace}" || exists=$?
+  if [[ "${exists}" == 0 ]]; then
+    terraform workspace delete "${workspace}"
+  fi
+}
+
+function terraform_init() {
+  local target_dir=${1:-$TARGET_DIR}
+  if [ -z "${target_dir-}" ]; then
+    log_error "Must provide TARGET_DIR for init"
+    return "${ERROR_NO_DIR}"
+  fi
+  pushd "${target_dir}" || return "${ERROR_CHANGE_DIR}"
+
+  terraform init
+  if [ -n "${WORKSPACE-}" ]; then
+    set_workspace "${WORKSPACE}"
+  fi
+
+  popd || return "${ERROR_CHANGE_DIR}"
+}
+
+function terraform_plan() {
+  local target_dir=${1:-$TARGET_DIR}
+  if [ -z "${target_dir-}" ]; then
+    log_error "Must provide TARGET_DIR for plan"
+    return "${ERROR_NO_DIR}"
+  fi
+
+  local static_plan
+  static_plan="$(realpath "${TARGET_DIR}")/$(basename "${TARGET_DIR}").plan"
+  local plan_location=${PLAN_LOCATION:-$static_plan}
+
+  pushd "${target_dir}" || return "${ERROR_CHANGE_DIR}"
+
+  # check if we should be in a workspace, and bail otherwise
+  if [ -n "${WORKSPACE-}" ]; then
+    check_workspace "${WORKSPACE}"
+  fi
+
+  # -detailed-exitcode will be 0=success no changes/1=failure/2=success changes
+  # therefore we capture the output so our function doesn't cause a script to terminate if the caller has `set -e`
+  EXIT_CODE=0
+  terraform plan -detailed-exitcode -out="${plan_location}" || EXIT_CODE=$?
+
+  if [[ ${EXIT_CODE} = 2 ]]; then
+    terraform show "${plan_location}"
+  fi
+
+  popd || exit "${ERROR_CHANGE_DIR}"
+
+  return "${EXIT_CODE}"
+}
+
+function terraform_apply() {
+  local target_dir=${1:-$TARGET_DIR}
+  if [ -z "${target_dir-}" ]; then
+    log_error "Must provide TARGET_DIR for apply"
+    return "${ERROR_NO_DIR}"
+  fi
+
+  local static_plan
+  static_plan="$(realpath "${TARGET_DIR}")/$(basename "${TARGET_DIR}").plan"
+  local plan_location=${PLAN_LOCATION:-$static_plan}
+
+  pushd "${target_dir}" || return "${ERROR_CHANGE_DIR}"
+
+  # check if we should be in a workspace, and bail otherwise
+  if [ -n "${WORKSPACE-}" ]; then
+    check_workspace "${WORKSPACE}"
+  fi
+
+  if [ -z "${plan_location-}" ]; then
+    log_error "Must provide PLAN_LOCATION for apply"
+    return "${ERROR_NO_PLAN}"
+  fi
+
+  # check if we should be in a workspace, and bail otherwise
+  if [ -n "${WORKSPACE-}" ]; then
+    check_workspace "${WORKSPACE}"
+  fi
+
+  terraform apply "${plan_location}"
+
+  popd || return "${ERROR_CHANGE_DIR}"
+}
diff --git a/dev/preview/workflow/preview/deploy-harvester.sh b/dev/preview/workflow/preview/deploy-harvester.sh
new file mode 100755
index 00000000000000..da1977d59a8ed8
--- /dev/null
+++ b/dev/preview/workflow/preview/deploy-harvester.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# shellcheck disable=SC2034
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+# shellcheck source=../lib/common.sh
+source "$(realpath "${SCRIPT_PATH}/../lib/common.sh")"
+
+# terraform function
+import "terraform.sh"
+
+PROJECT_ROOT=$(realpath "${SCRIPT_PATH}/../../../../")
+
+if [[ -n ${WERFT_HOST+x} ]]; then
+  TF_CLI_ARGS="-input=false"
+  TF_IN_AUTOMATION=true
+fi
+
+WORKSPACE="${TF_VAR_preview_name:-}"
+TARGET_DIR="${PROJECT_ROOT}/dev/preview/infrastructure/harvester"
+# Setting the TF_DATA_DIR is advisable if we set the PLAN_LOCATION in a different place than the dir with the tf
+TF_DATA_DIR="${TARGET_DIR}"
+
+# Illustration purposes, but this will set the plan location to $TARGET_DIR/harvester.plan if PLAN_LOCATION is not set
+static_plan="$(realpath "${TARGET_DIR}")/$(basename "${TARGET_DIR}").plan"
+PLAN_LOCATION="${PLAN_LOCATION:-$static_plan}"
+
+# export all variables
+shopt -os allexport
+
+terraform_init
+
+PLAN_EXIT_CODE=0
+terraform_plan || PLAN_EXIT_CODE=$?
+
+# If there are changes
+if [[ ${PLAN_EXIT_CODE} == 2 ]]; then
+  # If we're NOT in werft, ask if we want to apply the plan
+  if [ -z ${WERFT_HOST+x} ]; then
+    ask "Do you want to apply the plan?"
+  fi
+  terraform_apply
+fi
+
+if [ -n "${DESTROY-}" ] && [ -n "${WORKSPACE}" ]; then
+  pushd "${TARGET_DIR}"
+  delete_workspace "${WORKSPACE}"
+  popd
+fi
diff --git a/dev/preview/workflow/terraform/terraform-apply.sh b/dev/preview/workflow/terraform/terraform-apply.sh
new file mode 100755
index 00000000000000..77f7cf9df70c72
--- /dev/null
+++ b/dev/preview/workflow/terraform/terraform-apply.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+# shellcheck source=../lib/common.sh
+source "$(realpath "${SCRIPT_PATH}/../lib/common.sh")"
+
+# terraform function
+import "terraform.sh"
+
+terraform_apply
diff --git a/dev/preview/workflow/terraform/terraform-init.sh b/dev/preview/workflow/terraform/terraform-init.sh
new file mode 100755
index 00000000000000..46b409016b5c70
--- /dev/null
+++ b/dev/preview/workflow/terraform/terraform-init.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+# shellcheck source=../lib/common.sh
+source "$(realpath "${SCRIPT_PATH}/../lib/common.sh")"
+
+# terraform function
+import "terraform.sh"
+
+terraform_init
diff --git a/dev/preview/workflow/terraform/terraform-plan.sh b/dev/preview/workflow/terraform/terraform-plan.sh
new file mode 100755
index 00000000000000..214548e410f6b8
--- /dev/null
+++ b/dev/preview/workflow/terraform/terraform-plan.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+# shellcheck source=../lib/common.sh
+source "$(realpath "${SCRIPT_PATH}/../lib/common.sh")"
+
+# terraform function
+import "terraform.sh"
+
+terraform_plan
diff --git a/dev/preview/workflow/terraform/terraform-workspace.sh b/dev/preview/workflow/terraform/terraform-workspace.sh
new file mode 100755
index 00000000000000..a5b92d70f83907
--- /dev/null
+++ b/dev/preview/workflow/terraform/terraform-workspace.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_PATH=$(realpath "$(dirname "$0")")
+
+# shellcheck source=../lib/common.sh
+source "$(realpath "${SCRIPT_PATH}/../lib/common.sh")"
+
+import "terraform.sh"
+
+if [ -z "${WORKSPACE-}" ]; then
+  log_error "Must provide WORKSPACE"
+  exit "${ERROR_NO_WORKSPACE}"
+fi
+
+if [ -z "${TARGET_DIR-}" ]; then
+  log_error "Must provide TARGET_DIR"
+  exit "${ERR_NO_DIR}"
+fi
+
+if [ -z "${DESTROY-}" ]; then
+  set_workspace "${WORKSPACE}"
+else
+  pushd "${TARGET_DIR}"
+  delete_workspace "${WORKSPACE}"
+  popd
+fi