Skip to content

Commit 1bc46bd

Browse files
iQQBotiQQBot
authored
Refactor preview environment and image build methods (#19652)
* [dev-image] upgrade terraform and gcloud * update leeway version and use new cache bucket * update image tag * use oidc * fix bob * Add code web extension as package * gcr token refresh * fluentbit use service account * Add xterm web ide as package * add ide configmap patch * fix ide first page 502 * remove secret manager * fix monitoring * fix integration and delete preview * cleanup * use previewctl:install * change folder * cleanup * change leeway cache bucket for main branch * cleanup * fix * hot-deploy
1 parent 8f3888d commit 1bc46bd

File tree

86 files changed

+797
-1318
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+797
-1318
lines changed

‎.github/actions/delete-preview/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go-122-gha.23879
1+
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525
22

33
COPY entrypoint.sh /entrypoint.sh
44
ENTRYPOINT ["/entrypoint.sh"]

‎.github/actions/delete-preview/entrypoint.sh

+2-7
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,14 @@
33
set -euo pipefail
44

55
export HOME=/home/gitpod
6-
export PREVIEW_ENV_DEV_SA_KEY_PATH="$HOME/.config/gcloud/preview-environment-dev-sa.json"
76
# shellcheck disable=SC2155
87
export LEEWAY_WORKSPACE_ROOT="$(pwd)"
98
export PATH="$PATH:$HOME/bin"
109

1110
mkdir $HOME/bin
1211

13-
echo "${INPUT_SA_KEY}" > "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
14-
gcloud auth activate-service-account --key-file "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
15-
16-
leeway run dev/preview/previewctl:download
17-
18-
previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
12+
gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" --activate --quiet
13+
leeway run dev/preview/previewctl:install
1914

2015
export TF_INPUT=0
2116
export TF_IN_AUTOMATION=true

‎.github/actions/delete-preview/metadata.yml

-6
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
name: "Delete preview environment"
22
description: "Deletes a preview environment"
33
inputs:
4-
sa_key:
5-
description: "The service account key to use when authenticating with GCP"
6-
required: true
74
name:
85
description: "The name of the preview environment"
96
required: true
10-
previewctl_hash:
11-
description: "The Leeway hash of the dev/preview/previewctl:docker package to be used when downloading previewclt"
12-
required: false
137
runs:
148
using: "docker"
159
image: "Dockerfile"

‎.github/actions/deploy-gitpod/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go-122-gha.23879
1+
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525
22

33
COPY entrypoint.sh /entrypoint.sh
44
ENTRYPOINT ["/entrypoint.sh"]

‎.github/actions/deploy-gitpod/entrypoint.sh

+6-10
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,20 @@ export PREVIEW_ENV_DEV_SA_KEY_PATH="$HOME/.config/gcloud/preview-environment-dev
77
# shellcheck disable=SC2155
88
export LEEWAY_WORKSPACE_ROOT="$(pwd)"
99
export VERSION="${INPUT_VERSION}"
10+
export IMAGE_REPO_BASE="${INPUT_IMAGE_REPO_BASE}"
1011
export PATH="$PATH:$HOME/bin"
1112

1213
mkdir $HOME/bin
1314

1415
echo "Downloading installer for ${VERSION}"
15-
oci-tool fetch file -o $HOME/bin/installer --platform=linux-amd64 "eu.gcr.io/gitpod-core-dev/build/installer:${VERSION}" app/installer
16+
oci-tool fetch file -o $HOME/bin/installer --platform=linux-amd64 "${IMAGE_REPO_BASE}/installer:${VERSION}" app/installer
1617
chmod +x $HOME/bin/installer
1718

1819
echo "Download versions.yaml"
19-
oci-tool fetch file -o /tmp/versions.yaml --platform=linux-amd64 "eu.gcr.io/gitpod-core-dev/build/versions:${VERSION}" versions.yaml
20+
oci-tool fetch file -o /tmp/versions.yaml --platform=linux-amd64 "${IMAGE_REPO_BASE}/versions:${VERSION}" versions.yaml
2021

21-
echo "${INPUT_SA_KEY}" > "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
22-
gcloud auth activate-service-account --key-file "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
23-
24-
leeway run dev/preview/previewctl:download
25-
26-
echo "Setting up access to core-dev and harvester"
27-
previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
22+
gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" --activate --quiet
23+
leeway run dev/preview/previewctl:install
2824

2925
PREVIEW_NAME="$(previewctl get-name --branch "${INPUT_NAME}")"
3026
export PREVIEW_NAME
@@ -36,7 +32,7 @@ for var in WITH_DEDICATED_EMU ANALYTICS WORKSPACE_FEATURE_FLAGS; do
3632
fi
3733
done
3834

39-
previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 10m --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
35+
previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 10m
4036
leeway run dev/preview:deploy-gitpod
4137
previewctl report --branch "${PREVIEW_NAME}" >> "${GITHUB_STEP_SUMMARY}"
4238

‎.github/actions/deploy-gitpod/metadata.yml

-6
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,12 @@
11
name: "Deploy Gitpod"
22
description: "Deploys Gitpod to an existing preview environment"
33
inputs:
4-
sa_key:
5-
description: "The service account key to use when authenticating with GCP"
6-
required: true
74
name:
85
description: "The name of the preview environment to deploy Gitpod to"
96
required: false
107
version:
118
description: "The version of Gitpod to install"
129
required: true
13-
previewctl_hash:
14-
description: "The Leeway hash of the dev/preview/previewctl:docker package to be used when downloading previewclt"
15-
required: false
1610
with_dedicated_emu:
1711
description: "Dedicated Config"
1812
required: false

‎.github/actions/deploy-monitoring-satellite/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go-122-gha.23879
1+
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525
22

33
COPY entrypoint.sh /entrypoint.sh
44
ENTRYPOINT ["/entrypoint.sh"]

‎.github/actions/deploy-monitoring-satellite/entrypoint.sh

+3-9
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,17 @@
33
set -euo pipefail
44

55
export HOME=/home/gitpod
6-
export PREVIEW_ENV_DEV_SA_KEY_PATH="$HOME/.config/gcloud/preview-environment-dev-sa.json"
76
# shellcheck disable=SC2155
87
export LEEWAY_WORKSPACE_ROOT="$(pwd)"
98
export PATH="$PATH:$HOME/bin"
109

1110
mkdir $HOME/bin
1211

13-
echo "${INPUT_SA_KEY}" > "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
14-
gcloud auth activate-service-account --key-file "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
15-
16-
leeway run dev/preview/previewctl:download
17-
18-
echo "previewctl get-credentials"
19-
previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
12+
gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" --activate --quiet
13+
leeway run dev/preview/previewctl:install
2014

2115
echo "previewctl install-context"
22-
previewctl install-context --log-level debug --timeout 10m --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
16+
previewctl install-context --log-level debug --timeout 10m
2317

2418
echo "leeway run dev/preview:deploy-monitoring-satellite"
2519
leeway run dev/preview:deploy-monitoring-satellite

‎.github/actions/deploy-monitoring-satellite/metadata.yml

-7
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,5 @@
11
name: "Deploy monitoring satellite"
22
description: "Deploys monitoring satellite to an existing preview environment"
3-
inputs:
4-
sa_key:
5-
description: "The service account key to use when authenticating with GCP"
6-
required: true
7-
previewctl_hash:
8-
description: "The Leeway hash of the dev/preview/previewctl:docker package to be used when downloading previewclt"
9-
required: false
103
runs:
114
using: "docker"
125
image: "Dockerfile"

‎.github/actions/integration-tests/action.yml

+40-29
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@ inputs:
1111
preview_name:
1212
description: "Name of the preview environment to run the tests against"
1313
required: true
14-
sa_key:
15-
description: "The service account key to use when authenticating with GCP"
16-
required: true
1714
github_token:
1815
description: "The GitHub token to use when authenticating with GitHub"
1916
required: true
@@ -30,50 +27,64 @@ inputs:
3027
test_build_ref:
3128
description: "The build ref of the test run. Used in the IDE integration tests."
3229
required: false
33-
30+
integration_test_username:
31+
description: "The username for integration test"
32+
required: true
33+
integration_test_usertoken:
34+
description: "The username for integration test"
35+
required: true
36+
identity_provider:
37+
description: "GCP workload identity provider"
38+
required: true
39+
service_account:
40+
description: "GCP service account"
41+
required: true
42+
leeway_segment_key:
43+
description: "leeway analytics key"
44+
required: true
3445
runs:
3546
using: "composite"
3647
steps:
48+
- uses: actions/checkout@v4
49+
- id: env-vars
50+
name: configure env variables
51+
shell: bash
52+
run: |
53+
{
54+
echo "LEEWAY_SEGMENT_KEY=${{ inputs.leeway_segment_key }}"
55+
echo "LEEWAY_WORKSPACE_ROOT=$GITHUB_WORKSPACE"
56+
echo "HOME=/home/gitpod"
57+
} >> "$GITHUB_ENV"
3758
- id: auth
38-
uses: google-github-actions/auth@v1
59+
name: Authenticate to Google Cloud
60+
uses: google-github-actions/auth@v2
3961
with:
40-
token_format: access_token
41-
credentials_json: "${{ inputs.sa_key }}"
42-
- name: Get Secrets from GCP
43-
id: "secrets"
44-
uses: "google-github-actions/get-secretmanager-secrets@v1"
62+
workload_identity_provider: "${{ inputs.identity_provider }}"
63+
service_account: "${{ inputs.service_account }}"
64+
- name: 'Set up Cloud SDK'
65+
uses: 'google-github-actions/setup-gcloud@v2'
4566
with:
46-
secrets: |-
47-
WORKSPACE_INTEGRATION_TEST_USERNAME:gitpod-core-dev/workspace-integration-test-username
48-
WORKSPACE_INTEGRATION_TEST_USER_TOKEN:gitpod-core-dev/workspace-integration-test-user-token
67+
skip_install: true
68+
- id: docker
69+
name: configure docker
70+
shell: bash
71+
run: |
72+
gcloud auth configure-docker eu.gcr.io --quiet
4973
- name: Setup
5074
shell: bash
5175
env:
52-
PREVIEW_ENV_DEV_SA_KEY: ${{ inputs.sa_key }}
5376
PREVIEW_NAME: ${{ inputs.preview_name }}
5477
run: |
55-
export LEEWAY_WORKSPACE_ROOT="$(pwd)"
56-
export HOME="/home/gitpod"
57-
export PREVIEW_ENV_DEV_SA_KEY_PATH="/home/gitpod/.config/gcloud/preview-environment-dev-sa.json"
58-
59-
echo "${PREVIEW_ENV_DEV_SA_KEY}" > "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
60-
gcloud auth activate-service-account --key-file "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
61-
6278
leeway run dev/preview/previewctl:install
63-
64-
echo "Setting up access to core-dev and harvester"
65-
previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
66-
67-
previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 1m --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
79+
previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 10m
6880
6981
- name: Integration Test
7082
id: integration-test
7183
shell: bash
7284
env:
7385
ROBOQUAT_TOKEN: ${{ inputs.github_token }}
74-
INTEGRATION_TEST_USERNAME: ${{ steps.secrets.outputs.WORKSPACE_INTEGRATION_TEST_USERNAME }}
75-
INTEGRATION_TEST_USER_TOKEN: ${{ steps.secrets.outputs.WORKSPACE_INTEGRATION_TEST_USER_TOKEN }}
76-
PREVIEW_ENV_DEV_SA_KEY: ${{ inputs.sa_key }}
86+
INTEGRATION_TEST_USERNAME: ${{ inputs.integration_test_username }}
87+
INTEGRATION_TEST_USER_TOKEN: ${{ inputs.integration_test_usertoken }}
7788
PREVIEW_NAME: ${{ inputs.preview_name }}
7889
TEST_USE_LATEST_VERSION: ${{ inputs.latest_ide_version }}
7990
TEST_BUILD_ID: ${{ inputs.test_build_id }}

‎.github/actions/preview-create/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go-122-gha.23879
1+
FROM eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525
22

33
COPY entrypoint.sh /entrypoint.sh
44
ENTRYPOINT ["/entrypoint.sh"]

‎.github/actions/preview-create/entrypoint.sh

+2-16
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,17 @@
33
set -euo pipefail
44

55
export HOME=/home/gitpod
6-
export PREVIEW_ENV_DEV_SA_KEY_PATH="$HOME/.config/gcloud/preview-environment-dev-sa.json"
76
# shellcheck disable=SC2155
87
export LEEWAY_WORKSPACE_ROOT="$(pwd)"
98
export PATH="$PATH:$HOME/bin"
109

1110
mkdir $HOME/bin
1211

13-
echo "${INPUT_SA_KEY}" >"${PREVIEW_ENV_DEV_SA_KEY_PATH}"
14-
gcloud auth activate-service-account --key-file "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
15-
16-
leeway run dev/preview/previewctl:download
17-
previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}"
18-
19-
replace="module.preview_gce[0].google_compute_instance.default"
20-
if [[ "${INPUT_INFRASTRUCTURE_PROVIDER}" = "harvester " ]]; then
21-
replace="module.preview_harvester[0].harvester_virtualmachine.harvester"
22-
fi
23-
24-
if [[ "${INPUT_RECREATE_VM:-x}" == "true" ]]; then
25-
export TF_CLI_ARGS_plan="-replace=${replace}"
26-
fi
12+
gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" --activate --quiet
13+
leeway run dev/preview/previewctl:install
2714

2815
TF_VAR_preview_name="$(previewctl get-name --branch "${INPUT_NAME}")"
2916
export TF_VAR_preview_name
30-
export TF_VAR_infra_provider="${INPUT_INFRASTRUCTURE_PROVIDER}"
3117
export TF_VAR_with_large_vm="${INPUT_LARGE_VM}"
3218
export TF_VAR_gce_use_spot="${INPUT_PREEMPTIBLE}"
3319
export TF_INPUT=0

‎.github/actions/preview-create/metadata.yml

-6
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,6 @@ inputs:
1515
description: "Whether to use preemptible VMs for the env"
1616
required: true
1717
default: true
18-
sa_key:
19-
description: "The service account key to use when authenticating with GCP"
20-
required: true
21-
previewctl_hash:
22-
description: "The Leeway hash of the dev/preview/previewctl:docker package to be used when downloading previewclt"
23-
required: false
2418
recreate_vm:
2519
description: "Whether to recreate the VM"
2620
required: false

‎.github/actions/setup-environment/action.yml

+12-21
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
name: Setup environment
2-
2+
description: "Setup environment"
33
inputs:
4-
sa_key:
4+
identity_provider:
5+
description: "GCP workload identity provider"
6+
required: true
7+
service_account:
58
description: "GCP service account"
69
required: true
710
leeway_segment_key:
@@ -14,33 +17,21 @@ runs:
1417
- uses: actions/checkout@v4
1518
- id: auth
1619
name: Authenticate to Google Cloud
17-
uses: google-github-actions/auth@v1
20+
uses: google-github-actions/auth@v2
1821
with:
19-
credentials_json: "${{ inputs.sa_key }}"
20-
- id: gcloud-auth
21-
name: gcloud auth activate-service-account
22-
shell: bash
23-
run: |
24-
gcloud auth activate-service-account --key-file ${{ steps.auth.outputs.credentials_file_path }}
22+
workload_identity_provider: "${{ inputs.identity_provider }}"
23+
service_account: "${{ inputs.service_account }}"
24+
- name: 'Set up Cloud SDK'
25+
uses: 'google-github-actions/setup-gcloud@v2'
26+
with:
27+
skip_install: true
2528
- id: env-vars
2629
name: configure env variables
2730
shell: bash
2831
run: |
29-
SA_KEY_FILE=$(mktemp)
30-
echo '${{ inputs.sa_key }}' > "$SA_KEY_FILE"
31-
32-
trap 'rm -f ${SA_KEY_FILE}' EXIT
33-
3432
gcloud auth configure-docker eu.gcr.io --quiet
3533
36-
PREVIEW_ENV_DEV_SA_KEY_PATH="$GITHUB_WORKSPACE/.config/gcloud/preview-environment-dev-sa.json"
37-
3834
{
3935
echo "LEEWAY_SEGMENT_KEY=${{ inputs.leeway_segment_key }}"
4036
echo "LEEWAY_WORKSPACE_ROOT=$GITHUB_WORKSPACE"
41-
echo "PREVIEW_ENV_DEV_SA_KEY_PATH=${PREVIEW_ENV_DEV_SA_KEY_PATH}"
4237
} >> "$GITHUB_ENV"
43-
44-
# Authenticate with GCP so we can use the Leeway cache
45-
mkdir -p "$(dirname "$PREVIEW_ENV_DEV_SA_KEY_PATH")"
46-
echo '${{ inputs.sa_key }}' > "${PREVIEW_ENV_DEV_SA_KEY_PATH}"

0 commit comments

Comments
 (0)