|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +set -euo pipefail |
| 4 | + |
| 5 | +sudo chown -R gitpod:gitpod /workspace |
| 6 | + |
| 7 | +# Fix weird repeat running behavior |
| 8 | +LAST_COMMIT_MSG=$(git log --pretty=format:"%s" -1) |
| 9 | +if [[ $LAST_COMMIT_MSG =~ "integration test" ]]; then exit 0; fi |
| 10 | + |
| 11 | +BRANCH="inte-test/"$(date +%Y%m%d%H%M%S) |
| 12 | +FAILURE_COUNT=0 |
| 13 | +RUN_COUNT=0 # Prevent multiple cleanup runs |
| 14 | +DO_CLEANUP=0 |
| 15 | +REVISION="" |
| 16 | +declare -A FAILURE_TESTS |
| 17 | +declare SIGNAL # used to record signal caught by trap |
| 18 | + |
| 19 | +context_name=$1 |
| 20 | +context_repo=$2 |
| 21 | + |
| 22 | +function cleanup () |
| 23 | +{ |
| 24 | + werft log phase "slack notification and cleanup $SIGNAL" "Slack notification and cleanup: $SIGNAL" |
| 25 | + echo "Check if cleanup has already done" | werft log slice "Check if clenup has already done" |
| 26 | + if [[ $DO_CLEANUP -eq 1 ]]; then |
| 27 | + echo "Skip clean up" | werft log slice "Check if clenup has already done" --done |
| 28 | + return 0 |
| 29 | + fi |
| 30 | + DO_CLEANUP=1 |
| 31 | + werft log slice "Check if clenup has already done" --done |
| 32 | + |
| 33 | + werftJobUrl="https://werft.gitpod-dev.com/job/${context_name}" |
| 34 | + |
| 35 | + if [ "${RUN_COUNT}" -eq "0" ]; then |
| 36 | + title=":x: *IDE integration test fail*" |
| 37 | + title=$title"\n_Repo:_ ${context_repo}\n_Revision:_ ${REVISION}\n_Build:_ ${context_name}" |
| 38 | + |
| 39 | + errs="Failed at preparing the preview environment" |
| 40 | + BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}" |
| 41 | + elif [ "${FAILURE_COUNT}" -ne "0" ]; then |
| 42 | + title=":x: *IDE integration test fail*" |
| 43 | + title=$title"\n_Repo:_ ${context_repo}\n_Revision:_ ${REVISION}\n_Build:_ ${context_name}" |
| 44 | + |
| 45 | + errs="" |
| 46 | + for TEST_NAME in ${!FAILURE_TESTS[*]}; do |
| 47 | + title=$title"\n_Tests_: ${TEST_NAME}" |
| 48 | + errs+="${FAILURE_TESTS["${TEST_NAME}"]}" |
| 49 | + done |
| 50 | + errs=$(echo "${errs}" | head) |
| 51 | + BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}" |
| 52 | + else |
| 53 | + title=":white_check_mark: *IDE integration test pass*" |
| 54 | + |
| 55 | + title=$title"\n_Repo:_ ${context_repo}\n_Revision:_ ${REVISION}\n_Build:_ ${context_name}" |
| 56 | + BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}}]}" |
| 57 | + fi |
| 58 | + |
| 59 | + echo "Sending Slack notificaition" | werft log slice "slack notification" |
| 60 | + curl -X POST \ |
| 61 | + -H 'Content-type: application/json' \ |
| 62 | + -d "${BODY}" \ |
| 63 | + "https://hooks.slack.com/${SLACK_NOTIFICATION_PATH}" |
| 64 | + werft log result "slack notification" "${PIPESTATUS[0]}" |
| 65 | + werft log slice "slack notification" --done |
| 66 | + |
| 67 | + git push origin :"${BRANCH}" | werft log slice "clean up" |
| 68 | + |
| 69 | + echo "Finished cleaning up based on signal $SIGNAL" | werft log slice "clean up" |
| 70 | + werft log slice "clean up" --done |
| 71 | +} |
| 72 | + |
| 73 | +sudo chown -R gitpod:gitpod /workspace |
| 74 | +gcloud auth activate-service-account --key-file /mnt/secrets/gcp-sa/service-account.json |
| 75 | +export GOOGLE_APPLICATION_CREDENTIALS= "/home/gitpod/.config/gcloud/legacy_credentials/[email protected]/adc.json" |
| 76 | + |
| 77 | +git config --global user.name roboquat |
| 78 | +git config --global user.email [email protected] |
| 79 | +git remote set-url origin https://oauth2:"${ROBOQUAT_TOKEN}"@github.com/gitpod-io/gitpod.git |
| 80 | + |
| 81 | +werft log phase "build preview environment" "build preview environment" |
| 82 | + |
| 83 | +REVISION=$(git show -s --format="%h" HEAD) |
| 84 | + |
| 85 | +# Create a new branch and asks Werft to create a preview environment for it |
| 86 | +( \ |
| 87 | + git checkout -B "${BRANCH}" && \ |
| 88 | + git commit -m "ide integration test" --allow-empty && \ |
| 89 | + git push --set-upstream origin "${BRANCH}" && \ |
| 90 | + werft run github -a with-preview=true -a with-large-vm=true |
| 91 | +) | werft log slice "build preview environment" |
| 92 | + |
| 93 | +for signal in SIGINT SIGTERM EXIT; do |
| 94 | + # shellcheck disable=SC2064 |
| 95 | + # We intentionally want the expansion to happen here as that's how we pass the signal to the function. |
| 96 | + trap "SIGNAL=${signal};cleanup" $signal |
| 97 | +done |
| 98 | + |
| 99 | +# Our current approach will start two Werft jobs. One triggered by Werft by the push to the branch and one |
| 100 | +# due to the `werft run` invocation above - the manual invocation is needed as we don't enable preview |
| 101 | +# environments by default. |
| 102 | +# |
| 103 | +# Below we find the job id of the the build that has 'with-preview' set. We don't care about the other job. |
| 104 | +# |
| 105 | +BUILD_ID=$(werft job list repo.ref==refs/heads/"${BRANCH}" -o yaml | yq4 '.result[] | select(.metadata.annotations[].key == "with-preview") | .name' | head -1) |
| 106 | +until [ "$BUILD_ID" != "" ] |
| 107 | +do |
| 108 | + sleep 1 |
| 109 | + BUILD_ID=$(werft job list repo.ref==refs/heads/"${BRANCH}" -o yaml | yq4 '.result[] | select(.metadata.annotations[].key == "with-preview") | .name' | head -1) |
| 110 | +done |
| 111 | + |
| 112 | +job_url="https://werft.gitpod-dev.com/job/${BUILD_ID}" |
| 113 | +echo "start build preview environment, job name: ${BUILD_ID}, job url: ${job_url}, this will take long time" | werft log slice "build preview environment" |
| 114 | +werft log result -d "Build job for integration test branch" url "${job_url}" |
| 115 | + |
| 116 | +while true; do |
| 117 | + set +e |
| 118 | + job_log=$(werft job get "${BUILD_ID}" -o json 2>&1) |
| 119 | + set -e |
| 120 | + if echo "$job_log" | grep -q "code = Unavailable"; then |
| 121 | + echo "Werft returned 50X for some reason. Waiting for ${BUILD_ID} to finish running. Sleeping 10 seconds." | werft log slice "build preview environment"; |
| 122 | + sleep 10 |
| 123 | + else |
| 124 | + job_phase=$(echo "$job_log" | jq --raw-output '.phase') |
| 125 | + if [[ ${job_phase} != "PHASE_DONE" ]]; then |
| 126 | + echo "Waiting for ${BUILD_ID} to finish running. Current phase: ${job_phase}. Sleeping 10 seconds." | werft log slice "build preview environment"; |
| 127 | + sleep 10 |
| 128 | + else |
| 129 | + echo "Phase reached ${job_phase}. continuing." | werft log slice "build preview environment"; |
| 130 | + break |
| 131 | + fi |
| 132 | + fi |
| 133 | +done |
| 134 | + |
| 135 | +job_success="$(werft job get "${BUILD_ID}" -o json | jq --raw-output '.conditions.success')" |
| 136 | +if [[ ${job_success} == "null" ]]; then |
| 137 | + ( |
| 138 | + echo "The build job for the preview environment failed." && \ |
| 139 | + echo "" && \ |
| 140 | + echo "See the logs for the job here for details on why: ${job_url}" && \ |
| 141 | + echo "" && \ |
| 142 | + echo "While this error is unrelated to our integration tests it does mean we can't continue as we don't have a target to run integration tests against without a working preview environment." && \ |
| 143 | + echo "" && \ |
| 144 | + echo "Marking this job as failed. Please retry the integration test job." && \ |
| 145 | + echo "" \ |
| 146 | + ) | werft log slice "build preview environment" |
| 147 | + werft log slice "build preview environment" --fail "Build job for preview environment failed" |
| 148 | + exit 1 |
| 149 | +fi |
| 150 | + |
| 151 | +echo "build success" | werft log slice "build preview environment" |
| 152 | +werft log slice "build preview environment" --done |
| 153 | + |
| 154 | +werft log phase "kubectx" "kubectx" |
| 155 | +mkdir -p /home/gitpod/.ssh |
| 156 | +/workspace/dev/preview/util/download-and-merge-harvester-kubeconfig.sh | werft log slice "kubectx" |
| 157 | +/workspace/dev/preview/install-k3s-kubeconfig.sh | werft log slice "kubectx" |
| 158 | +werft log slice "kubectx" --done |
| 159 | + |
| 160 | +werft log phase "integration test" "integration test" |
| 161 | +args=() |
| 162 | +args+=( "-kubeconfig=/home/gitpod/.kube/config" ) |
| 163 | +args+=( "-namespace=default" ) |
| 164 | +[[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) |
| 165 | +args+=( "-timeout=60m" ) |
| 166 | + |
| 167 | +IDE_TEST_LIST=(/workspace/test/tests/ide/ssh /workspace/test/tests/ide/vscode /workspace/test/tests/ide/jetbrains) |
| 168 | +for TEST_PATH in "${IDE_TEST_LIST[@]}" |
| 169 | +do |
| 170 | + TEST_NAME=$(basename "${TEST_PATH}") |
| 171 | + echo "running integration for ${TEST_NAME}" | werft log slice "test-${TEST_NAME}" |
| 172 | + RUN_COUNT=$((RUN_COUNT+1)) |
| 173 | + |
| 174 | + cd "${TEST_PATH}" |
| 175 | + set +e |
| 176 | + go test -v ./... "${args[@]}" 2>&1 | tee "${TEST_NAME}".log | werft log slice "test-${TEST_NAME}" |
| 177 | + RC=${PIPESTATUS[0]} |
| 178 | + set -e |
| 179 | + |
| 180 | + if [ "${RC}" -ne "0" ]; then |
| 181 | + FAILURE_COUNT=$((FAILURE_COUNT+1)) |
| 182 | + FAILURE_TESTS["${TEST_NAME}"]=$(grep "\-\-\- FAIL: " "${TEST_PATH}"/"${TEST_NAME}".log) |
| 183 | + werft log slice "test-${TEST_NAME}" --fail "${RC}" |
| 184 | + else |
| 185 | + werft log slice "test-${TEST_NAME}" --done |
| 186 | + fi |
| 187 | +done |
| 188 | + |
| 189 | +exit $FAILURE_COUNT |
0 commit comments