Skip to content

Improve the script of integration test #12793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 10, 2022
Merged

Improve the script of integration test #12793

Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8422bb3
werft: Retry when werfts returns 50X.
utam0k Sep 9, 2022
21010a2
werft: extend the timeout of workspace's integration test.
utam0k Sep 9, 2022
2914ee8
werft: Prevet multiple cleanup runs
utam0k Sep 9, 2022
4347334
werft: Fix weird repeat running behavior
Sep 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 60 additions & 47 deletions .werft/workspace-run-integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ set -euo pipefail

BRANCH="wk-inte-test/"$(date +%Y%m%d%H%M%S)
FAILURE_COUNT=0
RUN_COUNT=0
RUN_COUNT=0 # Prevent multiple cleanup runs
DO_CLEANUP=0
declare -A FAILURE_TESTS
declare SIGNAL # used to record signal caught by trap

Expand All @@ -13,46 +14,50 @@ context_repo=$2

function cleanup ()
{
werft log phase "slack notification and cleanup $SIGNAL" "Slack notification and cleanup: $SIGNAL"

werftJobUrl="https://werft.gitpod-dev.com/job/${context_name}"

if [ "${RUN_COUNT}" -eq "0" ]; then
title=":x: *Workspace integration test fail*"
title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"

errs="Failed at preparing the preview environment"
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}"
elif [ "${FAILURE_COUNT}" -ne "0" ]; then
title=":x: *Workspace integration test fail*"
title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"

errs=""
for TEST_NAME in ${!FAILURE_TESTS[*]}; do
title=$title"\n_Tests_: ${TEST_NAME}"
errs+="${FAILURE_TESTS["${TEST_NAME}"]}"
done
errs=$(echo "${errs}" | head)
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}"
else
title=":white_check_mark: *Workspace integration test pass*"

title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}}]}"
fi

echo "Sending Slack notificaition" | werft log slice "slack notification"
curl -X POST \
-H 'Content-type: application/json' \
-d "${BODY}" \
"https://hooks.slack.com/${SLACK_NOTIFICATION_PATH}"
werft log result "slack notification" "${PIPESTATUS[0]}"
werft log slice "slack notification" --done

git push origin :"${BRANCH}" | werft log slice "clean up"

echo "Finished cleaning up based on signal $SIGNAL" | werft log slice "clean up"
werft log slice "clean up" --done
if [[ $DO_CLEANUP -eq 0 ]]; then
return 0
fi
DO_CLEANUP=1
werft log phase "slack notification and cleanup $SIGNAL" "Slack notification and cleanup: $SIGNAL"

werftJobUrl="https://werft.gitpod-dev.com/job/${context_name}"

if [ "${RUN_COUNT}" -eq "0" ]; then
title=":x: *Workspace integration test fail*"
title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"

errs="Failed at preparing the preview environment"
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}"
elif [ "${FAILURE_COUNT}" -ne "0" ]; then
title=":x: *Workspace integration test fail*"
title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"

errs=""
for TEST_NAME in ${!FAILURE_TESTS[*]}; do
title=$title"\n_Tests_: ${TEST_NAME}"
errs+="${FAILURE_TESTS["${TEST_NAME}"]}"
done
errs=$(echo "${errs}" | head)
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}},{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\`\`\`\\n${errs}\\n\`\`\`\"}}]}"
else
title=":white_check_mark: *Workspace integration test pass*"

title=$title"\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}"
BODY="{\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"${title}\"},\"accessory\":{\"type\":\"button\",\"text\":{\"type\":\"plain_text\",\"text\":\":werft: Go to Werft\",\"emoji\":true},\"value\":\"click_me_123\",\"url\":\"${werftJobUrl}\",\"action_id\":\"button-action\"}}]}"
fi

echo "Sending Slack notificaition" | werft log slice "slack notification"
curl -X POST \
-H 'Content-type: application/json' \
-d "${BODY}" \
"https://hooks.slack.com/${SLACK_NOTIFICATION_PATH}"
werft log result "slack notification" "${PIPESTATUS[0]}"
werft log slice "slack notification" --done

git push origin :"${BRANCH}" | werft log slice "clean up"

echo "Finished cleaning up based on signal $SIGNAL" | werft log slice "clean up"
werft log slice "clean up" --done
}

sudo chown -R gitpod:gitpod /workspace
Expand Down Expand Up @@ -97,13 +102,21 @@ echo "start build preview environment, job name: ${BUILD_ID}, job url: ${job_url
werft log result -d "Build job for integration test branch" url "${job_url}"

while true; do
job_phase=$(werft job get "${BUILD_ID}" -o json | jq --raw-output '.phase')
if [[ ${job_phase} != "PHASE_DONE" ]]; then
echo "Waiting for ${BUILD_ID} to finish running. Current phase: ${job_phase}. Sleeping 10 seconds." | werft log slice "build preview environment";
set +e
job_log=$(werft job get "${BUILD_ID}" -o json 2>&1)
set -e
if echo "$job_log" | grep -q "code = Unavailable"; then
echo "Werft returned 50X for some reason. Waiting for ${BUILD_ID} to finish running. Sleeping 10 seconds." | werft log slice "build preview environment";
sleep 10
else
echo "Phase reached ${job_phase}. continuing." | werft log slice "build preview environment";
break
job_phase=$(echo "$job_log" | jq --raw-output '.phase')
if [[ ${job_phase} != "PHASE_DONE" ]]; then
echo "Waiting for ${BUILD_ID} to finish running. Current phase: ${job_phase}. Sleeping 10 seconds." | werft log slice "build preview environment";
sleep 10
else
echo "Phase reached ${job_phase}. continuing." | werft log slice "build preview environment";
break
fi
fi
done

Expand Down Expand Up @@ -137,7 +150,7 @@ args=()
args+=( "-kubeconfig=/home/gitpod/.kube/config" )
args+=( "-namespace=default" )
[[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" )
args+=( "-timeout=60m" )
args+=( "-timeout=120m" )
args+=( "-p=1" )

WK_TEST_LIST=(/workspace/test/tests/components/content-service /workspace/test/tests/components/image-builder /workspace/test/tests/components/ws-daemon /workspace/test/tests/components/ws-manager /workspace/test/tests/workspace)
Expand Down