fix: do not deregister GitLab tokens which are still in use (#1102)

## Description

So far we haven't tracked the usage of the GitLab Runner token. In case
the Runner is stopped and a new one is spawned, race conditions might
occur as explained in #1062. In consequence the new Runner used a token
deleted by the old Runner. This leads into downtimes as the token is no
longer valid and can't be used.

This PR converts the token into a JSON format and ensures that tokens
which are still in use, are not deregistered.

Fixes #1062 

## Migrations needed

In case you want to rollback to a previous version you have to convert
the SSM parameter containing the token and usage counter in JSON format
back to a plain token string.

## Verification

- [x] script changes were tested locally
- [x] Test Runner was started to ensure that the token conversion works

---------

Co-authored-by: long-wan-ep <[email protected]>

Author: kayman-mk

template/gitlab-runner.tftpl

@@ -18,7 +18,26 @@ sed -i.bak s/__PARENT_TAG__/$PARENT_TAG/g /etc/gitlab-runner/config.toml
 ${pre_install_certificates}
 
 # fetch Runner token from SSM and validate it
-token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters | .[0] | .Value")
+json_token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -cr ".Parameters[0].Value" | tr -d '\r\n')
+
+# TODO 2024-03-22: this conversion can be removed as soon as all callers switched to version 7.4.1+
+if [[ ! "$json_token" =~ ^\{.* ]]; then
+  # plain text token -> convert to JSON and store
+  json_token="{\"token\": \"$json_token\", \"usage_counter\": 1}"
+
+  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1
+
+  echo "GitLab Runner token converted into new JSON format"
+else
+  # increment the usage_counter as we are using the token now
+  usage_counter=$(echo $json_token | jq -r .usage_counter)
+  usage_counter=$(($usage_counter+1))
+  json_token=$(echo $json_token | jq -c ".usage_counter = $usage_counter")
+
+  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1
+
+  token=$(echo $json_token | jq -r '.token')
+fi
 
 valid_token=true
 if [[ "$token" != "null" ]]
@@ -89,7 +108,9 @@ then
     --form "access_level=${gitlab_runner_access_level}" \
     | jq -r .token)
   fi
-  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --value="$token" --region "${secure_parameter_store_region}"
+
+  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" \
+    --value="{\"token\": \"$token\", \"usage_counter\": 1}" --region "${secure_parameter_store_region}"
 fi
 
 sed -i.bak s/__REPLACED_BY_USER_DATA__/$token/g /etc/gitlab-runner/config.toml
@@ -181,11 +202,35 @@ WantedBy=multi-user.target
 
 EOF
 
-cat <<EOF > /opt/remove_gitlab_registration.sh
+cat <<'EOF' > /opt/remove_gitlab_registration.sh
 #!/bin/bash
-echo "Removing Gitlab Runner ..."
-aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="null" 2>&1
-curl -sS ${curl_cacert} --request DELETE "${runners_gitlab_url}/api/v4/runners" --form "token=$token" 2>&1
+json_token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters[0].Value" | tr -d '\r\n')
+deregister_runner=true
+
+usage_counter=$(echo $json_token | jq -r .usage_counter)
+
+# ensure that the token is not in use by another Runner
+if [[ $usage_counter -gt 1 ]]; then
+  deregister_runner=false
+  token="not needed"
+else
+  token=$(echo $json_token | jq -r .token)
+fi
+
+if [[ $deregister_runner == "true" ]]; then
+  echo "Removing Gitlab Runner ..."
+
+  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="{\"token\": \"null\", \"usage_counter\": 0}" 2>&1
+  curl -sS ${curl_cacert} --request DELETE "${runners_gitlab_url}/api/v4/runners" --form "token=$token" 2>&1
+else
+  usage_counter=$(echo $json_token | jq -r .usage_counter)
+  usage_counter=$(($usage_counter-1))
+  json_token=$(echo $json_token | jq -c ".usage_counter = $usage_counter")
+
+  aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1
+
+  echo "Token still in use. GitLab Runner not removed from GitLab."
+fi
 
 EOF