gitlab-runner.tftpl

# Provide the parent instance id in the spawned runner tags
PARENT_INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
PARENT_TAG="gitlab-runner-parent-id,$PARENT_INSTANCE_ID"

mkdir -p /etc/gitlab-runner
cat > /etc/gitlab-runner/config.toml <<- EOF

${runners_config}

EOF

cat > /etc/gitlab-runner/runners_userdata.sh <<- EOF
${runners_userdata}
EOF

sed -i.bak s/__PARENT_TAG__/$PARENT_TAG/g /etc/gitlab-runner/config.toml

${pre_install_certificates}

# TODO remove the "else" block after v8.0.0
if [[ "${preregistered_runner_token_ssm_parameter_name}" != "" ]]; then
  # this is the new standard registration method: the runner is already registered in GitLab and the token is stored in SSM
  preregistered_runner_token=$(aws ssm get-parameter --name "${preregistered_runner_token_ssm_parameter_name}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameter | .Value")

  if [ -z "$preregistered_runner_token" ]; then
    echo "ERROR: The preregistered runner token is not available in SSM."
    exit 1
  fi

  sed -i.bak s/__REPLACED_BY_USER_DATA__/$preregistered_runner_token/g /etc/gitlab-runner/config.toml

  # TODO after v8.0.0: this logic can be removed. We can insert the token directly into the config.toml
else
    # fetch Runner token from SSM and validate it
    json_token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -cr ".Parameters[0].Value" | tr -d '\r\n')

    # TODO 2024-03-22: this conversion can be removed as soon as all callers switched to version 7.4.1+
    if [[ ! "$json_token" =~ ^\{.* ]]; then
      # plain text token -> convert to JSON and store
      json_token="{\"token\": \"$json_token\", \"usage_counter\": 1}"

      aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1

      echo "GitLab Runner token converted into new JSON format"
    else
      # increment the usage_counter as we are using the token now
      usage_counter=$(echo $json_token | jq -r .usage_counter)
      usage_counter=$(($usage_counter+1))
      json_token=$(echo $json_token | jq -c ".usage_counter = $usage_counter")

      aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1

      token=$(echo $json_token | jq -r '.token')
    fi

    valid_token=true
    if [[ "$token" != "null" ]]
    then
      valid_token_response=$(curl -s -o /dev/null -w "%%{response_code}" ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners/verify" --form "token=$token" )
      [[ "$valid_token_response" != "200" ]] && valid_token=false
    fi

    if [[ "${runners_token}" == "__REPLACED_BY_USER_DATA__" && "$token" == "null" ]] || [[ "$valid_token" == "false" ]]
    then
      if [ "${use_new_runner_authentication_gitlab_16}" == "true" ]
      then
        runner_type_param=""
        if [ "${gitlab_runner_type}" = "group" ]; then
          if [ -z "${gitlab_runner_group_id}" ]; then
            echo "ERROR: If the runner type is group, you must specify a group_id".
            exit 1
          fi
          runner_type_param='--form group_id=${gitlab_runner_group_id}'
        elif [ "${gitlab_runner_type}" = "project" ]; then
          if [ -z "${gitlab_runner_project_id}" ]; then
            echo "ERROR: If the runner type is project_type, you must specify a project_id".
            exit 1
          fi
          runner_type_param='--form project_id=${gitlab_runner_project_id}'
        fi

        # fetch gitlab token from SSM
        gitlab_token=$(aws ssm get-parameter --name "${secure_parameter_store_gitlab_token_name}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameter | .Value")

        response=$(curl ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/user/runners" \
          --header "private-token: $gitlab_token" \
          --form "tag_list=${gitlab_runner_tag_list}" \
          --form "description=${gitlab_runner_description}" \
          --form "locked=${gitlab_runner_locked_to_project}" \
          --form "run_untagged=${gitlab_runner_run_untagged}" \
          --form "maximum_timeout=${gitlab_runner_maximum_timeout}" \
          --form "runner_type=${gitlab_runner_type}_type" \
          $runner_type_param \
          --form "access_level=${gitlab_runner_access_level}")

        token=$(echo $response | jq -r '.token')
        if [[ "$token" == null ]]
        then
            message=$(echo $response | jq -r '.message // .error_description')
            if [[ "$message" != null ]]
            then
                echo "ERROR: Couldn't register the Runner. GitLab API call returned $message".
                exit 1
            fi
        fi
      else
        gitlab_runner_registration_token=${gitlab_runner_registration_token}

        # fetch registration token from SSM
        if [[ "$gitlab_runner_registration_token" == "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__" ]]
        then
          gitlab_runner_registration_token=$(aws ssm get-parameter --name "${secure_parameter_store_gitlab_runner_registration_token_name}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameter | .Value")
        fi

        token=$(curl ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners" \
        --form "token=$gitlab_runner_registration_token" \
        --form "tag_list=${gitlab_runner_tag_list}" \
        --form "description=${gitlab_runner_description}" \
        --form "locked=${gitlab_runner_locked_to_project}" \
        --form "run_untagged=${gitlab_runner_run_untagged}" \
        --form "maximum_timeout=${gitlab_runner_maximum_timeout}" \
        --form "access_level=${gitlab_runner_access_level}" \
        | jq -r .token)
      fi

      aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" \
        --value="{\"token\": \"$token\", \"usage_counter\": 1}" --region "${secure_parameter_store_region}"
    fi

    sed -i.bak s/__REPLACED_BY_USER_DATA__/$token/g /etc/gitlab-runner/config.toml
fi

if [[ "${use_private_key}" == "true" ]]
then
  echo "${public_key}" > /root/.ssh/id_rsa.pub
  echo "${private_key}" > /root/.ssh/id_rsa

  chmod 600 /root/.ssh/id_rsa
fi

ssm_sentry_dsn=$(aws ssm get-parameters --names "${secure_parameter_store_runner_sentry_dsn}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters | .[0] | .Value")
if [[ "${sentry_dsn}" == "__SENTRY_DSN_REPLACED_BY_USER_DATA__" && "$ssm_sentry_dsn" == "null" ]]
then
  ssm_sentry_dsn=""
fi

# For those of you wondering why commas are used in the sed below instead of forward slashes, see https://stackoverflow.com/a/16778711/13169919
# It is because the Sentry DSN contains forward slashes as it is an URL so it would break out of the sed command with forward slashes as delimiters :)
sed -i.bak s,__SENTRY_DSN_REPLACED_BY_USER_DATA__,"$ssm_sentry_dsn",g /etc/gitlab-runner/config.toml

${pre_install}

if [[ "${runners_executor}" == "docker" ]]
then
  echo 'installing docker'
  if grep -q ':2$' /etc/system-release-cpe  ; then
    # AWS Linux 2 provides docker via extras only and uses systemd (https://aws.amazon.com/amazon-linux-2/release-notes/)
    amazon-linux-extras install docker
    usermod -a -G docker ec2-user
    systemctl enable docker
    systemctl start docker
  else
    yum install docker -y
    usermod -a -G docker ec2-user
    service docker start
  fi
fi

if [[ "${runners_install_amazon_ecr_credential_helper}" == "true" ]]
then
  yum install amazon-ecr-credential-helper -y
fi

if [[ "${runners_executor}" == "docker+machine" ]]
then
  if [[ "${docker_machine_download_url}" == "" ]]
  then
    echo "Installing Docker Machine using preconfigured URL"
    curl --fail --retry 6 -L https://arr-cki-prod-docker-machine.s3.amazonaws.com/v${docker_machine_version}/docker-machine-$(uname -s)-$(uname -m) > /tmp/docker-machine
  else
    echo "Installing Docker Machine using custom URL"
    curl --fail --retry 6 -L ${docker_machine_download_url} >/tmp/docker-machine
  fi

  chmod +x /tmp/docker-machine && \
    mv /tmp/docker-machine /usr/local/bin/docker-machine && \
    ln -s /usr/local/bin/docker-machine /usr/bin/docker-machine
  docker-machine --version

  # Create a dummy machine so that the cert is generated properly
  # See: https://gitlab.com/gitlab-org/gitlab-runner/issues/3676
  # See: https://github.com/docker/machine/issues/3845#issuecomment-280389178
  export USER=root
  export HOME=/root
  docker-machine create --driver none --url localhost dummy-machine
  docker-machine rm -y dummy-machine
  unset HOME
  unset USER
fi

if [[ "${runners_executor}" == "docker-autoscaler" ]]; then
  echo "installing docker"
  yum install docker -y
  usermod -a -G docker ec2-user
  service docker start

  echo "Installing Docker fleeting plugin"
  # Determine architecture
  if [[ "$(uname -m)" == "x86_64" ]]; then
    ARCH="amd64"
  elif [[ "$(uname -m)" == "i686" ]]; then
    ARCH="386"
  elif [[ "$(uname -m)" == "armv7l" ]]; then
    ARCH="arm"
  elif [[ "$(uname -m)" == "aarch64" ]]; then
    ARCH="arm64"
  else
    echo "Unsupported architecture"
    exit 1
  fi

  wget "https://gitlab.com/gitlab-org/fleeting/plugins/aws/-/releases/v${fleeting_plugin_version}/downloads/fleeting-plugin-aws-$(uname -s | tr '[:upper:]' '[:lower:]')-$ARCH"
  chmod +x fleeting-plugin-aws-*
  mv fleeting-plugin-aws-* /bin/fleeting-plugin-aws

  mkdir ~/.aws
  cat <<EOF > ~/.aws/config
[default]
region = "${aws_region}"

EOF

fi

# A small script to remove this runner from being registered with Gitlab. Executed at shutdown.
cat <<EOF > /etc/systemd/system/remove-gitlab-registration.service
[Unit]
Description=Remove the GitLab Runner from GitLab at shutdown
After=network-online.target
Wants=network-online.target
Before=shutdown.target reboot.target halt.target kexec.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/true
ExecStop=/opt/remove_gitlab_registration.sh

[Install]
WantedBy=multi-user.target

EOF

# TODO remove the following block after v8.0.0
# no longer needed as we use preregistered runners
if [[ "${preregistered_runner_token_ssm_parameter_name}" == "" ]]; then
    cat <<'EOF' > /opt/remove_gitlab_registration.sh
    #!/bin/bash
    json_token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters[0].Value" | tr -d '\r\n')
    deregister_runner=true

    usage_counter=$(echo $json_token | jq -r .usage_counter)

    # ensure that the token is not in use by another Runner
    if [[ $usage_counter -gt 1 ]]; then
      deregister_runner=false
      token="not needed"
    else
      token=$(echo $json_token | jq -r .token)
    fi

    if [[ $deregister_runner == "true" ]]; then
      echo "Removing Gitlab Runner ..."

      aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="{\"token\": \"null\", \"usage_counter\": 0}" 2>&1
      curl -sS ${curl_cacert} --request DELETE "${runners_gitlab_url}/api/v4/runners" --form "token=$token" 2>&1
    else
      usage_counter=$(echo $json_token | jq -r .usage_counter)
      usage_counter=$(($usage_counter-1))
      json_token=$(echo $json_token | jq -c ".usage_counter = $usage_counter")

      aws ssm put-parameter --overwrite --type SecureString  --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="$json_token" 2>&1

      echo "Token still in use. GitLab Runner not removed from GitLab."
    fi

EOF

    chmod a+x /opt/remove_gitlab_registration.sh
    systemctl enable remove-gitlab-registration.service

    # start the service. Otherwise the stop action will not be triggered at shutdown.
    service remove-gitlab-registration start
fi

if ! ( rpm -q gitlab-runner >/dev/null )
then
  curl --fail --retry 6 -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh | bash
  yum install gitlab-runner-${gitlab_runner_version} -y
fi

#############################
# GRACEFUL SHUTDOWN SERVICE #
#############################

# set timeout for runner service to wait before stopping
mkdir /etc/systemd/system/gitlab-runner.service.d
cat <<EOF > /etc/systemd/system/gitlab-runner.service.d/kill.conf
[Service]
# This initiates a graceful shutdown of the service: https://docs.gitlab.com/runner/commands/#signals
KillSignal=SIGQUIT
TimeoutStopSec=0
EOF

cat <<'EOF' > /opt/monitor_runner.sh
#!/bin/bash

# Fetch instance ID since we need it for the lifecycle hook actions.
imds_token=$(curl -sSf -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
instance_id=$(curl -sSf -H "X-aws-ec2-metadata-token: $imds_token" "http://169.254.169.254/latest/meta-data/instance-id")
region=$(curl -sSf -H "X-aws-ec2-metadata-token: $imds_token" "http://169.254.169.254/latest/meta-data/placement/region")

# Function to send complete-lifecycle-action
send_complete_lifecycle_action() {
    echo "Sending complete-lifecycle-action for instance $instance_id..."

    if [ -z "$region" ]; then
      echo "Failed to retrieve AWS region."
      exit 1
    fi
    echo "Current AWS Region: $region"

    # Retrieve the Auto Scaling Group name associated with the instance
    asg_name=$(aws --region $region ec2 describe-tags --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:autoscaling:groupName" --query 'Tags[].Value' --output text)
    if [ -z "$asg_name" ]; then
      echo "Failed to retrieve Auto Scaling Group name."
      exit 1
    fi
    echo "Auto Scaling Group Name: $asg_name"

    # Retrieve the lifecycle hook names associated with the Auto Scaling Group
    lifecycle_hooks=$(aws --region $region autoscaling describe-lifecycle-hooks --auto-scaling-group-name $asg_name --query "LifecycleHooks[?LifecycleTransition=='autoscaling:EC2_INSTANCE_TERMINATING'].LifecycleHookName" --output text)
    if [ -z "$lifecycle_hooks" ]; then
      echo "No lifecycle hooks found for Auto Scaling Group: $asg_name"
      exit 1
    fi
    echo "Lifecycle Hooks: $lifecycle_hooks"

    # This is needed to let CloudWatch Logs catch up.
    echo "Will send complete actions in 10 seconds..."
    sleep 10

    for hook_name in $lifecycle_hooks; do
      echo "Completing lifecycle action for hook: $hook_name"

      aws autoscaling complete-lifecycle-action \
          --lifecycle-hook-name $hook_name \
          --auto-scaling-group-name $asg_name \
          --lifecycle-action-result CONTINUE \
          --instance-id $instance_id \
          --region $region

      if [ $? -eq 0 ]; then
          echo "Lifecycle action completed successfully for hook: $hook_name"
      else
          echo "Failed to complete lifecycle action for hook: $hook_name"
      fi
    done
}

# Function to monitor the existence of the Runner process
monitor_process() {
      systemctl --no-block stop gitlab-runner.service
      sleep 5 # grace period to allow GitLab Runner to shutdown

      while true; do
        status=$(systemctl is-active gitlab-runner.service)
        if [ "$status" = "inactive" ] || [ "$status" = "failed" ]; then
          echo "GitLab Runner service $status. Proceeding with cleanup."
          send_complete_lifecycle_action
        else
          echo "GitLab Runner Service still running, sleeping..."
        fi
        # it usually takes a while to complete the jobs, so don't poll too often
        sleep 30
    done
}

# Main function
main() {
    # Poll whether the instance is in the terminating state using instance metadata API
    imds_token=$(curl -sSf -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
    target_lifecycle_state=$(curl -sSf -H "X-aws-ec2-metadata-token: $imds_token" "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state")

    if [ "$target_lifecycle_state" = "Terminated" ]; then
        # Monitor the existence of the Runner process
        monitor_process
    else
        if [ "${user_data_trace_log}" = "true" ]; then
            echo "Instance target lifecycle state is $target_lifecycle_state. No action required."
        fi
    fi
}

# Execute the main function
while true; do
    main
    sleep 10
done

EOF

cat <<EOF > /etc/systemd/system/monitor-runner.service
[Unit]
Description=Monitor Gitlab Runner process and manage graceful shutdown
After=network.target

[Service]
Type=simple
ExecStart=/opt/monitor_runner.sh
Restart=always

[Install]
WantedBy=multi-user.target
EOF

chmod a+x /opt/monitor_runner.sh
systemctl enable monitor-runner.service
systemctl start monitor-runner.service


${post_install}

chkconfig gitlab-runner on