From 9da61f45a381401fab3832e8a5fda7b5023a19cb Mon Sep 17 00:00:00 2001 From: Bjorn Olsen Date: Mon, 5 Dec 2022 09:34:33 +0200 Subject: [PATCH 1/3] feat: Support self-signed certificates --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 2 +- README.md | 2 +- examples/runner-certificates/README.md | 120 ++++++++++++++++++ examples/runner-certificates/main.tf | 73 +++++++++++ .../my_company_ca_cert_bundle.crt | 15 +++ .../my_gitlab_instance_cert.crt | 4 + examples/runner-certificates/providers.tf | 11 ++ examples/runner-certificates/variables.tf | 29 +++++ examples/runner-certificates/versions.tf | 26 ++++ locals.tf | 38 ++++++ main.tf | 7 +- outputs.tf | 6 +- tags.tf | 4 - template/gitlab-runner.tpl | 10 +- template/runner-config.tpl | 1 + variables.tf | 12 ++ 17 files changed, 346 insertions(+), 15 deletions(-) create mode 100644 examples/runner-certificates/README.md create mode 100644 examples/runner-certificates/main.tf create mode 100644 examples/runner-certificates/my_company_ca_cert_bundle.crt create mode 100644 examples/runner-certificates/my_gitlab_instance_cert.crt create mode 100644 examples/runner-certificates/providers.tf create mode 100644 examples/runner-certificates/variables.tf create mode 100644 examples/runner-certificates/versions.tf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08bbfb1c7..92c3c2c61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: "runner-multi-region", "runner-pre-registered", "runner-public", + "runner-certificates" ] defaults: run: diff --git a/CHANGELOG.md b/CHANGELOG.md index ae72833b0..28c2437c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,7 @@ ### ⚠ BREAKING CHANGES -* The module is upgraded to Terraform AWS provider 4.x. All new development will only support the new AWS Terraform provider. We keep a branch `terraform-aws-provider-3` to witch we welcome backports to AWS Terraform 3.x provider. Besides reviewing PR's we will do not any active checking on maintance on this branch. We strongly advise to update your deployment to the new provider version. For more details about upgrading see the [upgrade guide](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/version-4-upgrade). +* The module is upgraded to Terraform AWS provider 4.x. All new development will only support the new AWS Terraform provider. We keep a branch `terraform-aws-provider-3` to witch we welcome backports to AWS Terraform 3.x provider. Besides reviewing PR's we will do not any active checking on maintenance on this branch. We strongly advise to update your deployment to the new provider version. For more details about upgrading see the [upgrade guide](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/version-4-upgrade). * By default, AWS metadata service ((IMDSv2)[https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html]) is enabled and required for both the agent instance and the docker machine instance. For docker machine this require the GitLab managed docker machines distribution is used. Which the module usages by default. diff --git a/README.md b/README.md index 42d1db7cd..19333ac2f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This [Terraform](https://www.terraform.io/) modules creates a [GitLab CI runner](https://docs.gitlab.com/runner/). A blog post describes the original version of the the runner. See the post at [040code](https://040code.github.io/2017/12/09/runners-on-the-spot/). The original setup of the module is based on the blog post: [Auto scale GitLab CI runners and save 90% on EC2 costs](https://about.gitlab.com/2017/11/23/autoscale-ci-runners/). -> BREAKING CHANGE: The module is upgraded to Terraform AWS provider 4.x. All new development will only support the new AWS Terraform provider. We keep a branch `terraform-aws-provider-3` to witch we welcome backports to AWS Terraform 3.x provider. Besides reviewing PR's we will do not any active checking on maintance on this branch. We strongly advise to update your deployment to the new provider version. For more details about upgrading see the [upgrade guide](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/version-4-upgrade). +> BREAKING CHANGE: The module is upgraded to Terraform AWS provider 4.x. All new development will only support the new AWS Terraform provider. We keep a branch `terraform-aws-provider-3` to witch we welcome backports to AWS Terraform 3.x provider. Besides reviewing PR's we will do not any active checking on maintenance on this branch. We strongly advise to update your deployment to the new provider version. For more details about upgrading see the [upgrade guide](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/version-4-upgrade). > BREAKING CHANGE: By default AWS metadata service ((IMDSv2)[https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html]) is enabled and required for both the agent instance and the docker machine instance. For docker machine this require the GitLab managed docker machines distribution is used. Which the module usages by default. diff --git a/examples/runner-certificates/README.md b/examples/runner-certificates/README.md new file mode 100644 index 000000000..a0f2ea03b --- /dev/null +++ b/examples/runner-certificates/README.md @@ -0,0 +1,120 @@ +# Example - Spot Runner - Private subnet + +In this scenario the runner agent is running on a single EC2 node. + +The example is intended to show how the runner can be configured for self-hosted Gitlab environments with certificates signed by a custom CA. + +> This currently only works with the `docker` executor. Support for the `docker+machine` executor is not yet implemented. Contributions are welcome. + +## Prerequisites + +The terraform version is managed using [tfenv](https://github.com/Zordrak/tfenv). If you are not using `tfenv` please check `.terraform-version` for the tested version. + +Before configuring certificates, it is important to review the [Gitlab documentation](https://docs.gitlab.com/runner/configuration/tls-self-signed.html). + +In particular, note the following docker images are involved: + +> - The **Runner helper image**, which is used to handle Git, artifacts, and cache operations. In this scenario, the user only needs to make a certificate file available at a specific location (for example, /etc/gitlab-runner/certs/ca.crt), and the Docker container will automatically install it for the user. + +> - The **user image**, which is used to run the user script. In this scenario, the user must take ownership regarding how to install a certificate, since this is highly dependent on the image itself, and the Runner has no way of knowing how to install a certificate in each possible scenario. + +### Certificates for the runner-helper image + +The Gitlab **runner-helper image** needs to communicate with your Gitlab instance. + +Create a PEM-encoded `.crt` file containing the public certificate of your Gitlab server instance. + +```hcl +module { + ... + # Public cert of my companys gitlab instance + runners_gitlab_certificate = file("${path.module}/my_gitlab_instance_cert.crt") + ... +} +``` + +Add your CA and intermediary certs to a second PEM-encoded `.crt` file. +```hcl +module { + ... + # Other public certs relating to my company. + runners_ca_certificate = file("${path.module}/my_company_ca_cert_bundle.crt") + ... +} +``` + +### Certificates for user images + +For **user images**, you must: + +1. Mount the certificates from the EC2 host into all user images. + +The runner module can be configured to do this step. Configure the module like so: + +```terraform +module { + # ... + + # Mount EC2 host certs in docker so all user docker images can reference them. + runners_additional_volumes = ["/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"] + + # ... +} +``` + +2. Trust the certificates from within the user image. + +Each user image will need to execute commands to copy the certificates into the correct place and trust them. + +The below examples some ways to do this, assuming user images with the Ubuntu OS or similar. +For Alpine OS user images, the specific commands may differ. + +**Option 1:** Build a custom user image and update your `Dockerfile`: +```docker + FROM python:3 # Some base image + + RUN apt-get -y update + RUN apt-get -y upgrade + + RUN apt-get install -y ca-certificates + RUN cp /etc/gitlab-runner/certs/* /usr/local/share/ca-certificates/ + RUN update-ca-certificates + ... +``` + +**Option 2:** Add a section to each pipeline using `before_script`: + +This change would need to be added to every pipeline file which requires certificates. +It could be customised depending on the OS of the pipeline user image. + +```yaml +default: + before_script: + # Install certificates into user image + - apt-get install -y ca-certificates + - cp /etc/gitlab-runner/certs/* /usr/local/share/ca-certificates/ + - update-ca-certificates +``` + +**Option 3:** Add the script from Option 2 into `runners_pre_build_script` variable: + +This avoids maintaining the script in each pipeline file, but expects that all user images use the same OS. + +```terraform +module { + # ... + + runners_pre_build_script = < + \ No newline at end of file diff --git a/examples/runner-certificates/main.tf b/examples/runner-certificates/main.tf new file mode 100644 index 000000000..d652c1328 --- /dev/null +++ b/examples/runner-certificates/main.tf @@ -0,0 +1,73 @@ +data "aws_availability_zones" "available" { + state = "available" +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.70" + + name = "vpc-${var.environment}" + cidr = "10.1.0.0/16" + + azs = [data.aws_availability_zones.available.names[0]] + public_subnets = ["10.1.101.0/24"] + enable_s3_endpoint = true + map_public_ip_on_launch = false + + tags = { + Environment = var.environment + } +} + +module "runner" { + source = "../../" + + ############################################### + # General + ############################################### + + runners_name = var.runner_name + runners_gitlab_url = var.gitlab_url + + runners_executor = "docker" + + aws_region = var.aws_region + environment = var.environment + + ############################################### + # Certificates + ############################################### + + # Public cert of my companys gitlab instance + runners_gitlab_certificate = file("${path.module}/my_gitlab_instance_cert.crt") + + # Other public certs relating to my company. + runners_ca_certificate = file("${path.module}/my_company_ca_cert_bundle.crt") + + # Mount EC2 host certs in docker so all user docker images can reference them. + # Each user image will need to do: + # cp /etc/gitlab-runner/certs/* /usr/local/share/ca-certificates/ + # update-ca-certificates + # Or similar OS-dependent commands. The above are an example for Ubuntu. + runners_additional_volumes = ["/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"] + + ############################################### + # Registration + ############################################### + + gitlab_runner_registration_config = { + registration_token = var.registration_token + tag_list = "docker_runner" + description = "runner docker - auto" + locked_to_project = "true" + run_untagged = "false" + maximum_timeout = "3600" + } + + ############################################### + # Network + ############################################### + vpc_id = module.vpc.vpc_id + subnet_id = element(module.vpc.public_subnets, 0) + +} \ No newline at end of file diff --git a/examples/runner-certificates/my_company_ca_cert_bundle.crt b/examples/runner-certificates/my_company_ca_cert_bundle.crt new file mode 100644 index 000000000..87f78bc03 --- /dev/null +++ b/examples/runner-certificates/my_company_ca_cert_bundle.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +Put your various CA / intermediary public certificates and other general public certs in this file. +-----END CERTIFICATE----- + +-----BEGIN CERTIFICATE----- +My certificate authority public cert 1 +-----END CERTIFICATE----- + +-----BEGIN CERTIFICATE----- +My intermediary public cert 1 +-----END CERTIFICATE----- + +-----BEGIN CERTIFICATE----- +Some other instance I need to get to public cert, eg my own docker registry server +-----END CERTIFICATE----- \ No newline at end of file diff --git a/examples/runner-certificates/my_gitlab_instance_cert.crt b/examples/runner-certificates/my_gitlab_instance_cert.crt new file mode 100644 index 000000000..7cc0336e2 --- /dev/null +++ b/examples/runner-certificates/my_gitlab_instance_cert.crt @@ -0,0 +1,4 @@ +-----BEGIN CERTIFICATE----- +Put your Gitlab instance public certificate in this file. +Intermediary and CA public certificates are not required. +-----END CERTIFICATE----- \ No newline at end of file diff --git a/examples/runner-certificates/providers.tf b/examples/runner-certificates/providers.tf new file mode 100644 index 000000000..161b858de --- /dev/null +++ b/examples/runner-certificates/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region +} + +provider "local" {} + +provider "null" {} + +provider "tls" {} + +provider "random" {} diff --git a/examples/runner-certificates/variables.tf b/examples/runner-certificates/variables.tf new file mode 100644 index 000000000..fd6fe3903 --- /dev/null +++ b/examples/runner-certificates/variables.tf @@ -0,0 +1,29 @@ +variable "aws_region" { + description = "AWS region." + type = string + default = "eu-west-1" +} + +variable "environment" { + description = "A name that identifies the environment, will used as prefix and for tagging." + default = "runners-docker" + type = string +} + +variable "runner_name" { + description = "Name of the runner, will be used in the runner config.toml" + type = string + default = "docker" +} + +variable "gitlab_url" { + description = "URL of the gitlab instance to connect to." + type = string + default = "https://gitlab.com" +} + +variable "registration_token" { + description = "Gitlab runner registration token" + type = string + default = "something" +} \ No newline at end of file diff --git a/examples/runner-certificates/versions.tf b/examples/runner-certificates/versions.tf new file mode 100644 index 000000000..1f30b55ce --- /dev/null +++ b/examples/runner-certificates/versions.tf @@ -0,0 +1,26 @@ + +terraform { + required_version = ">= 1" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.7" + } + local = { + source = "hashicorp/local" + version = "~> 2" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + tls = { + source = "hashicorp/tls" + version = "~> 3" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} \ No newline at end of file diff --git a/locals.tf b/locals.tf index 627134c35..483b2db89 100644 --- a/locals.tf +++ b/locals.tf @@ -1,4 +1,42 @@ locals { + # Manage certificates + pre_install_gitlab_certificate = ( + length(var.runners_gitlab_certificate) > 0 + ? <<-EOT + mkdir -p /etc/gitlab-runner/certs/ + cat <<- EOF > /etc/gitlab-runner/certs/gitlab.crt + ${var.runners_gitlab_certificate} + EOF + EOT + : "" + ) + pre_install_ca_certificate = ( + length(var.runners_ca_certificate) > 0 + ? <<-EOT + mkdir -p /etc/gitlab-runner/certs/ + cat <<- EOF > /etc/gitlab-runner/certs/ca.crt + ${var.runners_ca_certificate} + EOF + EOT + : "" + ) + pre_install_certificates_end = <<-EOT + chmod 600 /etc/gitlab-runner/certs/*.crt + chmod -R a+r /etc/gitlab-runner + cp /etc/gitlab-runner/certs/*.crt /etc/pki/ca-trust/source/anchors + update-ca-trust extract + EOT + pre_install_certificates = ( + # If either (or both) _certificate variables are specified + length(var.runners_gitlab_certificate) + length(var.runners_ca_certificate) > 0 + ? join("\n", [ + local.pre_install_gitlab_certificate, + local.pre_install_ca_certificate, + local.pre_install_certificates_end + ]) + : "" + ) + # Determine IAM role for runner instance aws_iam_role_instance_name = coalesce( var.runner_iam_role_name, diff --git a/main.tf b/main.tf index 679a0f02a..dc537dd27 100644 --- a/main.tf +++ b/main.tf @@ -47,7 +47,7 @@ locals { file_yum_update = file("${path.module}/template/yum_update.tpl") template_eip = templatefile("${path.module}/template/eip.tpl", { - eip = join(",", aws_eip.gitlab_runner.*.public_ip) + eip = join(",", aws_eip.gitlab_runner[*].public_ip) }) template_gitlab_runner = templatefile("${path.module}/template/gitlab-runner.tpl", @@ -58,6 +58,8 @@ locals { runners_config = local.template_runner_config runners_executor = var.runners_executor runners_install_amazon_ecr_credential_helper = var.runners_install_amazon_ecr_credential_helper + curl_cacert = length(var.runners_gitlab_certificate) > 0 ? "--cacert /etc/gitlab-runner/certs/gitlab.crt" : "" + pre_install_certificates = local.pre_install_certificates pre_install = var.userdata_pre_install post_install = var.userdata_post_install runners_gitlab_url = var.runners_gitlab_url @@ -66,7 +68,7 @@ locals { secure_parameter_store_runner_sentry_dsn = local.secure_parameter_store_runner_sentry_dsn secure_parameter_store_region = var.aws_region gitlab_runner_registration_token = var.gitlab_runner_registration_config["registration_token"] - giltab_runner_description = var.gitlab_runner_registration_config["description"] + gitlab_runner_description = var.gitlab_runner_registration_config["description"] gitlab_runner_tag_list = var.gitlab_runner_registration_config["tag_list"] gitlab_runner_locked_to_project = var.gitlab_runner_registration_config["locked_to_project"] gitlab_runner_run_untagged = var.gitlab_runner_registration_config["run_untagged"] @@ -80,6 +82,7 @@ locals { aws_region = var.aws_region gitlab_url = var.runners_gitlab_url gitlab_clone_url = var.runners_clone_url + tls_ca_file = length(var.runners_gitlab_certificate) > 0 ? "tls-ca-file=\"/etc/gitlab-runner/certs/gitlab.crt\"" : "" runners_extra_hosts = var.runners_extra_hosts runners_vpc_id = var.vpc_id runners_subnet_id = length(var.subnet_id) > 0 ? var.subnet_id : var.subnet_id_runners diff --git a/outputs.tf b/outputs.tf index 644d5398e..9a9442047 100644 --- a/outputs.tf +++ b/outputs.tf @@ -5,12 +5,12 @@ output "runner_as_group_name" { output "runner_cache_bucket_arn" { description = "ARN of the S3 for the build cache." - value = element(concat(module.cache.*.arn, [""]), 0) + value = element(concat(module.cache[*].arn, [""]), 0) } output "runner_cache_bucket_name" { description = "Name of the S3 for the build cache." - value = element(concat(module.cache.*.bucket, [""]), 0) + value = element(concat(module.cache[*].bucket, [""]), 0) } output "runner_agent_role_arn" { @@ -45,7 +45,7 @@ output "runner_sg_id" { output "runner_eip" { description = "EIP of the Gitlab Runner" - value = element(concat(aws_eip.gitlab_runner.*.public_ip, [""]), 0) + value = element(concat(aws_eip.gitlab_runner[*].public_ip, [""]), 0) } output "runner_launch_template_name" { diff --git a/tags.tf b/tags.tf index 290fb7062..ba03d4f08 100644 --- a/tags.tf +++ b/tags.tf @@ -30,10 +30,6 @@ locals { # remove the `Name` tag if docker+machine adds one to avoid a failure due to a duplicate `Name` tag runner_tags = local.docker_machine_adds_name_tag ? { for k, v in local.runner_tags_merged : k => v if k != "Name" } : local.runner_tags_merged - tags_string = join(",", flatten([ - for key in keys(local.tags) : [key, lookup(local.tags, key)] - ])) - runner_tags_string = join(",", flatten([ for key in keys(local.runner_tags) : [key, lookup(local.runner_tags, key)] ])) diff --git a/template/gitlab-runner.tpl b/template/gitlab-runner.tpl index ea05e3094..0be20949e 100644 --- a/template/gitlab-runner.tpl +++ b/template/gitlab-runner.tpl @@ -16,22 +16,24 @@ EOF sed -i.bak s/__PARENT_TAG__/`echo $PARENT_TAG`/g /etc/gitlab-runner/config.toml +${pre_install_certificates} + # fetch Runner token from SSM and validate it token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters | .[0] | .Value") valid_token=true if [[ `echo $token` != "null" ]] then - valid_token_response=$(curl -s -o /dev/null -w "%%{response_code}" --request POST -L "${runners_gitlab_url}/api/v4/runners/verify" --form "token=$token" ) + valid_token_response=$(curl -s -o /dev/null -w "%%{response_code}" ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners/verify" --form "token=$token" ) [[ `echo $valid_token_response` != "200" ]] && valid_token=false fi if [[ `echo ${runners_token}` == "__REPLACED_BY_USER_DATA__" && `echo $token` == "null" ]] || [[ `echo $valid_token` == "false" ]] then - token=$(curl --request POST -L "${runners_gitlab_url}/api/v4/runners" \ + token=$(curl ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners" \ --form "token=${gitlab_runner_registration_token}" \ --form "tag_list=${gitlab_runner_tag_list}" \ - --form "description=${giltab_runner_description}" \ + --form "description=${gitlab_runner_description}" \ --form "locked=${gitlab_runner_locked_to_project}" \ --form "run_untagged=${gitlab_runner_run_untagged}" \ --form "maximum_timeout=${gitlab_runner_maximum_timeout}" \ @@ -122,7 +124,7 @@ start() { stop() { logger "Removing Gitlab Runner Token" aws ssm put-parameter --overwrite --type SecureString --name "${secure_parameter_store_runner_token_key}" --region "${secure_parameter_store_region}" --value="null" 2>&1 | logger & - curl -sS --request DELETE "${runners_gitlab_url}/api/v4/runners" --form "token=$token" 2>&1 | logger & + curl -sS ${curl_cacert} --request DELETE "${runners_gitlab_url}/api/v4/runners" --form "token=$token" 2>&1 | logger & retval=\$? rm -f \$lockfile return \$retval diff --git a/template/runner-config.tpl b/template/runner-config.tpl index 85ece8aa8..d4a127473 100644 --- a/template/runner-config.tpl +++ b/template/runner-config.tpl @@ -7,6 +7,7 @@ listen_address = "${prometheus_listen_address}" [[runners]] name = "${runners_name}" url = "${gitlab_url}" + ${tls_ca_file} clone_url = "${gitlab_clone_url}" token = "${runners_token}" executor = "${runners_executor}" diff --git a/variables.tf b/variables.tf index a8f019dc6..4fb3f2e5a 100644 --- a/variables.tf +++ b/variables.tf @@ -854,3 +854,15 @@ variable "runner_yum_update" { type = bool default = true } + +variable "runners_gitlab_certificate" { + description = "Certificate of the GitLab instance to connect to. Example: `file(\"$${path.module}/my-gitlab.crt\")`" + type = string + default = "" +} + +variable "runners_ca_certificate" { + description = "Trusted CA certificate bundle. Example: `file(\"$${path.module}/ca.crt\")`" + type = string + default = "" +} \ No newline at end of file From 2d5db6f9c3bf07357e812e7fc8d0277650487871 Mon Sep 17 00:00:00 2001 From: kayma Date: Thu, 2 Mar 2023 11:58:43 +0100 Subject: [PATCH 2/3] to make the PR check happy From 635165a2017b25739da1c5bf9ee7c645292e61cd Mon Sep 17 00:00:00 2001 From: kayma Date: Thu, 2 Mar 2023 14:55:25 +0100 Subject: [PATCH 3/3] fix merge --- template/gitlab-runner.tpl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/template/gitlab-runner.tpl b/template/gitlab-runner.tpl index f9bb40999..3b0fd5ede 100644 --- a/template/gitlab-runner.tpl +++ b/template/gitlab-runner.tpl @@ -28,13 +28,8 @@ token=$(aws ssm get-parameters --names "${secure_parameter_store_runner_token_ke valid_token=true if [[ "$token" != "null" ]] then -<<<<<<< HEAD valid_token_response=$(curl -s -o /dev/null -w "%%{response_code}" ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners/verify" --form "token=$token" ) - [[ `echo $valid_token_response` != "200" ]] && valid_token=false -======= - valid_token_response=$(curl -s -o /dev/null -w "%%{response_code}" --request POST -L "${runners_gitlab_url}/api/v4/runners/verify" --form "token=$token" ) [[ "$valid_token_response" != "200" ]] && valid_token=false ->>>>>>> upstream/main fi if [[ "${runners_token}" == "__REPLACED_BY_USER_DATA__" && "$token" == "null" ]] || [[ "$valid_token" == "false" ]]