feat!: allow to set all docker options for the Executor (#511)

Adds a new variable `runners_docker_options` which holds all values for
the `[runners.docker]` section and makes the single variables
- `runners_image`
- `runners_privileged`
- `runners_disable_cache`
- `runners_additional_volumes`
- `runners_shm_size`
- `runners_docker_runtime`
- `runners_helper_image`
- `runners_pull_policy`

obsolete.

Yes, as the minimum Terraform version is 1.3.0 to support optional block
variables with defaults.

A migration script is provided to restructure the variables. See
`/migrations/migrate-to-7-0-0.sh`. Attention Mac users: The script will
not work out of the box as the `sed` implementation is different. Use a
Docker container with Alpine or Ubuntu to run the script.

```hcl
module "gitlab_ci_runner" {
  ...
  runners_docker_options {
    # set whatever is necessary
  }
```

- [x] Use current configuration and ensure that the `config.toml`
remains unchanged
- [x] Set all new block variables and ensure that the `config.toml` is
valid (use `gitlab-runner verify)
- [x] Check that the default settings with Terraform 1.3 work as
expected
- [x] Verify all docker settings against the documentation to ensure
correct names

The runner starts in both cases and is available in Gitlab. No example
tested but used our active configuration at Hapag-Lloyd.

---------

Co-authored-by: Tyrone Meijn <[email protected]>

Author: kayman-mk

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        terraform: [1.3.9, latest]
+        terraform: [ 1.3.9, latest ]
         example:
           [
             "runner-default",

examples/runner-certificates/main.tf

@@ -51,7 +51,12 @@ module "runner" {
   # cp /etc/gitlab-runner/certs/* /usr/local/share/ca-certificates/
   # update-ca-certificates
   # Or similar OS-dependent commands. The above are an example for Ubuntu.
-  runners_additional_volumes = ["/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"]
+  runners_docker_options = {
+    volumes = [
+      "/cache",
+      "/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"
+    ]
+  }
 
   ###############################################
   # Registration

examples/runner-default/main.tf

@@ -79,9 +79,6 @@ module "runner" {
     "tf-aws-gitlab-runner:instancelifecycle" = "spot:yes"
   }
 
-  runners_privileged         = "true"
-  runners_additional_volumes = ["/certs/client"]
-
   runners_volumes_tmpfs = [
     {
       volume  = "/var/opt/cache",
@@ -106,6 +103,11 @@ module "runner" {
     }
   ]
 
+  runners_docker_options = {
+    privileged = "true"
+    volumes    = ["/cache", "/certs/client"]
+  }
+
   runners_pre_build_script = <<EOT
   '''
   echo 'multiline 1'

examples/runner-multi-region/main.tf

@@ -38,8 +38,10 @@ module "runner_main_region" {
   runners_gitlab_url       = var.gitlab_url
   runners_environment_vars = ["KEY=Value", "FOO=bar"]
 
-  runners_privileged         = "false"
-  runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
+  runners_docker_options = {
+    privileged = "false"
+    volumes    = ["/var/run/docker.sock:/var/run/docker.sock"]
+  }
 
   gitlab_runner_registration_config = {
     registration_token = var.registration_token
@@ -108,8 +110,10 @@ module "runner_alternate_region" {
   runners_gitlab_url       = var.gitlab_url
   runners_environment_vars = ["KEY=Value", "FOO=bar"]
 
-  runners_privileged         = "false"
-  runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
+  runners_docker_options = {
+    privileged = "false"
+    volumes    = ["/var/run/docker.sock:/var/run/docker.sock"]
+  }
 
   gitlab_runner_registration_config = {
     registration_token = var.registration_token

examples/runner-public/main.tf

@@ -43,8 +43,10 @@ module "runner" {
   runners_gitlab_url       = var.gitlab_url
   runners_environment_vars = ["KEY=Value", "FOO=bar"]
 
-  runners_privileged         = "false"
-  runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
+  runners_docker_options = {
+    privileged = "false"
+    volumes    = ["/var/run/docker.sock:/var/run/docker.sock"]
+  }
 
   gitlab_runner_registration_config = {
     registration_token = var.registration_token

locals.tf

@@ -52,6 +52,16 @@ locals {
 
   runners_docker_registry_mirror_option = var.runners_docker_registry_mirror == "" ? [] : ["engine-registry-mirror=${var.runners_docker_registry_mirror}"]
 
+  runners_docker_options_toml = templatefile("${path.module}/template/runners_docker_options.tftpl", {
+    options = merge({
+      for key, value in var.runners_docker_options : key => value if value != null && key != "volumes"
+      }, {
+      volumes = local.runners_volumes
+    })
+    }
+  )
+
+
   # Ensure max builds is optional
   runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds)
 
@@ -64,17 +74,13 @@ locals {
   name_sg                    = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
   name_iam_objects           = lookup(var.overrides, "name_iam_objects", "") == "" ? local.tags["Name"] : var.overrides["name_iam_objects"]
 
-  runners_additional_volumes = <<-EOT
-  %{~if var.runners_add_dind_volumes~},"/certs/client", "/builds", "/var/run/docker.sock:/var/run/docker.sock"%{endif~}%{~for volume in var.runners_additional_volumes~},"${volume}"%{endfor~}
-  EOT
+  runners_volumes = concat(var.runners_docker_options.volumes, var.runners_add_dind_volumes ? ["/certs/client", "/builds", "/var/run/docker.sock:/var/run/docker.sock"] : [])
 
   runners_docker_services = templatefile("${path.module}/template/runners_docker_services.tftpl", {
     runners_docker_services = var.runners_docker_services
     }
   )
 
-  runners_pull_policies = "[\"${join("\",\"", var.runners_pull_policies)}\"]"
-
   /* determines if the docker machine executable adds the Name tag automatically (versions >= 0.16.2) */
   # make sure to skip pre-release stuff in the semver by ignoring everything after "-"
   docker_machine_version_used          = split(".", split("-", var.docker_machine_version)[0])

main.tf

@@ -99,7 +99,6 @@ locals {
       gitlab_url                        = var.runners_gitlab_url
       gitlab_clone_url                  = var.runners_clone_url
       tls_ca_file                       = length(var.runners_gitlab_certificate) > 0 ? "tls-ca-file=\"/etc/gitlab-runner/certs/gitlab.crt\"" : ""
-      runners_extra_hosts               = var.runners_extra_hosts
       runners_vpc_id                    = var.vpc_id
       runners_subnet_id                 = var.subnet_id
       runners_subnet_ids                = length(var.fleet_executor_subnet_ids) > 0 ? var.fleet_executor_subnet_ids : [var.subnet_id]
@@ -112,7 +111,6 @@ locals {
       runners_monitoring                = var.runners_monitoring
       runners_ebs_optimized             = var.runners_ebs_optimized
       runners_instance_profile          = var.runners_executor == "docker+machine" ? aws_iam_instance_profile.docker_machine[0].name : ""
-      runners_additional_volumes        = local.runners_additional_volumes
       docker_machine_options            = length(local.docker_machine_options_string) == 1 ? "" : local.docker_machine_options_string
       docker_machine_name               = format("%s-%s", local.runner_tags_merged["Name"], "%s") # %s is always needed
       runners_name                      = var.runners_name
@@ -122,13 +120,6 @@ locals {
       runners_executor                  = var.runners_executor
       runners_limit                     = var.runners_limit
       runners_concurrent                = var.runners_concurrent
-      runners_image                     = var.runners_image
-      runners_privileged                = var.runners_privileged
-      runners_disable_cache             = var.runners_disable_cache
-      runners_docker_runtime            = var.runners_docker_runtime
-      runners_helper_image              = var.runners_helper_image
-      runners_shm_size                  = var.runners_shm_size
-      runners_pull_policies             = local.runners_pull_policies
       runners_idle_count                = var.runners_idle_count
       runners_idle_time                 = var.runners_idle_time
       runners_max_builds                = local.runners_max_builds_string
@@ -146,6 +137,7 @@ locals {
       runners_request_concurrency       = var.runners_request_concurrency
       runners_output_limit              = var.runners_output_limit
       runners_check_interval            = var.runners_check_interval
+      runners_docker_options            = local.runners_docker_options_toml
       runners_volumes_tmpfs             = join("\n", [for v in var.runners_volumes_tmpfs : format("\"%s\" = \"%s\"", v.volume, v.options)])
       runners_services_volumes_tmpfs    = join("\n", [for v in var.runners_services_volumes_tmpfs : format("\"%s\" = \"%s\"", v.volume, v.options)])
       runners_docker_services           = local.runners_docker_services

migrations/migrate-to-7-0-0.sh

@@ -28,3 +28,54 @@ sed -i '/asg_terminate_lifecycle_lambda_timeout/d' "$converted_file"
 #  PR #711 feat!: refactor Docker Machine autoscaling options 
 #
 sed -i 's/runners_machine_autoscaling/runners_machine_autoscaling_options/g' "$converted_file"
+
+#
+# PR #710 chore!: remove old variable `runners_pull_policy`
+#
+sed -i '/runners_pull_policy/d' "$converted_file"
+
+#
+# PR #511 feat!: allow to set all docker options for the Executor
+#
+extracted_variables=$(grep -E '(runners_docker_runtime|runners_helper_image|runners_shm_size|runners_shm_size|runners_extra_hosts|runners_disable_cache|runners_image|runners_privileged)' "$converted_file")
+
+sed -i '/runners_image/d' "$converted_file"
+sed -i '/runners_privileged/d' "$converted_file"
+sed -i '/runners_disable_cache/d' "$converted_file"
+sed -i '/runners_extra_hosts/d' "$converted_file"
+sed -i '/runners_shm_size/d' "$converted_file"
+sed -i '/runners_docker_runtime/d' "$converted_file"
+sed -i '/runners_helper_image/d' "$converted_file"
+
+# content to be added to `volumes`
+volumes=$(grep "runners_additional_volumes" "$converted_file" | cut -d '=' -f 2 | tr -d '[]')
+
+if [ -n "$volumes" ]; then
+  extracted_variables="$extracted_variables
+    volumes = [\"/cache\", $volumes]"
+fi
+
+sed -i '/runners_additional_volumes/d' "$converted_file"
+
+
+# rename the variables
+extracted_variables=$(echo "$extracted_variables" | \
+                      sed 's/runners_image/image/g' | \
+                      sed 's/runners_privileged/privileged/g' | \
+                      sed 's/runners_disable_cache/disable_cache/g' | \
+                      sed 's/runners_extra_hosts/extra_hosts/g' | \
+                      sed 's/runners_shm_size/shm_size/g' | \
+                      sed 's/runners_docker_runtime/runtime/g' | \
+                      sed 's/runners_helper_image/helper_image/g'
+                    )
+
+# add new block runners_docker_options at the end
+echo "$(head -n -1 "$converted_file")
+runners_docker_options {
+  $extracted_variables
+}
+}" > x
+
+mv x "$converted_file"
+
+echo "Module call converted. Output: $converted_file"

template/runner-config.tftpl

@@ -18,23 +18,17 @@ listen_address = "${prometheus_listen_address}"
   request_concurrency = ${runners_request_concurrency}
   output_limit = ${runners_output_limit}
   limit = ${runners_limit}
-  [runners.docker]
-    tls_verify = false
-    image = "${runners_image}"
-    privileged = ${runners_privileged}
-    disable_cache = ${runners_disable_cache}
-    volumes = ["/cache"${runners_additional_volumes}]
-    extra_hosts = ${jsonencode(runners_extra_hosts)}
-    shm_size = ${runners_shm_size}
-    pull_policy = ${runners_pull_policies}
-    runtime = "${runners_docker_runtime}"
-    helper_image = "${runners_helper_image}"
-    wait_for_services_timeout = ${runners_wait_for_services_timeout}
-    ${runners_docker_services}
+
+  ${runners_docker_options}
+
+  ${runners_docker_services}
+
   [runners.docker.tmpfs]
     ${runners_volumes_tmpfs}
+
   [runners.docker.services_tmpfs]
     ${runners_services_volumes_tmpfs}
+
   [runners.cache]
     Type = "s3"
     Shared = ${shared_cache}
@@ -44,6 +38,7 @@ listen_address = "${prometheus_listen_address}"
       BucketName = "${bucket_name}"
       BucketLocation = "${aws_region}"
       Insecure = false
+
   [runners.machine]
     IdleCount = ${runners_idle_count}
     IdleTime = ${runners_idle_time}

template/runners_docker_options.tftpl

@@ -0,0 +1,4 @@
+  [runners.docker]
+%{ for key, value in options ~}
+    ${key} = ${jsonencode(value)}
+%{ endfor ~}

variables.tf

@@ -206,64 +206,77 @@ variable "runners_max_builds" {
   default     = 0
 }
 
-variable "runners_image" {
-  description = "Image to run builds, will be used in the runner config.toml"
-  type        = string
-  default     = "docker:18.03.1-ce"
-}
-
-variable "runners_privileged" {
-  description = "Runners will run in privileged mode, will be used in the runner config.toml"
-  type        = bool
-  default     = true
-}
-
-variable "runners_disable_cache" {
-  description = "Runners will not use local cache, will be used in the runner config.toml"
-  type        = bool
-  default     = false
-}
-
 variable "runners_add_dind_volumes" {
   description = "Add certificates and docker.sock to the volumes to support docker-in-docker (dind)"
   type        = bool
   default     = false
 }
 
-variable "runners_additional_volumes" {
-  description = "Additional volumes that will be used in the runner config.toml, e.g Docker socket"
-  type        = list(any)
-  default     = []
-}
-
-variable "runners_extra_hosts" {
-  description = "Extra hosts that will be used in the runner config.toml, e.g other-host:127.0.0.1"
-  type        = list(any)
-  default     = []
-}
-
-variable "runners_shm_size" {
-  description = "shm_size for the runners, will be used in the runner config.toml"
-  type        = number
-  default     = 0
-}
-
-variable "runners_docker_runtime" {
-  description = "docker runtime for runners, will be used in the runner config.toml"
-  type        = string
-  default     = ""
-}
+variable "runners_docker_options" {
+  description = <<EOT
+    Options added to the [runners.docker] section of config.toml to configure the Docker container of the Executors. For
+    details check https://docs.gitlab.com/runner/configuration/advanced-configuration.html
+
+    Default values if the option is not given:
+      disable_cache = "false"
+      image         = "docker:18.03.1-ce"
+      privileged    = "true"
+      pull_policy   = "always"
+      shm_size      = 0
+      tls_verify    = "false"
+      volumes       = "/cache"
+  EOT
 
-variable "runners_helper_image" {
-  description = "Overrides the default helper image used to clone repos and upload artifacts, will be used in the runner config.toml"
-  type        = string
-  default     = ""
-}
+  type = object({
+    allowed_images               = optional(list(string))
+    allowed_pull_policies        = optional(list(string))
+    allowed_services             = optional(list(string))
+    cache_dir                    = optional(string)
+    cap_add                      = optional(list(string))
+    cap_drop                     = optional(list(string))
+    container_labels             = optional(list(string))
+    cpuset_cpus                  = optional(string)
+    cpu_shares                   = optional(number)
+    cpus                         = optional(string)
+    devices                      = optional(list(string))
+    device_cgroup_rules          = optional(list(string))
+    disable_cache                = optional(bool, false)
+    disable_entrypoint_overwrite = optional(bool)
+    dns                          = optional(list(string))
+    dns_search                   = optional(list(string))
+    extra_hosts                  = optional(list(string))
+    gpus                         = optional(string)
+    helper_image                 = optional(string)
+    helper_image_flavor          = optional(string)
+    host                         = optional(string)
+    hostname                     = optional(string)
+    image                        = optional(string, "docker:18.03.1-ce")
+    isolation                    = optional(string)
+    links                        = optional(list(string))
+    mac_address                  = optional(string)
+    memory                       = optional(string)
+    memory_swap                  = optional(string)
+    memory_reservation           = optional(string)
+    network_mode                 = optional(string)
+    oom_kill_disable             = optional(bool)
+    oom_score_adjust             = optional(number)
+    privileged                   = optional(bool, true)
+    pull_policies                = optional(list(string), ["always"])
+    runtime                      = optional(string)
+    security_opt                 = optional(list(string))
+    shm_size                     = optional(number, 0)
+    sysctls                      = optional(list(string))
+    tls_cert_path                = optional(string)
+    tls_verify                   = optional(bool, false)
+    user                         = optional(string)
+    userns_mode                  = optional(string)
+    volumes                      = optional(list(string), ["/cache"])
+    volumes_from                 = optional(list(string))
+    volume_driver                = optional(string)
+    wait_for_services_timeout    = optional(number)
+  })
 
-variable "runners_pull_policies" {
-  description = "pull policies for the runners, will be used in the runner config.toml, for Gitlab Runner >= 13.8, see https://docs.gitlab.com/runner/executors/docker.html#using-multiple-pull-policies "
-  type        = list(string)
-  default     = ["always"]
+  default = null
 }
 
 variable "runners_monitoring" {

versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 1"
+  required_version = ">= 1.3"
 
   required_providers {
     aws = {