Skip to content

Commit 7e05787

Browse files
kayman-mktmeijn
kayman-mk
and
tmeijn
committed
feat!: allow to set all docker options for the Executor (#511)
## Description Adds a new variable `runners_docker_options` which holds all values for the `[runners.docker]` section and makes the single variables - `runners_image` - `runners_privileged` - `runners_disable_cache` - `runners_additional_volumes` - `runners_shm_size` - `runners_docker_runtime` - `runners_helper_image` - `runners_pull_policy` obsolete. ## Migrations required Yes, as the minimum Terraform version is 1.3.0 to support optional block variables with defaults. A migration script is provided to restructure the variables. See `/migrations/migrate-to-7-0-0.sh`. Attention Mac users: The script will not work out of the box as the `sed` implementation is different. Use a Docker container with Alpine or Ubuntu to run the script. ```hcl module "gitlab_ci_runner" { ... runners_docker_options { # set whatever is necessary } ``` ## Verification - [x] Use current configuration and ensure that the `config.toml` remains unchanged - [x] Set all new block variables and ensure that the `config.toml` is valid (use `gitlab-runner verify) - [x] Check that the default settings with Terraform 1.3 work as expected - [x] Verify all docker settings against the documentation to ensure correct names The runner starts in both cases and is available in Gitlab. No example tested but used our active configuration at Hapag-Lloyd. --------- Co-authored-by: Tyrone Meijn <[email protected]>
1 parent 4128105 commit 7e05787

File tree

12 files changed

+163
-88
lines changed

12 files changed

+163
-88
lines changed

Diff for: ‎.github/workflows/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
strategy:
3232
fail-fast: false
3333
matrix:
34-
terraform: [1.3.9, latest]
34+
terraform: [ 1.3.9, latest ]
3535
example:
3636
[
3737
"runner-default",

Diff for: ‎examples/runner-certificates/main.tf

+6-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,12 @@ module "runner" {
5151
# cp /etc/gitlab-runner/certs/* /usr/local/share/ca-certificates/
5252
# update-ca-certificates
5353
# Or similar OS-dependent commands. The above are an example for Ubuntu.
54-
runners_additional_volumes = ["/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"]
54+
runners_docker_options = {
55+
volumes = [
56+
"/cache",
57+
"/etc/gitlab-runner/certs/:/etc/gitlab-runner/certs:ro"
58+
]
59+
}
5560

5661
###############################################
5762
# Registration

Diff for: ‎examples/runner-default/main.tf

+5-3
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,6 @@ module "runner" {
7979
"tf-aws-gitlab-runner:instancelifecycle" = "spot:yes"
8080
}
8181

82-
runners_privileged = "true"
83-
runners_additional_volumes = ["/certs/client"]
84-
8582
runners_volumes_tmpfs = [
8683
{
8784
volume = "/var/opt/cache",
@@ -106,6 +103,11 @@ module "runner" {
106103
}
107104
]
108105

106+
runners_docker_options = {
107+
privileged = "true"
108+
volumes = ["/cache", "/certs/client"]
109+
}
110+
109111
runners_pre_build_script = <<EOT
110112
'''
111113
echo 'multiline 1'

Diff for: ‎examples/runner-multi-region/main.tf

+8-4
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ module "runner_main_region" {
3838
runners_gitlab_url = var.gitlab_url
3939
runners_environment_vars = ["KEY=Value", "FOO=bar"]
4040

41-
runners_privileged = "false"
42-
runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
41+
runners_docker_options = {
42+
privileged = "false"
43+
volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
44+
}
4345

4446
gitlab_runner_registration_config = {
4547
registration_token = var.registration_token
@@ -108,8 +110,10 @@ module "runner_alternate_region" {
108110
runners_gitlab_url = var.gitlab_url
109111
runners_environment_vars = ["KEY=Value", "FOO=bar"]
110112

111-
runners_privileged = "false"
112-
runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
113+
runners_docker_options = {
114+
privileged = "false"
115+
volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
116+
}
113117

114118
gitlab_runner_registration_config = {
115119
registration_token = var.registration_token

Diff for: ‎examples/runner-public/main.tf

+4-2
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ module "runner" {
4343
runners_gitlab_url = var.gitlab_url
4444
runners_environment_vars = ["KEY=Value", "FOO=bar"]
4545

46-
runners_privileged = "false"
47-
runners_additional_volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
46+
runners_docker_options = {
47+
privileged = "false"
48+
volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
49+
}
4850

4951
gitlab_runner_registration_config = {
5052
registration_token = var.registration_token

Diff for: ‎locals.tf

+11-5
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,16 @@ locals {
5252

5353
runners_docker_registry_mirror_option = var.runners_docker_registry_mirror == "" ? [] : ["engine-registry-mirror=${var.runners_docker_registry_mirror}"]
5454

55+
runners_docker_options_toml = templatefile("${path.module}/template/runners_docker_options.tftpl", {
56+
options = merge({
57+
for key, value in var.runners_docker_options : key => value if value != null && key != "volumes"
58+
}, {
59+
volumes = local.runners_volumes
60+
})
61+
}
62+
)
63+
64+
5565
# Ensure max builds is optional
5666
runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds)
5767

@@ -64,17 +74,13 @@ locals {
6474
name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
6575
name_iam_objects = lookup(var.overrides, "name_iam_objects", "") == "" ? local.tags["Name"] : var.overrides["name_iam_objects"]
6676

67-
runners_additional_volumes = <<-EOT
68-
%{~if var.runners_add_dind_volumes~},"/certs/client", "/builds", "/var/run/docker.sock:/var/run/docker.sock"%{endif~}%{~for volume in var.runners_additional_volumes~},"${volume}"%{endfor~}
69-
EOT
77+
runners_volumes = concat(var.runners_docker_options.volumes, var.runners_add_dind_volumes ? ["/certs/client", "/builds", "/var/run/docker.sock:/var/run/docker.sock"] : [])
7078

7179
runners_docker_services = templatefile("${path.module}/template/runners_docker_services.tftpl", {
7280
runners_docker_services = var.runners_docker_services
7381
}
7482
)
7583

76-
runners_pull_policies = "[\"${join("\",\"", var.runners_pull_policies)}\"]"
77-
7884
/* determines if the docker machine executable adds the Name tag automatically (versions >= 0.16.2) */
7985
# make sure to skip pre-release stuff in the semver by ignoring everything after "-"
8086
docker_machine_version_used = split(".", split("-", var.docker_machine_version)[0])

Diff for: ‎main.tf

+1-9
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,6 @@ locals {
9595
gitlab_url = var.runners_gitlab_url
9696
gitlab_clone_url = var.runners_clone_url
9797
tls_ca_file = length(var.runners_gitlab_certificate) > 0 ? "tls-ca-file=\"/etc/gitlab-runner/certs/gitlab.crt\"" : ""
98-
runners_extra_hosts = var.runners_extra_hosts
9998
runners_vpc_id = var.vpc_id
10099
runners_subnet_id = var.subnet_id
101100
runners_subnet_ids = length(var.fleet_executor_subnet_ids) > 0 ? var.fleet_executor_subnet_ids : [var.subnet_id]
@@ -108,7 +107,6 @@ locals {
108107
runners_monitoring = var.runners_monitoring
109108
runners_ebs_optimized = var.runners_ebs_optimized
110109
runners_instance_profile = var.runners_executor == "docker+machine" ? aws_iam_instance_profile.docker_machine[0].name : ""
111-
runners_additional_volumes = local.runners_additional_volumes
112110
docker_machine_options = length(local.docker_machine_options_string) == 1 ? "" : local.docker_machine_options_string
113111
docker_machine_name = format("%s-%s", local.runner_tags_merged["Name"], "%s") # %s is always needed
114112
runners_name = var.runners_name
@@ -118,13 +116,6 @@ locals {
118116
runners_executor = var.runners_executor
119117
runners_limit = var.runners_limit
120118
runners_concurrent = var.runners_concurrent
121-
runners_image = var.runners_image
122-
runners_privileged = var.runners_privileged
123-
runners_disable_cache = var.runners_disable_cache
124-
runners_docker_runtime = var.runners_docker_runtime
125-
runners_helper_image = var.runners_helper_image
126-
runners_shm_size = var.runners_shm_size
127-
runners_pull_policies = local.runners_pull_policies
128119
runners_idle_count = var.runners_idle_count
129120
runners_idle_time = var.runners_idle_time
130121
runners_max_builds = local.runners_max_builds_string
@@ -141,6 +132,7 @@ locals {
141132
runners_request_concurrency = var.runners_request_concurrency
142133
runners_output_limit = var.runners_output_limit
143134
runners_check_interval = var.runners_check_interval
135+
runners_docker_options = local.runners_docker_options_toml
144136
runners_volumes_tmpfs = join("\n", [for v in var.runners_volumes_tmpfs : format("\"%s\" = \"%s\"", v.volume, v.options)])
145137
runners_services_volumes_tmpfs = join("\n", [for v in var.runners_services_volumes_tmpfs : format("\"%s\" = \"%s\"", v.volume, v.options)])
146138
runners_docker_services = local.runners_docker_services

Diff for: ‎migrations/migrate-to-7-0-0.sh

+51
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,54 @@ sed -i '/asg_terminate_lifecycle_lambda_timeout/d' "$converted_file"
2828
# PR #711 feat!: refactor Docker Machine autoscaling options
2929
#
3030
sed -i 's/runners_machine_autoscaling/runners_machine_autoscaling_options/g' "$converted_file"
31+
32+
#
33+
# PR #710 chore!: remove old variable `runners_pull_policy`
34+
#
35+
sed -i '/runners_pull_policy/d' "$converted_file"
36+
37+
#
38+
# PR #511 feat!: allow to set all docker options for the Executor
39+
#
40+
extracted_variables=$(grep -E '(runners_docker_runtime|runners_helper_image|runners_shm_size|runners_shm_size|runners_extra_hosts|runners_disable_cache|runners_image|runners_privileged)' "$converted_file")
41+
42+
sed -i '/runners_image/d' "$converted_file"
43+
sed -i '/runners_privileged/d' "$converted_file"
44+
sed -i '/runners_disable_cache/d' "$converted_file"
45+
sed -i '/runners_extra_hosts/d' "$converted_file"
46+
sed -i '/runners_shm_size/d' "$converted_file"
47+
sed -i '/runners_docker_runtime/d' "$converted_file"
48+
sed -i '/runners_helper_image/d' "$converted_file"
49+
50+
# content to be added to `volumes`
51+
volumes=$(grep "runners_additional_volumes" "$converted_file" | cut -d '=' -f 2 | tr -d '[]')
52+
53+
if [ -n "$volumes" ]; then
54+
extracted_variables="$extracted_variables
55+
volumes = [\"/cache\", $volumes]"
56+
fi
57+
58+
sed -i '/runners_additional_volumes/d' "$converted_file"
59+
60+
61+
# rename the variables
62+
extracted_variables=$(echo "$extracted_variables" | \
63+
sed 's/runners_image/image/g' | \
64+
sed 's/runners_privileged/privileged/g' | \
65+
sed 's/runners_disable_cache/disable_cache/g' | \
66+
sed 's/runners_extra_hosts/extra_hosts/g' | \
67+
sed 's/runners_shm_size/shm_size/g' | \
68+
sed 's/runners_docker_runtime/runtime/g' | \
69+
sed 's/runners_helper_image/helper_image/g'
70+
)
71+
72+
# add new block runners_docker_options at the end
73+
echo "$(head -n -1 "$converted_file")
74+
runners_docker_options {
75+
$extracted_variables
76+
}
77+
}" > x
78+
79+
mv x "$converted_file"
80+
81+
echo "Module call converted. Output: $converted_file"

Diff for: ‎template/runner-config.tftpl

+8-12
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,17 @@ listen_address = "${prometheus_listen_address}"
1818
request_concurrency = ${runners_request_concurrency}
1919
output_limit = ${runners_output_limit}
2020
limit = ${runners_limit}
21-
[runners.docker]
22-
tls_verify = false
23-
image = "${runners_image}"
24-
privileged = ${runners_privileged}
25-
disable_cache = ${runners_disable_cache}
26-
volumes = ["/cache"${runners_additional_volumes}]
27-
extra_hosts = ${jsonencode(runners_extra_hosts)}
28-
shm_size = ${runners_shm_size}
29-
pull_policy = ${runners_pull_policies}
30-
runtime = "${runners_docker_runtime}"
31-
helper_image = "${runners_helper_image}"
32-
${runners_docker_services}
21+
22+
${runners_docker_options}
23+
24+
${runners_docker_services}
25+
3326
[runners.docker.tmpfs]
3427
${runners_volumes_tmpfs}
28+
3529
[runners.docker.services_tmpfs]
3630
${runners_services_volumes_tmpfs}
31+
3732
[runners.cache]
3833
Type = "s3"
3934
Shared = ${shared_cache}
@@ -43,6 +38,7 @@ listen_address = "${prometheus_listen_address}"
4338
BucketName = "${bucket_name}"
4439
BucketLocation = "${aws_region}"
4540
Insecure = false
41+
4642
[runners.machine]
4743
IdleCount = ${runners_idle_count}
4844
IdleTime = ${runners_idle_time}

Diff for: ‎template/runners_docker_options.tftpl

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[runners.docker]
2+
%{ for key, value in options ~}
3+
${key} = ${jsonencode(value)}
4+
%{ endfor ~}

Diff for: ‎variables.tf

+63-50
Original file line numberDiff line numberDiff line change
@@ -206,64 +206,77 @@ variable "runners_max_builds" {
206206
default = 0
207207
}
208208

209-
variable "runners_image" {
210-
description = "Image to run builds, will be used in the runner config.toml"
211-
type = string
212-
default = "docker:18.03.1-ce"
213-
}
214-
215-
variable "runners_privileged" {
216-
description = "Runners will run in privileged mode, will be used in the runner config.toml"
217-
type = bool
218-
default = true
219-
}
220-
221-
variable "runners_disable_cache" {
222-
description = "Runners will not use local cache, will be used in the runner config.toml"
223-
type = bool
224-
default = false
225-
}
226-
227209
variable "runners_add_dind_volumes" {
228210
description = "Add certificates and docker.sock to the volumes to support docker-in-docker (dind)"
229211
type = bool
230212
default = false
231213
}
232214

233-
variable "runners_additional_volumes" {
234-
description = "Additional volumes that will be used in the runner config.toml, e.g Docker socket"
235-
type = list(any)
236-
default = []
237-
}
238-
239-
variable "runners_extra_hosts" {
240-
description = "Extra hosts that will be used in the runner config.toml, e.g other-host:127.0.0.1"
241-
type = list(any)
242-
default = []
243-
}
244-
245-
variable "runners_shm_size" {
246-
description = "shm_size for the runners, will be used in the runner config.toml"
247-
type = number
248-
default = 0
249-
}
250-
251-
variable "runners_docker_runtime" {
252-
description = "docker runtime for runners, will be used in the runner config.toml"
253-
type = string
254-
default = ""
255-
}
215+
variable "runners_docker_options" {
216+
description = <<EOT
217+
Options added to the [runners.docker] section of config.toml to configure the Docker container of the Executors. For
218+
details check https://docs.gitlab.com/runner/configuration/advanced-configuration.html
219+
220+
Default values if the option is not given:
221+
disable_cache = "false"
222+
image = "docker:18.03.1-ce"
223+
privileged = "true"
224+
pull_policy = "always"
225+
shm_size = 0
226+
tls_verify = "false"
227+
volumes = "/cache"
228+
EOT
256229

257-
variable "runners_helper_image" {
258-
description = "Overrides the default helper image used to clone repos and upload artifacts, will be used in the runner config.toml"
259-
type = string
260-
default = ""
261-
}
230+
type = object({
231+
allowed_images = optional(list(string))
232+
allowed_pull_policies = optional(list(string))
233+
allowed_services = optional(list(string))
234+
cache_dir = optional(string)
235+
cap_add = optional(list(string))
236+
cap_drop = optional(list(string))
237+
container_labels = optional(list(string))
238+
cpuset_cpus = optional(string)
239+
cpu_shares = optional(number)
240+
cpus = optional(string)
241+
devices = optional(list(string))
242+
device_cgroup_rules = optional(list(string))
243+
disable_cache = optional(bool, false)
244+
disable_entrypoint_overwrite = optional(bool)
245+
dns = optional(list(string))
246+
dns_search = optional(list(string))
247+
extra_hosts = optional(list(string))
248+
gpus = optional(string)
249+
helper_image = optional(string)
250+
helper_image_flavor = optional(string)
251+
host = optional(string)
252+
hostname = optional(string)
253+
image = optional(string, "docker:18.03.1-ce")
254+
isolation = optional(string)
255+
links = optional(list(string))
256+
mac_address = optional(string)
257+
memory = optional(string)
258+
memory_swap = optional(string)
259+
memory_reservation = optional(string)
260+
network_mode = optional(string)
261+
oom_kill_disable = optional(bool)
262+
oom_score_adjust = optional(number)
263+
privileged = optional(bool, true)
264+
pull_policies = optional(list(string), ["always"])
265+
runtime = optional(string)
266+
security_opt = optional(list(string))
267+
shm_size = optional(number, 0)
268+
sysctls = optional(list(string))
269+
tls_cert_path = optional(string)
270+
tls_verify = optional(bool, false)
271+
user = optional(string)
272+
userns_mode = optional(string)
273+
volumes = optional(list(string), ["/cache"])
274+
volumes_from = optional(list(string))
275+
volume_driver = optional(string)
276+
wait_for_services_timeout = optional(number)
277+
})
262278

263-
variable "runners_pull_policies" {
264-
description = "pull policies for the runners, will be used in the runner config.toml, for Gitlab Runner >= 13.8, see https://docs.gitlab.com/runner/executors/docker.html#using-multiple-pull-policies "
265-
type = list(string)
266-
default = ["always"]
279+
default = null
267280
}
268281

269282
variable "runners_monitoring" {

Diff for: ‎versions.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
terraform {
2-
required_version = ">= 1"
2+
required_version = ">= 1.3"
33

44
required_providers {
55
aws = {

0 commit comments

Comments
 (0)