feat!: add idle_count_min and idle_scale_factor` to Docker Machine autoscaling options (#711)

## Description

Switches from hardcoded options to free-from scaling configuration. This
reduces the module complexity by allowing to get rid of a number of
variables while giving more control to the user to define their options
without us having to build support into it for.

Adds `idle_scale_factor` and `idle_count_min` Docker Machine options.
See
[documentation](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersmachine-section").

## Migrations required

YES - users will have to change the input name from
`runners_machine_autoscaling` to `runners_machine_autoscaling_options`.
No other changes should be needed, we just support _more_ options. A
migration script is available.

## Verification

No input given:

(end of rendered `config.toml`)


![image](https://user-images.githubusercontent.com/17970041/225890782-02fe4adc-4c6a-4237-9752-a64349464113.png)

Input:

```hcl

runners_machine_autoscaling_options = [
    {
      periods           = ["* * 9-17 * * mon-fri *", "* * 9-17 * * mon-fri *"]
      idle_count        = 50
      idle_count_min    = 10
      idle_time         = 3600
      timezone          = "UTC"
      idle_scale_factor = 1.5
    },
    {
      periods    = ["* * 9-17 * * mon-fri *", "* * 9-17 * * mon-fri *"]
      idle_count = 50
      idle_time  = 3600
      timezone   = "Europe/Amsterdam"
    }
  ]
```

Rendered `config.toml`:


![image](https://user-images.githubusercontent.com/17970041/225891085-add03ee8-3943-4c56-96a4-d1a8c252deb0.png)

Apply results:


![image](https://user-images.githubusercontent.com/17970041/225893020-a9850486-4aa6-4eb0-b996-558ec7bccfea.png)


Closes #556

---------

Co-authored-by: Matthias Kay <[email protected]>

Author: tmeijn

.cspell.json

@@ -2,6 +2,7 @@
   "version": "0.2",
   "language": "en",
   "words": [
+    "alltrue",
     "amazonec",
     "amannn",
     "amazonec",
@@ -22,9 +23,7 @@
     "endfor",
     "formatlist",
     "gitter",
-    "godotenv",
-    "golangci",
-    "gruntwork",
+    "glrunners",
     "instancelifecycle",
     "kics",
     "joho",
@@ -39,10 +38,12 @@
     "pylint",
     "pylintrc",
     "pyright",
+    "setsubtract",
     "shuf",
     "signoff",
     "signum",
     "stretchr",
+    "subkey",
     "substr",
     "templatefile",
     "terrascan",

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        terraform: [ 1.0.11, 1.3.9, latest ]
+        terraform: [1.3.9, latest]
         example:
           [
             "runner-default",

.terraform-version

@@ -1 +1 @@
-1.0.8
+1.3.0

examples/runner-default/main.tf

@@ -97,9 +97,9 @@ module "runner" {
   ]
 
   # working 9 to 5 :)
-  runners_machine_autoscaling = [
+  runners_machine_autoscaling_options = [
     {
-      periods    = ["\"* * 0-9,17-23 * * mon-fri *\"", "\"* * * * * sat,sun *\""]
+      periods    = ["* * 0-9,17-23 * * mon-fri *", "* * * * * sat,sun *"]
       idle_count = 0
       idle_time  = 60
       timezone   = var.timezone

examples/runner-pre-registered/main.tf

@@ -40,9 +40,9 @@ module "runner" {
   runners_token      = var.runner_token
 
   # working 9 to 5 :)
-  runners_machine_autoscaling = [
+  runners_machine_autoscaling_options = [
     {
-      periods    = ["\"* * 0-9,17-23 * * mon-fri *\"", "\"* * * * * sat,sun *\""]
+      periods    = ["* * 0-9,17-23 * * mon-fri *", "* * * * * sat,sun *"]
       idle_count = 0
       idle_time  = 60
       timezone   = var.timezone

locals.tf

@@ -68,11 +68,6 @@ locals {
   %{~if var.runners_add_dind_volumes~},"/certs/client", "/builds", "/var/run/docker.sock:/var/run/docker.sock"%{endif~}%{~for volume in var.runners_additional_volumes~},"${volume}"%{endfor~}
   EOT
 
-  runners_machine_autoscaling = templatefile("${path.module}/template/runners_machine_autoscaling.tftpl", {
-    runners_machine_autoscaling = var.runners_machine_autoscaling
-    }
-  )
-
   runners_docker_services = templatefile("${path.module}/template/runners_docker_services.tftpl", {
     runners_docker_services = var.runners_docker_services
     }

main.tf

@@ -84,6 +84,12 @@ locals {
 
   template_runner_config = templatefile("${path.module}/template/runner-config.tftpl",
     {
+      runners_machine_autoscaling = [for config in var.runners_machine_autoscaling_options : {
+        for key, value in config :
+        # Convert key from snake_case to PascalCase which is the casing for this section.
+        join("", [for subkey in split("_", key) : title(subkey)]) => jsonencode(value) if value != null
+      }]
+
       aws_region                        = var.aws_region
       gitlab_url                        = var.runners_gitlab_url
       gitlab_clone_url                  = var.runners_clone_url
@@ -121,7 +127,6 @@ locals {
       runners_idle_count                = var.runners_idle_count
       runners_idle_time                 = var.runners_idle_time
       runners_max_builds                = local.runners_max_builds_string
-      runners_machine_autoscaling       = local.runners_machine_autoscaling
       runners_root_size                 = var.runners_root_size
       runners_volume_type               = var.runners_volume_type
       runners_iam_instance_profile_name = var.runners_iam_instance_profile_name

migrations/migrate-to-7-0-0.sh

@@ -23,3 +23,8 @@ sed -i '/asg_terminate_lifecycle_hook_heartbeat_timeout/d' "$converted_file"
 sed -i '/asg_terminate_lifecycle_lambda_memory_size/d' "$converted_file"
 sed -i '/asg_terminate_lifecycle_lambda_runtime/d' "$converted_file"
 sed -i '/asg_terminate_lifecycle_lambda_timeout/d' "$converted_file"
+
+#
+#  PR #711 feat!: refactor Docker Machine autoscaling options 
+#
+sed -i 's/runners_machine_autoscaling/runners_machine_autoscaling_options/g' "$converted_file"

outputs.tf

@@ -57,3 +57,8 @@ output "runner_user_data" {
   description = "The user data of the Gitlab Runner Agent's launch template."
   value       = local.template_user_data
 }
+
+output "runner_config_toml_rendered" {
+  description = "The rendered config.toml given to the Runner Manager."
+  value       = local.template_runner_config
+}

template/runner-config.tftpl

@@ -80,4 +80,10 @@ listen_address = "${prometheus_listen_address}"
       ${docker_machine_options}
     ]
 
-${runners_machine_autoscaling}
+%{~ for config in runners_machine_autoscaling ~}
+    [[runners.machine.autoscaling]]
+    %{~ for key, value in config ~}
+      ${key} = ${value}
+      %{~ endfor ~}
+%{~ endfor ~}
+

template/runners_machine_autoscaling.tftpl

@@ -278,14 +278,27 @@ variable "runners_ebs_optimized" {
   default     = true
 }
 
-variable "runners_machine_autoscaling" {
+variable "runners_machine_autoscaling_options" {
   description = "Set autoscaling parameters based on periods, see https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersmachine-section"
   type = list(object({
-    periods    = list(string)
-    idle_count = number
-    idle_time  = number
-    timezone   = string
+    periods           = list(string)
+    idle_count        = optional(number)
+    idle_scale_factor = optional(number)
+    idle_count_min    = optional(number)
+    idle_time         = optional(number)
+    timezone          = optional(string, "UTC")
   }))
+
+  validation {
+    condition = alltrue([
+      for options in var.runners_machine_autoscaling_options :
+      length(
+        setsubtract([for key, value in options : key if value != null], ["periods", "timezone"])
+      ) > 0
+    ])
+
+    error_message = "Please specify an attribute that affects Autoscaling."
+  }
   default = []
 }