feat: cancel spot requests (#653)

## Description

Whenever a spot executor is created a spot request is created. Occasionally there might be open spot requests which have not been fulfilled. These spot requests have to be deleted as soon as the agent terminates as no executors are needed.

Challenge: Make sure that spot requests are deleted only which belong to our module. Unfortunately the GitLab Runner does not tag them. We use the following logic for determinination:

- state is open or active
- they are associated to a SSH key which name starts with `runner`
- they have the `var.environment`/`var.overrides['name_docker_machine']` somewhere in the name

Solves part of #623 
Closes #493

Author: kayman-mk

modules/terminate-agent-hook/README.md

@@ -17,6 +17,12 @@ instances with no running parent runner.
 See [issue #214](https://github.com/npalm/terraform-aws-gitlab-runner/issues/214) for
 discussion on the scenario this module addresses.
 
+Furthermore, all spot requests which are still open are cancelled. Otherwise they might be fulfilled later but
+without the creating instance running, these spot request are never terminated and costs incur. The problem here
+is, that no tags are added to the spot request by the docker+machine driver and we can only guess which ones belong
+to our module. The rule is, that parts of the Executor's name become part of the related SSH key which is in turn part
+of the spot request.
+
 ## Usage
 
 ### Default Behavior - Package With the Module
@@ -157,4 +163,4 @@ No modules.
 | <a name="output_lambda_function_invoke_arn"></a> [lambda\_function\_invoke\_arn](#output\_lambda\_function\_invoke\_arn) | Lambda function invoke arn. |
 | <a name="output_lambda_function_name"></a> [lambda\_function\_name](#output\_lambda\_function\_name) | Lambda function name. |
 | <a name="output_lambda_function_source_code_hash"></a> [lambda\_function\_source\_code\_hash](#output\_lambda\_function\_source\_code\_hash) | Lambda function source code hash. |
-<!-- END_TF_DOCS -->
+<!-- END_TF_DOCS -->

modules/terminate-agent-hook/iam.tf

@@ -114,6 +114,20 @@ data "aws_iam_policy_document" "lambda" {
   }
 }
 
+data "aws_iam_policy_document" "spot_request_housekeeping" {
+  statement {
+    sid = "SpotRequestHousekeepingList"
+
+    effect = "Allow"
+    actions = [
+      "ec2:CancelSpotInstanceRequests",
+      "ec2:DescribeSpotInstanceRequests"
+    ]
+    # I didn't found any condition to limit the access
+    resources = ["*"]
+  }
+}
+
 resource "aws_iam_policy" "lambda" {
   name   = "${var.name_iam_objects}-${var.name}-lambda"
   path   = "/"
@@ -126,3 +140,16 @@ resource "aws_iam_role_policy_attachment" "lambda" {
   role       = aws_iam_role.lambda.name
   policy_arn = aws_iam_policy.lambda.arn
 }
+
+resource "aws_iam_policy" "spot_request_housekeeping" {
+  name   = "${var.name_iam_objects}-${var.name}-cancel-spot"
+  path   = "/"
+  policy = data.aws_iam_policy_document.spot_request_housekeeping.json
+
+  tags = var.tags
+}
+
+resource "aws_iam_role_policy_attachment" "spot_request_housekeeping" {
+  role       = aws_iam_role.lambda.name
+  policy_arn = aws_iam_policy.spot_request_housekeeping.arn
+}

modules/terminate-agent-hook/lambda/lambda_function.py

@@ -15,6 +15,7 @@
 import os
 
 
+
 def ec2_list(client, **args):
     print(json.dumps({
         "Level": "info",
@@ -90,6 +91,63 @@ def ec2_list(client, **args):
     return _terminate_list
 
 
+def cancel_active_spot_requests(ec2_client, executor_name_part):
+    print(json.dumps({
+        "Level": "info",
+        "Message": f"Removing open spot requests for environment {executor_name_part}"
+    }))
+
+    spot_requests_to_cancel = []
+
+    next_token = ''
+    has_more_spot_requests = True
+
+    while has_more_spot_requests:
+        response = ec2_client.describe_spot_instance_requests(Filters=[
+            {
+                "Name": "state",
+                "Values": ['active', 'open']
+            },
+            {
+                "Name": "launch.key-name",
+                "Values": ["runner-*"]
+            }
+        ], MaxResults=1000, NextToken=next_token)
+
+        for spot_request in response["SpotInstanceRequests"]:
+            if executor_name_part in spot_request["LaunchSpecification"]["KeyName"]:
+                spot_requests_to_cancel.append(spot_request["SpotInstanceRequestId"])
+
+                print(json.dumps({
+                    "Level": "info",
+                    "Message": f"Identified spot request {spot_request['SpotInstanceRequestId']}"
+                }))
+
+        if 'NextToken' in response and response['NextToken']:
+            next_token = response['NextToken']
+        else:
+            has_more_spot_requests = False
+
+    if spot_requests_to_cancel:
+        try:
+            ec2_client.cancel_spot_instance_requests(SpotInstanceRequestIds=spot_requests_to_cancel)
+
+            print(json.dumps({
+                "Level": "info",
+                "Message": "Spot requests deleted"
+            }))
+        except Exception as e:
+            print(json.dumps({
+                "Level": "exception",
+                "Message": "Bulk cancelling spot requests failed",
+                "Exception": str(e)
+            }))
+    else:
+        print(json.dumps({
+            "Level": "info",
+            "Message": "No spot requests to cancel"
+        }))
+
 def remove_unused_ssh_key_pairs(client, executor_name_part):
     print(json.dumps({
         "Level": "info",
@@ -147,19 +205,31 @@ def remove_unused_ssh_key_pairs(client, executor_name_part):
 def handler(event, context):
     response = []
     event_detail = event['detail']
-    client = boto3.client("ec2", region_name=event['region'])
+
     if event_detail['LifecycleTransition'] != "autoscaling:EC2_INSTANCE_TERMINATING":
         exit()
 
+    client = boto3.client("ec2", region_name=event['region'])
+
+    # make sure that no new instances are created
+    cancel_active_spot_requests(ec2_client=client, executor_name_part=os.environ['NAME_EXECUTOR_INSTANCE'])
+
     # find the executors connected to this agent and terminate them as well
     _terminate_list = ec2_list(client=client, parent=event_detail['EC2InstanceId'])
+
     if len(_terminate_list) > 0:
         print(json.dumps({
             "Level": "info",
             "Message": f"Terminating instances {', '.join(_terminate_list)}"
         }))
         try:
             client.terminate_instances(InstanceIds=_terminate_list, DryRun=False)
+
+            print(json.dumps({
+                "Level": "info",
+                "Message": "Instances terminated"
+            }))
+
         except Exception as e:
             print(json.dumps({
                 "Level": "exception",