feat: Experimental feature - Duplicate workflow job event to extra queue (#2268)

* Update variables.tf

* Update main.tf

* feat:  Changes to publish gh actions event to second queue for monitoring.

* feat: Changes to publish gh actions event to second queue for monitoring.

* feat:  Changes to publish gh actions event to second queue for monitoring.

* Update index.ts

* feat: Changes to publish gh actions event to second queue for monitoring.

* feat: reverted

* Added test cases.

* fixed test case name.

* fix: formatting

* Review comments

* Review comments

* fixed output.

* feat: Few more changes.

* Update README.md

* Additional review comments.

* fixed formatting.

* prettified

* Changed secondary queue config to a single object

* declared the object type.

* fixed style issues

* fixed formatting

* fixed linting errors

* fix: fixed warnings.

* Changed the name of the queue to workflow_job_queue.

* fix: formatting.

* fix:formatting.

* Updated as per the comments.

Co-authored-by: Niek Palm <[email protected]>

* fix: review comments.

* fix: comments

* fix: formatting

* fix: addressed review comments.

* fix: reverted.

* fix: formatting.

* fix: updated tests.

* Update README.md

Co-authored-by: Niek Palm <[email protected]>

* Update README.md

Co-authored-by: Niek Palm <[email protected]>

* Update README.md

Co-authored-by: Niek Palm <[email protected]>

* Update README.md

Co-authored-by: Niek Palm <[email protected]>

* fix: addressed review comments.

* Update variables.tf

Co-authored-by: Niek Palm <[email protected]>

* Update main.tf

Co-authored-by: Niek Palm <[email protected]>

* fix: formatting.

* fix: cross reference.

* fix: formatting.

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: root <[email protected]>
Co-authored-by: navdeepg2021 <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+=======
 ## [1.11.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.10.0...v1.11.0) (2022-10-06)
 
 
@@ -84,6 +85,7 @@
 * Update ubuntu example to fix /opt/hostedtoolcache ([#2302](https://github.com/philips-labs/terraform-aws-github-runner/issues/2302)) ([8eea748](https://github.com/philips-labs/terraform-aws-github-runner/commit/8eea74817a9817ca386b77f1b90ae9ef721e250e))
 * Webhook lambda misleading log ([#2291](https://github.com/philips-labs/terraform-aws-github-runner/issues/2291)) ([c6275f9](https://github.com/philips-labs/terraform-aws-github-runner/commit/c6275f9d5a68c962e32596e4abf77b1fda6dd18f))
 
+
 ## [1.5.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.4.1...v1.5.0) (2022-07-08)
 
 

README.md

@@ -22,6 +22,7 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr
   - [Idle runners](#idle-runners)
   - [Ephemeral runners](#ephemeral-runners)
   - [Prebuilt Images](#prebuilt-images)
+  - [Experimental - Optional queue to publish GitHub workflow job events](#experimental---optional-queue-to-publish-github-workflow-job-events)
 - [Examples](#examples)
 - [Sub modules](#sub-modules)
   - [ARM64 configuration for submodules](#arm64-configuration-for-submodules)
@@ -310,7 +311,23 @@ The example for [ephemeral runners](./examples/ephemeral) is based on the [defau
 
 ### Prebuilt Images
 
-This module also allows you to run agents from a prebuilt AMI to gain faster startup times. You can find more information in [the image README.md](/images/README.md). When the GitHub runner is part of the AMI you can disable the binary syncer by setting `enable_runner_binaries_syncer = false`.
+This module also allows you to run agents from a prebuilt AMI to gain faster startup times. You can find more information in [the image README.md](/images/README.md)
+
+### Experimental - Optional queue to publish GitHub workflow job events
+
+This queue is an experimental feature to allow you to receive a copy of the wokflow_jobs events sent by the GItHub App. For example to calculate matrix or monitor the system.
+
+To enable the feature set `enable_workflow_job_events_queue = true`. Be-aware the feature in experimental!
+
+Messages received on the queue are using the same format as published by GitHub wrapped in a property `workflowJobEvent`.
+
+```
+export interface GithubWorkflowEvent {
+  workflowJobEvent: WorkflowJobEvent;
+}
+```
+This extendible format allows to add more fields to be added if needed.
+You can configure the queue by setting properties to `workflow_job_events_queue_config`
 
 ## Examples
 

examples/arm64/README.md

@@ -28,4 +28,4 @@ You can receive the webhook details by running:
 terraform output -raw webhook_secret
 ```
 
-Be-aware some shells will print some end of line character `%`. 
+Be-aware some shells will print some end of line character `%`.

examples/default/main.tf

@@ -83,4 +83,6 @@ module "runners" {
 
   # override scaling down
   scale_down_schedule_expression = "cron(* * * * ? *)"
+  # enable this flag to publish webhook events to workflow job queue
+  # enable_workflow_job_events_queue  = true
 }

main.tf

@@ -49,6 +49,12 @@ resource "aws_sqs_queue_policy" "build_queue_policy" {
   policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
 }
 
+resource "aws_sqs_queue_policy" "webhook_events_workflow_job_queue_policy" {
+  count     = var.enable_workflow_job_events_queue ? 1 : 0
+  queue_url = aws_sqs_queue.webhook_events_workflow_job_queue[0].id
+  policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
+}
+
 resource "aws_sqs_queue" "queued_builds" {
   name                        = "${var.prefix}-queued-builds${var.fifo_build_queue ? ".fifo" : ""}"
   delay_seconds               = var.delay_webhook_event
@@ -69,6 +75,24 @@ resource "aws_sqs_queue" "queued_builds" {
   tags = var.tags
 }
 
+resource "aws_sqs_queue" "webhook_events_workflow_job_queue" {
+  count                       = var.enable_workflow_job_events_queue ? 1 : 0
+  name                        = "${var.prefix}-webhook_events_workflow_job_queue"
+  delay_seconds               = var.workflow_job_queue_configuration.delay_seconds
+  visibility_timeout_seconds  = var.workflow_job_queue_configuration.visibility_timeout_seconds
+  message_retention_seconds   = var.workflow_job_queue_configuration.message_retention_seconds
+  fifo_queue                  = false
+  receive_wait_time_seconds   = 0
+  content_based_deduplication = false
+  redrive_policy              = null
+
+  sqs_managed_sse_enabled           = var.queue_encryption.sqs_managed_sse_enabled
+  kms_master_key_id                 = var.queue_encryption.kms_master_key_id
+  kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
+
+  tags = var.tags
+}
+
 resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
   count     = var.redrive_build_queue.enabled ? 1 : 0
   queue_url = aws_sqs_queue.queued_builds.id
@@ -98,13 +122,13 @@ module "ssm" {
 module "webhook" {
   source = "./modules/webhook"
 
-  aws_region  = var.aws_region
-  prefix      = var.prefix
-  tags        = local.tags
-  kms_key_arn = var.kms_key_arn
-
+  aws_region                    = var.aws_region
+  prefix                        = var.prefix
+  tags                          = local.tags
+  kms_key_arn                   = var.kms_key_arn
   sqs_build_queue               = aws_sqs_queue.queued_builds
   sqs_build_queue_fifo          = var.fifo_build_queue
+  sqs_workflow_job_queue        = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null
   github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn
 
   lambda_s3_bucket                              = var.lambda_s3_bucket

modules/runners/variables.tf

@@ -565,7 +565,6 @@ variable "lambda_architecture" {
     error_message = "`lambda_architecture` value is not valid, valid values are: `arm64` and `x86_64`."
   }
 }
-
 variable "enable_runner_binaries_syncer" {
   description = "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
   type        = bool

modules/webhook/lambdas/webhook/src/lambda.ts

@@ -1,4 +1,4 @@
-import { APIGatewayEvent, Callback, Context } from 'aws-lambda';
+import { APIGatewayEvent, Context } from 'aws-lambda';
 
 import { handle } from './webhook/handler';
 import { logger } from './webhook/logger';

modules/webhook/lambdas/webhook/src/sqs/index.test.ts

@@ -1,6 +1,7 @@
 import { SQS } from 'aws-sdk';
 
-import { ActionRequestMessage, sendActionRequest } from '.';
+import { ActionRequestMessage, GithubWorkflowEvent, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '.';
+import workflowjob_event from '../../test/resources/github_workflowjob_event.json';
 
 const mockSQS = {
   sendMessage: jest.fn(() => {
@@ -25,7 +26,9 @@ describe('Test sending message to SQS.', () => {
     QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds',
     MessageBody: JSON.stringify(message),
   };
-
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
   it('no fifo queue, based on defaults', async () => {
     // Arrange
     process.env.SQS_URL_WEBHOOK = sqsMessage.QueueUrl;
@@ -64,3 +67,35 @@ describe('Test sending message to SQS.', () => {
     expect(result).resolves;
   });
 });
+describe('Test sending message to SQS.', () => {
+  const message: GithubWorkflowEvent = {
+    workflowJobEvent: JSON.parse(JSON.stringify(workflowjob_event)),
+  };
+  const sqsMessage: SQS.Types.SendMessageRequest = {
+    QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue',
+    MessageBody: JSON.stringify(message),
+  };
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+  it('sends webhook events to workflow job queue', async () => {
+    // Arrange
+    process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl;
+
+    // Act
+    const result = await sendWebhookEventToWorkflowJobQueue(message);
+
+    // Assert
+    expect(mockSQS.sendMessage).toBeCalledWith(sqsMessage);
+    expect(result).resolves;
+  });
+  it('Does not send webhook events to workflow job event copy queue', async () => {
+    // Arrange
+    process.env.SQS_WORKFLOW_JOB_QUEUE = '';
+    // Act
+    await sendWebhookEventToWorkflowJobQueue(message);
+
+    // Assert
+    expect(mockSQS.sendMessage).not.toBeCalledWith(sqsMessage);
+  });
+});

modules/webhook/lambdas/webhook/src/sqs/index.ts

@@ -1,3 +1,4 @@
+import { WorkflowJobEvent } from '@octokit/webhooks-types';
 import { SQS } from 'aws-sdk';
 
 import { LogFields, logger } from '../webhook/logger';
@@ -9,6 +10,9 @@ export interface ActionRequestMessage {
   repositoryOwner: string;
   installationId: number;
 }
+export interface GithubWorkflowEvent {
+  workflowJobEvent: WorkflowJobEvent;
+}
 
 export const sendActionRequest = async (message: ActionRequestMessage): Promise<void> => {
   const sqs = new SQS({ region: process.env.AWS_REGION });
@@ -28,3 +32,24 @@ export const sendActionRequest = async (message: ActionRequestMessage): Promise<
 
   await sqs.sendMessage(sqsMessage).promise();
 };
+
+export const sendWebhookEventToWorkflowJobQueue = async (message: GithubWorkflowEvent): Promise<void> => {
+  const webhook_events_workflow_job_queue = process.env.SQS_WORKFLOW_JOB_QUEUE || undefined;
+
+  if (webhook_events_workflow_job_queue != undefined) {
+    const sqs = new SQS({ region: process.env.AWS_REGION });
+    const sqsMessage: SQS.Types.SendMessageRequest = {
+      QueueUrl: String(process.env.SQS_WORKFLOW_JOB_QUEUE),
+      MessageBody: JSON.stringify(message),
+    };
+    logger.debug(
+      `Sending Webhook events to the workflow job queue: ${webhook_events_workflow_job_queue}`,
+      LogFields.print(),
+    );
+    try {
+      await sqs.sendMessage(sqsMessage).promise();
+    } catch (e) {
+      logger.warn(`Error in sending webhook events to workflow job queue: ${(e as Error).message}`, LogFields.print());
+    }
+  }
+};

modules/webhook/lambdas/webhook/src/webhook/handler.test.ts

@@ -4,7 +4,7 @@ import nock from 'nock';
 
 import checkrun_event from '../../test/resources/github_check_run_event.json';
 import workflowjob_event from '../../test/resources/github_workflowjob_event.json';
-import { sendActionRequest } from '../sqs';
+import { sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs';
 import { getParameterValue } from '../ssm';
 import { handle } from './handler';
 
@@ -309,4 +309,20 @@ describe('handler', () => {
       expect(sendActionRequest).toBeCalled();
     });
   });
+
+  describe('Test for webhook events to be sent to workflow job queue: ', () => {
+    beforeEach(() => {
+      process.env.SQS_WORKFLOW_JOB_QUEUE =
+        'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue';
+    });
+    it('sends webhook events to workflow job queue', async () => {
+      const event = JSON.stringify(workflowjob_event);
+      const resp = await handle(
+        { 'X-Hub-Signature': await webhooks.sign(event), 'X-GitHub-Event': 'workflow_job' },
+        event,
+      );
+      expect(resp.statusCode).toBe(201);
+      expect(sendWebhookEventToWorkflowJobQueue).toBeCalled();
+    });
+  });
 });

modules/webhook/lambdas/webhook/src/webhook/handler.ts

@@ -3,7 +3,7 @@ import { CheckRunEvent, WorkflowJobEvent } from '@octokit/webhooks-types';
 import { IncomingHttpHeaders } from 'http';
 
 import { Response } from '../lambda';
-import { sendActionRequest } from '../sqs';
+import { sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs';
 import { getParameterValue } from '../ssm';
 import { LogFields, logger as rootLogger } from './logger';
 
@@ -63,19 +63,26 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis
   logger.info(`Processing Github event`, LogFields.print());
 
   if (githubEvent == 'workflow_job') {
+    const workflowEventPayload = payload as WorkflowJobEvent;
     response = await handleWorkflowJob(
-      payload as WorkflowJobEvent,
+      workflowEventPayload,
       githubEvent,
       enableWorkflowLabelCheck,
       workflowLabelCheckAll,
       runnerLabels,
     );
+    await sendWorkflowJobEvents(githubEvent, workflowEventPayload);
   } else if (githubEvent == 'check_run') {
     response = await handleCheckRun(payload as CheckRunEvent, githubEvent);
   }
 
   return response;
 }
+async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) {
+  await sendWebhookEventToWorkflowJobQueue({
+    workflowJobEvent: workflowEventPayload,
+  });
+}
 
 function readEnvironmentVariables() {
   const environment = process.env.ENVIRONMENT;

modules/webhook/variables.tf

@@ -37,7 +37,14 @@ variable "sqs_build_queue" {
     arn = string
   })
 }
-
+variable "sqs_workflow_job_queue" {
+  description = "SQS queue to monitor github events."
+  type = object({
+    id  = string
+    arn = string
+  })
+  default = null
+}
 variable "lambda_zip" {
   description = "File location of the lambda zip file."
   type        = string

modules/webhook/webhook.tf

@@ -22,6 +22,7 @@ resource "aws_lambda_function" "webhook" {
       RUNNER_LABELS                    = jsonencode(split(",", lower(var.runner_labels)))
       SQS_URL_WEBHOOK                  = var.sqs_build_queue.id
       SQS_IS_FIFO                      = var.sqs_build_queue_fifo
+      SQS_WORKFLOW_JOB_QUEUE           = try(var.sqs_workflow_job_queue, null) != null ? var.sqs_workflow_job_queue.id : ""
     }
   }
 
@@ -78,6 +79,15 @@ resource "aws_iam_role_policy" "webhook_sqs" {
     sqs_resource_arn = var.sqs_build_queue.arn
   })
 }
+resource "aws_iam_role_policy" "webhook_workflow_job_sqs" {
+  count = var.sqs_workflow_job_queue != null ? 1 : 0
+  name  = "${var.prefix}-lambda-webhook-publish-workflow-job-sqs-policy"
+  role  = aws_iam_role.webhook_lambda.name
+
+  policy = templatefile("${path.module}/policies/lambda-publish-sqs-policy.json", {
+    sqs_resource_arn = var.sqs_workflow_job_queue.arn
+  })
+}
 
 resource "aws_iam_role_policy" "webhook_ssm" {
   name = "${var.prefix}-lambda-webhook-publish-ssm-policy"

outputs.tf

@@ -39,7 +39,8 @@ output "ssm_parameters" {
 output "queues" {
   description = "SQS queues."
   value = {
-    build_queue_arn     = aws_sqs_queue.queued_builds.arn
-    build_queue_dlq_arn = var.redrive_build_queue.enabled ? aws_sqs_queue.queued_builds_dlq[0].arn : null
+    build_queue_arn            = aws_sqs_queue.queued_builds.arn
+    build_queue_dlq_arn        = var.redrive_build_queue.enabled ? aws_sqs_queue.queued_builds_dlq[0].arn : null
+    webhook_workflow_job_queue = try(aws_sqs_queue.webhook_events_workflow_job_queue[0], null) != null ? aws_sqs_queue.webhook_events_workflow_job_queue[0].arn : ""
   }
 }

variables.tf

@@ -702,6 +702,25 @@ variable "lambda_architecture" {
   }
 }
 
+variable "enable_workflow_job_events_queue" {
+  description = "Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow_job event will be delivered."
+  type        = bool
+  default     = false
+}
+
+variable "workflow_job_queue_configuration" {
+  description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true."
+  type = object({
+    delay_seconds              = number
+    visibility_timeout_seconds = number
+    message_retention_seconds  = number
+  })
+  default = {
+    "delay_seconds" : null,
+    "visibility_timeout_seconds" : null,
+    "message_retention_seconds" : null
+  }
+}
 variable "enable_runner_binaries_syncer" {
   description = "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
   type        = bool