feat: added publishing to workflow_job event queue for multi runner module. (#2570)

GuptaNavdeep1983 · npalm · navdeepg2021 · web-flow · commit 81dc490226eb · 2022-11-14T17:26:06.000+05:30
* feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * chore: Enable pre releases for next branch * feat: Add multi-runner capability (#2472) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * feat: added changes for multi runner. * fix: region. * fix: more fixes. * tuple to list. * fixes. * fixes. * fixes. * fixes. * fixes. * fixes. * fix: formatting. * fix: formatting. * fix: formatting. * fix: moved some blocks outside runner config. * fix: few more updates * fix: liniting. * fix: updated example output * changed runner group name. * fix: updated the tests. * fix: addressed review comments. * fix: linting issues. * fix: formatting. * fix: updated tf version. * fix: Remove removed prerelease option * Add ubuntu runner to example * refactor: use each instead of count * fix: few small issues. * refactor: syncer to count for multi runner * fix: comments. * fix: added Readme. * fix: errors. * move variable to runner config * fix: updated the readme. * Add todos * feat: added windows runner configuration, completed todos and added the weight for runner config matchers. * chore: Update docs * fix: reverted tf versions. * fix: addressed comments. * fix: missed. * fix: formatting. * Update terraform versions in CI * Update terraform versions in CI * Update docs * fix: coverage. * Update docs * improve test coverage webhook * Apply suggestions from code review * fix: formatting. * fix: fixed merge issues. * fix: syntax. Co-authored-by: Niek Palm <npalm@users.noreply.github.com> Co-authored-by: Niek Palm <niek.palm@philips.com> Co-authored-by: navdeepg2021 <navdeepg2021@gmail.com> * chore(ci): Enable ci for next branch BREAKING CHANGE: - Add option to deploy multiple runner at once, see for details: #2472 #2517 - Drop support check_run event, see for details: #2521 #2517 - Remove pre 0.19.0 scale down mechanism, see details: #2519 #2517 * chore(release): 2.0.0-next.1 [skip ci] ## [2.0.0-next.1](philips-labs/terraform-aws-github-runner@v1.12.0...v2.0.0-next.1) (2022-10-19) ### ⚠ BREAKING CHANGES * **ci:** ### Features * Add multi-runner capability ([#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472)) ([c08b335](philips-labs/terraform-aws-github-runner@c08b335)), closes [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([ac046b8](philips-labs/terraform-aws-github-runner@ac046b8)) * Remove old scale down mechanism (< 0.19.0) ([#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)) ([721d7c3](philips-labs/terraform-aws-github-runner@721d7c3)) * Remove support check_run ([#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521)) ([272a293](philips-labs/terraform-aws-github-runner@272a293)) ### Miscellaneous Chores * **ci:** Enable ci for next branch ([48769ca](philips-labs/terraform-aws-github-runner@48769ca)), closes [#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) * chore: fix release workflow * fix: added changes for publishing events to secondary queue. Co-authored-by: Niek Palm <niek.palm@philips.com> Co-authored-by: Niek Palm <npalm@users.noreply.github.com> Co-authored-by: navdeepg2021 <navdeepg2021@gmail.com> Co-authored-by: semantic-release-bot <semantic-release-bot@martynus.net>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,32 +1,11 @@
 # Changelog
 
-<<<<<<< HEAD
 ## [1.13.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.12.0...v1.13.0) (2022-10-14)
 
 
 ### Features
 
 * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([985e722](https://github.com/philips-labs/terraform-aws-github-runner/commit/985e722229ce464235d206484df3d989db03e143))
-=======
-## [2.0.0-next.1](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.12.0...v2.0.0-next.1) (2022-10-19)
-
-
-### ⚠ BREAKING CHANGES
-
-* **ci:** 
-
-### Features
-
-* Add multi-runner capability ([#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472)) ([c08b335](https://github.com/philips-labs/terraform-aws-github-runner/commit/c08b335ade4e16a89d6e1068d7226c16b1eb1450)), closes [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)
-* Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([ac046b8](https://github.com/philips-labs/terraform-aws-github-runner/commit/ac046b8eb2a0d2d5e2219ae9ee0023fd8bdf7460))
-* Remove old scale down mechanism (< 0.19.0) ([#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)) ([721d7c3](https://github.com/philips-labs/terraform-aws-github-runner/commit/721d7c3287fd3e0caa57942d0d830b072d90c433))
-* Remove support check_run ([#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521)) ([272a293](https://github.com/philips-labs/terraform-aws-github-runner/commit/272a293613d3a1da360f0d61bff92d16ea64216c))
-
-
-### Miscellaneous Chores
-
-* **ci:** Enable ci for next branch ([48769ca](https://github.com/philips-labs/terraform-aws-github-runner/commit/48769cacc53657ac740537a9dc63d4628b8c2562)), closes [#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517)
->>>>>>> e694884 (chore(release): 2.0.0-next.1 [skip ci])
 
 ## [1.12.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.11.0...v1.12.0) (2022-10-12)
 
@@ -729,7 +708,7 @@
 - Fix missing permissions for CloudWatch Agent #445 @bennettp123
 - Swap scale up/scale down timeout description #468 @jonico
 - Fix for invalid configuration #466 @jonico
-- Add ssm:GetParameter to runner-ssm-parameters #446 @bennettp123  
+- Add ssm:GetParameter to runner-ssm-parameters #446 @bennettp123
 - Replace crypto #429
 - Scale up lambda deprecated attribute #410
 
@@ -766,17 +745,17 @@ terraform state rm $RESOURCE
 ### Added
 
 - Allow operator to pass in a list of managed IAM policy ARNs for the runner role #361 @jpalomaki
-- expand options for sourcing lambda to include S3 #292 @eky5006 
+- expand options for sourcing lambda to include S3 #292 @eky5006
 
 ## [0.6.0] - 2020-10-10
 
 ### Added
 
-- Only allow tagging and termination of runner instances #201 @jpalomaki 
+- Only allow tagging and termination of runner instances #201 @jpalomaki
 
 ### Fixed
 
-- Fix pagination with listing self-hosted runners #202 @HenryNguyen5 
+- Fix pagination with listing self-hosted runners #202 @HenryNguyen5
 
 
 ## [0.5.0] - 2020-08-25
diff --git a/examples/multi-runner/main.tf b/examples/multi-runner/main.tf
@@ -159,7 +159,9 @@ module "multi-runner" {
   # runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
   # runners_lambda_zip                = "lambdas-download/runners.zip"
 
+  # enable_workflow_job_events_queue = true
   # override delay of events in seconds
 
   # log_level = "debug"
+
 }
diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf
@@ -70,3 +70,22 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
   queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id
   policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
 }
+
+resource "aws_sqs_queue" "webhook_events_workflow_job_queue" {
+  count                       = var.enable_workflow_job_events_queue ? 1 : 0
+  name                        = "${var.prefix}-webhook_events_workflow_job_queue"
+  delay_seconds               = var.workflow_job_queue_configuration.delay_seconds
+  visibility_timeout_seconds  = var.workflow_job_queue_configuration.visibility_timeout_seconds
+  message_retention_seconds   = var.workflow_job_queue_configuration.message_retention_seconds
+  fifo_queue                  = false
+  receive_wait_time_seconds   = 0
+  content_based_deduplication = false
+  redrive_policy              = null
+
+  sqs_managed_sse_enabled           = var.queue_encryption.sqs_managed_sse_enabled
+  kms_master_key_id                 = var.queue_encryption.kms_master_key_id
+  kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
+
+  tags = var.tags
+}
+
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
@@ -489,3 +489,24 @@ variable "pool_lambda_reserved_concurrent_executions" {
   type        = number
   default     = 1
 }
+
+variable "enable_workflow_job_events_queue" {
+  description = "Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow_job event will be delivered."
+  type        = bool
+  default     = false
+}
+
+variable "workflow_job_queue_configuration" {
+  description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true."
+  type = object({
+    delay_seconds              = number
+    visibility_timeout_seconds = number
+    message_retention_seconds  = number
+  })
+  default = {
+    "delay_seconds" : null,
+    "visibility_timeout_seconds" : null,
+    "message_retention_seconds" : null
+  }
+}
+
diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf
@@ -6,6 +6,8 @@ module "webhook" {
 
   runner_config                 = local.runner_config
   github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn
+  sqs_workflow_job_queue        = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null
+
 
   lambda_s3_bucket                              = var.lambda_s3_bucket
   webhook_lambda_s3_key                         = var.webhook_lambda_s3_key
diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock
@@ -358,6 +358,7 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.9.tgz#f2dde0c682ccc264a9a8595efd030a5cc8fd2539"
   integrity sha512-9uJveS9eY9DJ0t64YbIBZICtJy8a5QrDEVdiLCG97fVLpDTpGX7t8mMSb6OWw6Lrnjqj4O8zwjELX3dhoMgiBg==
 
+
 "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.16.0", "@babel/parser@^7.16.4", "@babel/parser@^7.16.7", "@babel/parser@^7.17.3", "@babel/parser@^7.17.8", "@babel/parser@^7.18.10", "@babel/parser@^7.18.6", "@babel/parser@^7.19.3", "@babel/parser@^7.19.4":
   version "7.20.0"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.0.tgz#b26133c888da4d79b0d3edcf42677bcadc783046"
diff --git a/modules/runners/lambdas/runners/yarn.lock b/modules/runners/lambdas/runners/yarn.lock
@@ -3704,29 +3704,10 @@ jest-snapshot@^29.2.1:
     pretty-format "^29.2.1"
     semver "^7.3.5"
 
-<<<<<<< HEAD
-jest-util@^29.0.0, jest-util@^29.2.1:
-  version "29.2.1"
-  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.2.1.tgz#f26872ba0dc8cbefaba32c34f98935f6cf5fc747"
-  integrity sha512-P5VWDj25r7kj7kl4pN2rG/RN2c1TLfYYYZYULnS/35nFDjBai+hBeo3MDrYZS7p6IoY3YHZnt2vq4L6mKnLk0g==
-=======
-jest-util@^29.0.0, jest-util@^29.2.0:
-  version "29.2.0"
-  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.2.0.tgz#797935697e83a5722aeba401ed6cd01264295566"
-  integrity sha512-8M1dx12ujkBbnhwytrezWY0Ut79hbflwodE+qZKjxSRz5qt4xDp6dQQJaOCFvCmE0QJqp9KyEK33lpPNjnhevw==
-  dependencies:
-    "@jest/types" "^29.2.0"
-    "@types/node" "*"
-    chalk "^4.0.0"
-    ci-info "^3.2.0"
-    graceful-fs "^4.2.9"
-    picomatch "^2.2.3"
-
 jest-util@^29.0.0, jest-util@^29.1.2:
   version "29.1.2"
   resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.1.2.tgz#ac5798e93cb6a6703084e194cfa0898d66126df1"
   integrity sha512-vPCk9F353i0Ymx3WQq3+a4lZ07NXu9Ca8wya6o4Fe4/aO1e1awMMprZ3woPFpKwghEOW+UXgd15vVotuNN9ONQ==
->>>>>>> 5e17d6f (feat: Add multi-runner capability (#2472))
   dependencies:
     "@jest/types" "^29.2.1"
     "@types/node" "*"
diff --git a/modules/webhook/lambdas/webhook/src/webhook/handler.ts b/modules/webhook/lambdas/webhook/src/webhook/handler.ts
@@ -63,10 +63,10 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis
 
   const workflowJobEvent = payload as WorkflowJobEvent;
   response = await handleWorkflowJob(workflowJobEvent, githubEvent, queuesConfig);
-  await sendWorkflowJobEvents(githubEvent, workflowJobEvent);
+  await sendWorkflowJobEvents(workflowJobEvent);
   return response;
 }
-async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) {
+async function sendWorkflowJobEvents(workflowEventPayload: WorkflowJobEvent) {
   await sendWebhookEventToWorkflowJobQueue({
     workflowJobEvent: workflowEventPayload,
   });

Original file line number	Diff line number	Diff line change
`@@ -159,7 +159,9 @@ module "multi-runner" {`
`159`	`159`	`# runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"`
`160`	`160`	`# runners_lambda_zip = "lambdas-download/runners.zip"`
`161`	`161`
	`162`	`+ # enable_workflow_job_events_queue = true`
`162`	`163`	`# override delay of events in seconds`
`163`	`164`
`164`	`165`	`# log_level = "debug"`
	`166`	`+`
`165`	`167`	`}`