feat: Added publishing to workflow_job event queue for multi runner module. (#2570)

* feat: Remove support check_run (#2521)

* chore: Remove support check_run

* format, lint

* feat: Remove old scale down mechanism (< 0.19.0) (#2519)

fix: Remove old cleanup mechanism (< 0.19.0)

* chore: Enable pre releases for next branch

* feat: Add multi-runner capability (#2472)

* feat: Remove support check_run (#2521)

* chore: Remove support check_run

* format, lint

* feat: Remove old scale down mechanism (< 0.19.0) (#2519)

fix: Remove old cleanup mechanism (< 0.19.0)

* feat: added changes for multi runner.

* fix: region.

* fix: more fixes.

* tuple to list.

* fixes.

* fixes.

* fixes.

* fixes.

* fixes.

* fixes.

* fix: formatting.

* fix: formatting.

* fix: formatting.

* fix: moved some blocks outside runner config.

* fix: few more updates

* fix: liniting.

* fix: updated example output

* changed runner group name.

* fix: updated the tests.

* fix: addressed review comments.

* fix: linting issues.

* fix: formatting.

* fix: updated tf version.

* fix: Remove removed prerelease option

* Add ubuntu runner to example

* refactor: use each instead of count

* fix: few small issues.

* refactor: syncer to count for multi runner

* fix: comments.

* fix: added Readme.

* fix: errors.

* move variable to runner config

* fix: updated the readme.

* Add todos

* feat: added windows runner configuration, completed todos and added the weight for runner config matchers.

* chore: Update docs

* fix: reverted tf versions.

* fix: addressed comments.

* fix: missed.

* fix: formatting.

* Update terraform versions in CI

* Update terraform versions in CI

* Update docs

* fix: coverage.

* Update docs

* improve test coverage webhook

* Apply suggestions from code review

* fix: formatting.

* fix: fixed merge issues.

* fix: syntax.

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: navdeepg2021 <[email protected]>

* chore(ci): Enable ci for next branch

BREAKING CHANGE:
- Add option to deploy multiple runner at once, see for details: #2472 #2517
- Drop support check_run event, see for details: #2521 #2517
- Remove pre 0.19.0 scale down mechanism, see details: #2519 #2517

* chore(release): 2.0.0-next.1 [skip ci]

* **ci:**

* Add multi-runner capability ([#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472)) ([c08b335](philips-labs/terraform-aws-github-runner@c08b335)), closes [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)
* Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([ac046b8](philips-labs/terraform-aws-github-runner@ac046b8))
* Remove old scale down mechanism (< 0.19.0) ([#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)) ([721d7c3](philips-labs/terraform-aws-github-runner@721d7c3))
* Remove support check_run ([#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521)) ([272a293](philips-labs/terraform-aws-github-runner@272a293))

* **ci:** Enable ci for next branch ([48769ca](philips-labs/terraform-aws-github-runner@48769ca)), closes [#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517)

* chore: fix release workflow

* fix: added changes for publishing events to secondary queue.

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: navdeepg2021 <[email protected]>
Co-authored-by: semantic-release-bot <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -31,7 +31,6 @@
 
 * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([985e722](https://github.com/philips-labs/terraform-aws-github-runner/commit/985e722229ce464235d206484df3d989db03e143))
 
-
 ## [1.12.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.11.0...v1.12.0) (2022-10-12)
 
 

examples/multi-runner/main.tf

@@ -159,7 +159,9 @@ module "multi-runner" {
   # runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
   # runners_lambda_zip                = "lambdas-download/runners.zip"
 
+  # enable_workflow_job_events_queue = true
   # override delay of events in seconds
 
   # log_level = "debug"
+
 }

modules/multi-runner/queues.tf

@@ -70,3 +70,22 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
   queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id
   policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
 }
+
+resource "aws_sqs_queue" "webhook_events_workflow_job_queue" {
+  count                       = var.enable_workflow_job_events_queue ? 1 : 0
+  name                        = "${var.prefix}-webhook_events_workflow_job_queue"
+  delay_seconds               = var.workflow_job_queue_configuration.delay_seconds
+  visibility_timeout_seconds  = var.workflow_job_queue_configuration.visibility_timeout_seconds
+  message_retention_seconds   = var.workflow_job_queue_configuration.message_retention_seconds
+  fifo_queue                  = false
+  receive_wait_time_seconds   = 0
+  content_based_deduplication = false
+  redrive_policy              = null
+
+  sqs_managed_sse_enabled           = var.queue_encryption.sqs_managed_sse_enabled
+  kms_master_key_id                 = var.queue_encryption.kms_master_key_id
+  kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
+
+  tags = var.tags
+}
+

modules/multi-runner/variables.tf

@@ -489,3 +489,24 @@ variable "pool_lambda_reserved_concurrent_executions" {
   type        = number
   default     = 1
 }
+
+variable "enable_workflow_job_events_queue" {
+  description = "Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow_job event will be delivered."
+  type        = bool
+  default     = false
+}
+
+variable "workflow_job_queue_configuration" {
+  description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true."
+  type = object({
+    delay_seconds              = number
+    visibility_timeout_seconds = number
+    message_retention_seconds  = number
+  })
+  default = {
+    "delay_seconds" : null,
+    "visibility_timeout_seconds" : null,
+    "message_retention_seconds" : null
+  }
+}
+

modules/multi-runner/webhook.tf

@@ -6,6 +6,8 @@ module "webhook" {
 
   runner_config                 = local.runner_config
   github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn
+  sqs_workflow_job_queue        = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null
+
 
   lambda_s3_bucket                              = var.lambda_s3_bucket
   webhook_lambda_s3_key                         = var.webhook_lambda_s3_key

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock

@@ -358,6 +358,7 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.9.tgz#f2dde0c682ccc264a9a8595efd030a5cc8fd2539"
   integrity sha512-9uJveS9eY9DJ0t64YbIBZICtJy8a5QrDEVdiLCG97fVLpDTpGX7t8mMSb6OWw6Lrnjqj4O8zwjELX3dhoMgiBg==
 
+
 "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.16.0", "@babel/parser@^7.16.4", "@babel/parser@^7.16.7", "@babel/parser@^7.17.3", "@babel/parser@^7.17.8", "@babel/parser@^7.18.10", "@babel/parser@^7.18.6", "@babel/parser@^7.19.3", "@babel/parser@^7.19.4":
   version "7.20.0"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.0.tgz#b26133c888da4d79b0d3edcf42677bcadc783046"

modules/webhook/lambdas/webhook/src/webhook/handler.ts

@@ -63,10 +63,10 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis
 
   const workflowJobEvent = payload as WorkflowJobEvent;
   response = await handleWorkflowJob(workflowJobEvent, githubEvent, queuesConfig);
-  await sendWorkflowJobEvents(githubEvent, workflowJobEvent);
+  await sendWorkflowJobEvents(workflowJobEvent);
   return response;
 }
-async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) {
+async function sendWorkflowJobEvents(workflowEventPayload: WorkflowJobEvent) {
   await sendWebhookEventToWorkflowJobQueue({
     workflowJobEvent: workflowEventPayload,
   });