Make SQS modulair (#7)

* Make queue modulair, refactor and fix naming according architcture

* fix creating fake zip file

* Fix ci for webhook

Author: npalm

.github/workflows/lambda-syncer.yml renamed to .github/workflows/lambda-runner-binaries-syncer.yml

@@ -1,20 +1,20 @@
-name: Lambda Runner Distribution Syncer
+name: Lambda Runner Binaries Syncer
 on:
   push:
     branches:
       - master
   pull_request:
     paths:
       - .github/workflows/lambda-agent-webhook.yml
-      - "modules/action-runner-binary-cache/lambdas/syncer/**"
+      - "modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/**"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     container: node:12
     defaults:
       run:
-        working-directory: modules/action-runner-binary-cache/lambdas/syncer
+        working-directory: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/lambda-agent-webhook.yml renamed to .github/workflows/lambda-webhook.yml

@@ -5,16 +5,16 @@ on:
       - master
   pull_request:
     paths:
-      - .github/workflows/lambda-agent-webhook.yml
-      - "modules/agent/lambdas/webhook/**"
+      - .github/workflows/lambda-webhook.yml
+      - "modules/webhook/lambdas/webhook/**"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     container: node:12
     defaults:
       run:
-        working-directory: modules/agent/lambdas/webhook
+        working-directory: modules/webhook/lambdas/webhook
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/terraform.yml

@@ -19,9 +19,9 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
-        run: |
-          touch modules/action-runner-binary-cache/lambdas/syncer/syncer.zip
-          touch modules/agent/lambdas/webhook/webhook.zip
+        run:
+          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
+          touch modules/webhook/lambdas/webhook/webhook.zip
       - name: "Terraform Format"
         uses: hashicorp/terraform-github-actions@master
         with:

examples/default/outputs.tf

@@ -5,11 +5,13 @@ output "action_runners" {
 }
 
 
-output "lambda_syncer_function_name" {
-  value = module.runners.lambda_s3_action_runner_dist_syncer.id
+output "lambda_binaries_syncer_name" {
+  value = module.runners.binaries_syncer.lambda.id
 }
 
 
 output "github_app_webhook_secret" {
   value = random_password.random.result
 }
+
+

main.tf

@@ -3,21 +3,34 @@ locals {
     Environment = var.environment
   })
 
+  s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}"
 }
+
 resource "random_string" "random" {
   length  = 24
   special = false
   upper   = false
 }
 
-module "dsitrubtion_cache" {
-  source = "./modules/action-runner-binary-cache"
+resource "aws_sqs_queue" "queued_builds" {
+  name                        = "${var.environment}-queued-builds.fifo"
+  delay_seconds               = 30
+  fifo_queue                  = true
+  receive_wait_time_seconds   = 10
+  content_based_deduplication = true
+
+  tags = var.tags
+}
+
+module "webhook" {
+  source = "./modules/webhook"
 
   aws_region  = var.aws_region
   environment = var.environment
   tags        = local.tags
 
-  distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
+  sqs_build_queue           = aws_sqs_queue.queued_builds
+  github_app_webhook_secret = var.github_app_webhook_secret
 }
 
 module "runners" {
@@ -28,36 +41,18 @@ module "runners" {
   environment = var.environment
   tags        = local.tags
 
-  s3_location_runner_distribution = module.dsitrubtion_cache.s3_location_runner_distribution
+  s3_bucket_runner_binaries   = module.runner_binaries.bucket
+  s3_location_runner_binaries = local.s3_action_runner_url
 }
 
-
-module "agent" {
-  source = "./modules/agent"
+module "runner_binaries" {
+  source = "./modules/runner-binaries-syncer"
 
   aws_region  = var.aws_region
   environment = var.environment
   tags        = local.tags
 
-  github_app_webhook_secret = var.github_app_webhook_secret
-}
-
-
-resource "aws_iam_policy" "dist_bucket" {
-  name        = "${var.environment}-gh-distribution-bucket"
-  path        = "/"
-  description = "Policy for the runner to download the github action runner."
-
-  policy = templatefile("${path.module}/policies/action-runner-s3-policy.json",
-    {
-      s3_arn = module.dsitrubtion_cache.distribution_bucket.arn
-    }
-  )
-}
-
-resource "aws_iam_role_policy_attachment" "dist_bucket" {
-  role       = module.runners.role.name
-  policy_arn = aws_iam_policy.dist_bucket.arn
+  distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
 }
 
 resource "aws_resourcegroups_group" "resourcegroups_group" {

modules/action-runner-binary-cache/outputs.tf

@@ -10,7 +10,7 @@
     "lint": "yarn eslint --ext ts,tsx src",
     "watch": "ts-node-dev --respawn --exit-child src/local.ts",
     "build": "ncc build src/lambda.ts -o dist",
-    "dist": "yarn build && cd dist && zip ../syncer.zip index.js"
+    "dist": "yarn build && cd dist && zip ../runner-binaries-syncer.zip index.js"
   },
   "devDependencies": {
     "@octokit/rest": "^17.6.0",

modules/agent/outputs.tf

@@ -8,4 +8,3 @@ resource "aws_s3_bucket" "action_dist" {
   force_destroy = true
   tags          = var.tags
 }
-

modules/agent/policies.tf

@@ -0,0 +1,15 @@
+output "bucket" {
+  value = aws_s3_bucket.action_dist
+}
+
+output "runner_distribution_object_key" {
+  value = local.action_runner_distribution_object_key
+}
+
+output "lambda" {
+  value = aws_lambda_function.syncer
+}
+
+output "lambda_role" {
+  value = aws_iam_role.syncer_lambda
+}

modules/agent/webhook_queue.tf

@@ -1,6 +1,6 @@
 resource "aws_lambda_function" "syncer" {
-  filename         = "${path.module}/lambdas/syncer/syncer.zip"
-  source_code_hash = filebase64sha256("${path.module}/lambdas/syncer/syncer.zip")
+  filename         = "${path.module}/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip"
+  source_code_hash = filebase64sha256("${path.module}/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip")
   function_name    = "${var.environment}-syncer"
   role             = aws_iam_role.syncer_lambda.arn
   handler          = "index.handler"

modules/action-runner-binary-cache/lambdas/syncer/.eslintrc.js renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.eslintrc.js

@@ -0,0 +1,3 @@
+environment              = "test"
+distribution_bucket_name = "alkj4klrj32trogjreoigfvj"
+aws_region               = "eu-west-1"

modules/action-runner-binary-cache/lambdas/syncer/.gitignore renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.gitignore

@@ -68,7 +68,7 @@ resource "aws_launch_template" "runner" {
     environment                     = var.environment
     pre_install                     = var.userdata_pre_install
     post_install                    = var.userdata_post_install
-    s3_location_runner_distribution = var.s3_location_runner_distribution
+    s3_location_runner_distribution = var.s3_location_runner_binaries
   }))
 }
 

modules/action-runner-binary-cache/lambdas/syncer/.nvmrc renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.nvmrc

@@ -45,3 +45,20 @@ resource "aws_iam_role_policy_attachment" "ssm_parameters" {
   role       = aws_iam_role.runner.name
   policy_arn = aws_iam_policy.ssm_parameters.arn
 }
+
+resource "aws_iam_policy" "dist_bucket" {
+  name        = "${var.environment}-gh-distribution-bucket"
+  path        = "/"
+  description = "Policy for the runner to download the github action runner."
+
+  policy = templatefile("${path.module}/policies/instance-s3-policy.json",
+    {
+      s3_arn = var.s3_bucket_runner_binaries.arn
+    }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "dist_bucket" {
+  role       = aws_iam_role.runner.name
+  policy_arn = aws_iam_policy.dist_bucket.arn
+}

modules/action-runner-binary-cache/lambdas/syncer/.prettierrc renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.prettierrc

@@ -0,0 +1,11 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "githubActionDist",
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:GetObjectAcl"],
+      "Resource": ["${s3_arn}/*"]
+    }
+  ]
+}

modules/action-runner-binary-cache/lambdas/syncer/jest.config.js renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/jest.config.js

@@ -28,7 +28,14 @@ variable "environment" {
   type        = string
 }
 
-variable "s3_location_runner_distribution" {
+variable "s3_bucket_runner_binaries" {
+  type = object({
+    arn = string
+  })
+}
+
+
+variable "s3_location_runner_binaries" {
   description = "S3 location of runner distribution."
   type        = string
 }

modules/action-runner-binary-cache/lambdas/syncer/package.json renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -0,0 +1,10 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/eslint-recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+};

modules/action-runner-binary-cache/lambdas/syncer/src/lambda.ts renamed to modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts

@@ -50,7 +50,7 @@ resource "aws_lambda_function" "webhook" {
   environment {
     variables = {
       GITHUB_APP_WEBHOOK_SECRET = var.github_app_webhook_secret
-      SQS_URL_WEBHOOK           = aws_sqs_queue.webhook_events.id
+      SQS_URL_WEBHOOK           = var.sqs_build_queue.id
     }
   }
 
@@ -65,31 +65,53 @@ resource "aws_lambda_permission" "webhook" {
   source_arn    = "${aws_apigatewayv2_api.webhook.execution_arn}/*/*/${local.webhook_endpoint}"
 }
 
+data "aws_iam_policy_document" "lambda_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
+}
+
 resource "aws_iam_role" "webhook_lambda" {
   name               = "${var.environment}-action-webhook-lambda-role"
   assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json
   tags               = var.tags
 }
 
+resource "aws_iam_policy" "webhook_logging" {
+  name        = "${var.environment}-lamda-logging-policy"
+  description = "Lambda logging policy"
+
+  policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {})
+}
+
 resource "aws_iam_policy_attachment" "webhook_logging" {
   name       = "${var.environment}-logging"
   roles      = [aws_iam_role.webhook_lambda.name]
-  policy_arn = aws_iam_policy.lambda_logging.arn
+  policy_arn = aws_iam_policy.webhook_logging.arn
 }
 
 resource "aws_iam_policy" "webhook" {
+  count = var.create_sqs_publish_policy ? 1 : 0
+
   name        = "${var.environment}-lamda-webhook-sqs-publish-policy"
   description = "Lambda webhook policy"
 
   policy = templatefile("${path.module}/policies/lambda-webhook.json", {
-    sqs_webhook_event_arn = aws_sqs_queue.webhook_events.arn
+    sqs_resource_arn = var.sqs_build_queue.arn
   })
 }
 
 resource "aws_iam_policy_attachment" "webhook" {
+  count = var.create_sqs_publish_policy ? 1 : 0
+
   name       = "${var.environment}-webhook"
   roles      = [aws_iam_role.webhook_lambda.name]
-  policy_arn = aws_iam_policy.webhook.arn
+  policy_arn = aws_iam_policy.webhook[0].arn
 }