Merge pull request #1840 from philips-labs/develop

Release

Author: npalm

.github/workflows/lambda-runner-binaries-syncer.yml

@@ -17,7 +17,7 @@ jobs:
         working-directory: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install dependencies
         run: yarn install
       - name: Run prettier

.github/workflows/lambda-runners.yml

@@ -17,7 +17,7 @@ jobs:
         working-directory: modules/runners/lambdas/runners
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install dependencies
         run: yarn install
       - name: Run prettier

.github/workflows/lambda-webhook.yml

@@ -17,7 +17,7 @@ jobs:
         working-directory: modules/webhook/lambdas/webhook
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install dependencies
         run: yarn install
       - name: Run prettier

.github/workflows/packer-build.yml

@@ -27,7 +27,7 @@ jobs:
         working-directory: images/${{ matrix.image }}
     steps:
       - name: "Checkout"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: packer init
         run: packer init .

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
         env:
           LAMBDA: ${{ matrix.lambda }}
         run: echo ::set-output name=name::${LAMBDA##*/}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Add zip
         run: apt update && apt install zip
       - name: Build dist
@@ -39,7 +39,7 @@ jobs:
     needs:
       prepare
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           persist-credentials: false

.github/workflows/terraform.yml

@@ -21,7 +21,7 @@ jobs:
       image: hashicorp/terraform:${{ matrix.terraform }}
     steps:
       - name: "Checkout"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
         run: |
           touch modules/webhook/lambdas/webhook/webhook.zip
@@ -53,7 +53,7 @@ jobs:
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: terraform init
         run: terraform init -get -backend=false -input=false
       - if: contains(matrix.terraform, '1.1.')

README.md

@@ -387,14 +387,18 @@ In case the setup does not work as intended follow the trace of events:
 | [aws_resourcegroups_group.resourcegroups_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/resourcegroups_group) | resource |
 | [aws_sqs_queue.queued_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
 | [aws_sqs_queue.queued_builds_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
+| [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
+| [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
 | [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
+| [aws_iam_policy_document.deny_unsecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_ami_filter"></a> [ami\_filter](#input\_ami\_filter) | List of maps used to create the AMI filter for the action runner AMI. By default amazon linux 2 is used. | `map(list(string))` | `null` | no |
 | <a name="input_ami_owners"></a> [ami\_owners](#input\_ami\_owners) | The list of owners used to select the AMI of action runner instances. | `list(string)` | <pre>[<br>  "amazon"<br>]</pre> | no |
+| <a name="input_aws_partition"></a> [aws\_partition](#input\_aws\_partition) | (optiona) partition in the arn namespace to use if not 'aws' | `string` | `"aws"` | no |
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes |
 | <a name="input_block_device_mappings"></a> [block\_device\_mappings](#input\_block\_device\_mappings) | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no |
 | <a name="input_cloudwatch_config"></a> [cloudwatch\_config](#input\_cloudwatch\_config) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no |
@@ -428,6 +432,7 @@ In case the setup does not work as intended follow the trace of events:
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
 | <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
+| <a name="input_logging_kms_key_id"></a> [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with | `string` | `null` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | <a name="input_market_options"></a> [market\_options](#input\_market\_options) | DEPCRECATED: Replaced by `instance_target_capacity_type`. | `string` | `null` | no |
 | <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if not busy. | `number` | `null` | no |

images/linux-amzn2/github_agent.linux.pkr.hcl

@@ -31,6 +31,12 @@ variable "subnet_id" {
   default     = null
 }
 
+variable "associate_public_ip_address" {
+  description = "If using a non-default VPC, there is no public IP address assigned to the EC2 instance. If you specified a public subnet, you probably want to set this to true. Otherwise the EC2 instance won't have access to the internet"
+  type        = string
+  default     = null
+}
+
 variable "instance_type" {
   description = "The instance type Packer will use for the builder"
   type        = string
@@ -66,15 +72,22 @@ variable "snapshot_tags" {
   default     = {}
 }
 
+variable "custom_shell_commands" {
+  description = "Additional commands to run on the EC2 instance, to customize the instance, like installing packages"
+  type        = list(string)
+  default     = []
+}
+
 source "amazon-ebs" "githubrunner" {
-  ami_name          = "github-runner-amzn2-x86_64-${formatdate("YYYYMMDDhhmm", timestamp())}"
-  instance_type     = var.instance_type
-  region            = var.region
-  security_group_id = var.security_group_id
-  subnet_id         = var.subnet_id
+  ami_name                    = "github-runner-amzn2-x86_64-${formatdate("YYYYMMDDhhmm", timestamp())}"
+  instance_type               = var.instance_type
+  region                      = var.region
+  security_group_id           = var.security_group_id
+  subnet_id                   = var.subnet_id
+  associate_public_ip_address = var.associate_public_ip_address
   source_ami_filter {
     filters = {
-      name                = "amzn2-ami-hvm-2.*-x86_64-ebs"
+      name                = "amzn2-ami-kernel-5.*-hvm-*-x86_64-gp2"
       root-device-type    = "ebs"
       virtualization-type = "hvm"
     }
@@ -111,15 +124,15 @@ build {
   ]
   provisioner "shell" {
     environment_vars = []
-    inline = [
+    inline = concat([
       "sudo yum update -y",
       "sudo yum install -y amazon-cloudwatch-agent curl jq git",
       "sudo amazon-linux-extras install docker",
       "sudo systemctl enable docker.service",
       "sudo systemctl enable containerd.service",
       "sudo service docker start",
       "sudo usermod -a -G docker ec2-user",
-    ]
+    ], var.custom_shell_commands)
   }
 
   provisioner "file" {

images/ubuntu-focal/github_agent.ubuntu.pkr.hcl

@@ -31,6 +31,12 @@ variable "subnet_id" {
   default     = null
 }
 
+variable "associate_public_ip_address" {
+  description = "If using a non-default VPC, there is no public IP address assigned to the EC2 instance. If you specified a public subnet, you probably want to set this to true. Otherwise the EC2 instance won't have access to the internet"
+  type        = string
+  default     = null
+}
+
 variable "instance_type" {
   description = "The instance type Packer will use for the builder"
   type        = string
@@ -66,12 +72,20 @@ variable "snapshot_tags" {
   default     = {}
 }
 
+variable "custom_shell_commands" {
+  description = "Additional commands to run on the EC2 instance, to customize the instance, like installing packages"
+  type        = list(string)
+  default     = []
+}
+
 source "amazon-ebs" "githubrunner" {
-  ami_name          = "github-runner-ubuntu-focal-amd64-${formatdate("YYYYMMDDhhmm", timestamp())}"
-  instance_type     = var.instance_type
-  region            = var.region
-  security_group_id = var.security_group_id
-  subnet_id         = var.subnet_id
+  ami_name                    = "github-runner-ubuntu-focal-amd64-${formatdate("YYYYMMDDhhmm", timestamp())}"
+  instance_type               = var.instance_type
+  region                      = var.region
+  security_group_id           = var.security_group_id
+  subnet_id                   = var.subnet_id
+  associate_public_ip_address = var.associate_public_ip_address
+
   source_ami_filter {
     filters = {
       name                = "*/ubuntu-focal-20.04-amd64-server-*"
@@ -112,7 +126,7 @@ build {
     environment_vars = [
       "DEBIAN_FRONTEND=noninteractive"
     ]
-    inline = [
+    inline = concat([
       "sudo apt-get -y update",
       "sudo apt-get -y install ca-certificates curl gnupg lsb-release",
       "sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg",
@@ -128,7 +142,7 @@ build {
       "sudo curl -f https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip",
       "unzip awscliv2.zip",
       "sudo ./aws/install",
-    ]
+    ], var.custom_shell_commands)
   }
 
   provisioner "file" {

images/windows-core-2019/github_agent.windows.pkr.hcl

@@ -25,11 +25,25 @@ variable "ebs_delete_on_termination" {
   default     = true
 }
 
+variable "associate_public_ip_address" {
+  description = "If using a non-default VPC, there is no public IP address assigned to the EC2 instance. If you specified a public subnet, you probably want to set this to true. Otherwise the EC2 instance won't have access to the internet"
+  type        = string
+  default     = null
+}
+
+variable "custom_shell_commands" {
+  description = "Additional commands to run on the EC2 instance, to customize the instance, like installing packages"
+  type        = list(string)
+  default     = []
+}
+
 source "amazon-ebs" "githubrunner" {
-  ami_name      = "github-runner-windows-core-2019-${formatdate("YYYYMMDDhhmm", timestamp())}"
-  communicator  = "winrm"
-  instance_type = "t3a.medium"
-  region        = var.region
+  ami_name                    = "github-runner-windows-core-2019-${formatdate("YYYYMMDDhhmm", timestamp())}"
+  communicator                = "winrm"
+  instance_type               = "t3a.medium"
+  region                      = var.region
+  associate_public_ip_address = var.associate_public_ip_address
+
   source_ami_filter {
     filters = {
       name                = "Windows_Server-2019-English-Core-ContainersLatest-*"
@@ -70,8 +84,10 @@ build {
   }
 
   provisioner "powershell" {
-    inline = [templatefile("./windows-provisioner.ps1", {
-      action_runner_url = var.action_runner_url
-    })]
+    inline = concat([
+      templatefile("./windows-provisioner.ps1", {
+        action_runner_url = var.action_runner_url
+      })
+    ], var.custom_shell_commands)
   }
 }

main.tf

@@ -17,6 +17,38 @@ resource "random_string" "random" {
   upper   = false
 }
 
+data "aws_iam_policy_document" "deny_unsecure_transport" {
+  statement {
+    sid = "DenyUnsecureTransport"
+
+    effect = "Deny"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    actions = [
+      "sqs:*"
+    ]
+
+    resources = [
+      "*"
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
+resource "aws_sqs_queue_policy" "build_queue_policy" {
+  queue_url = aws_sqs_queue.queued_builds.id
+  policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
+}
+
 resource "aws_sqs_queue" "queued_builds" {
   name                        = "${var.environment}-queued-builds${var.fifo_build_queue ? ".fifo" : ""}"
   delay_seconds               = var.delay_webhook_event
@@ -33,6 +65,13 @@ resource "aws_sqs_queue" "queued_builds" {
   tags = var.tags
 }
 
+
+resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
+  count     = var.redrive_build_queue.enabled ? 1 : 0
+  queue_url = aws_sqs_queue.queued_builds.id
+  policy    = data.aws_iam_policy_document.deny_unsecure_transport.json
+}
+
 resource "aws_sqs_queue" "queued_builds_dlq" {
   count = var.redrive_build_queue.enabled ? 1 : 0
   name  = "${var.environment}-queued-builds_dead_letter"
@@ -67,6 +106,7 @@ module "webhook" {
   lambda_zip                       = var.webhook_lambda_zip
   lambda_timeout                   = var.webhook_lambda_timeout
   logging_retention_in_days        = var.logging_retention_in_days
+  logging_kms_key_id               = var.logging_kms_key_id
 
   # labels
   enable_workflow_job_labels_check = var.runner_enable_workflow_job_labels_check
@@ -83,11 +123,12 @@ module "webhook" {
 module "runners" {
   source = "./modules/runners"
 
-  aws_region  = var.aws_region
-  vpc_id      = var.vpc_id
-  subnet_ids  = var.subnet_ids
-  environment = var.environment
-  tags        = local.tags
+  aws_region    = var.aws_region
+  aws_partition = var.aws_partition
+  vpc_id        = var.vpc_id
+  subnet_ids    = var.subnet_ids
+  environment   = var.environment
+  tags          = local.tags
 
   s3_bucket_runner_binaries   = module.runner_binaries.bucket
   s3_location_runner_binaries = local.s3_action_runner_url
@@ -132,6 +173,7 @@ module "runners" {
   lambda_subnet_ids                = var.lambda_subnet_ids
   lambda_security_group_ids        = var.lambda_security_group_ids
   logging_retention_in_days        = var.logging_retention_in_days
+  logging_kms_key_id               = var.logging_kms_key_id
   enable_cloudwatch_agent          = var.enable_cloudwatch_agent
   cloudwatch_config                = var.cloudwatch_config
   runner_log_files                 = var.runner_log_files
@@ -187,6 +229,7 @@ module "runner_binaries" {
   lambda_zip                      = var.runner_binaries_syncer_lambda_zip
   lambda_timeout                  = var.runner_binaries_syncer_lambda_timeout
   logging_retention_in_days       = var.logging_retention_in_days
+  logging_kms_key_id              = var.logging_kms_key_id
 
   server_side_encryption_configuration = var.runner_binaries_s3_sse_configuration
 

modules/runner-binaries-syncer/README.md

@@ -87,6 +87,7 @@ No modules.
 | <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
 | <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
+| <a name="input_logging_kms_key_id"></a> [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with | `string` | `null` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
 | <a name="input_role_path"></a> [role\_path](#input\_role\_path) | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
 | <a name="input_role_permissions_boundary"></a> [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -17,21 +17,21 @@
   "devDependencies": {
     "@octokit/rest": "^18.12.0",
     "@trivago/prettier-plugin-sort-imports": "^3.2.0",
-    "@types/jest": "^27.4.0",
+    "@types/jest": "^27.4.1",
     "@types/node": "^17.0.21",
     "@types/request": "^2.48.8",
     "@typescript-eslint/eslint-plugin": "^4.33.0",
     "@typescript-eslint/parser": "^4.33.0",
     "@vercel/ncc": "^0.33.3",
-    "aws-sdk": "^2.1081.0",
+    "aws-sdk": "^2.1090.0",
     "eslint": "^7.32.0",
     "eslint-plugin-prettier": "4.0.0",
     "jest": "^27.5.1",
     "jest-mock": "^27.5.1",
     "prettier": "2.5.1",
     "ts-jest": "^27.1.3",
     "ts-node-dev": "^1.1.6",
-    "typescript": "^4.5.5"
+    "typescript": "^4.6.2"
   },
   "dependencies": {
     "axios": "^0.26.0",