feat(log): groups per type of logging (#476)

* Refactor to loggroup per tuype of logging

* Update modules/runners/variables.tf

Co-authored-by: Gertjan Maas <[email protected]>

* fix: Queue visibility time has to be >= lambda timeout (#467)

* AWS SQS queue visibility times have to be >= the timeout of the associated lambda
* error to do so will result in a terraform provisioning error during initial resource creation (apparently not a modification)
* hence, setting module.runners.aws_sqs_queue.queued_builds.visibility_timeout_seconds  to module.runners.module.runners.aws_lambda_function.scale_up.timeout

* Typo

Co-authored-by: Gertjan Maas <[email protected]>
Co-authored-by: Johannes Nicolai <[email protected]>

Author: 3 people

README.md

@@ -325,7 +325,7 @@ No requirements.
 | lambda\_s3\_bucket | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
 | lambda\_security\_group\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | lambda\_subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
-| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
+| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | manage\_kms\_key | Let the module manage the KMS key. | `bool` | `true` | no |
 | minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
 | role\_path | The path that will be added to role path for created roles, if not set the environment name will be used. | `string` | `null` | no |
@@ -337,13 +337,13 @@ No requirements.
 | runner\_binaries\_syncer\_lambda\_zip | File location of the binaries sync lambda zip file. | `string` | `null` | no |
 | runner\_extra\_labels | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
 | runner\_iam\_role\_managed\_policy\_arns | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
-| runner\_log\_files | (optional) List of logfiles to send to cloudwatch. | <pre>list(object({<br>    file_path       = string<br>    log_stream_name = string<br>  }))</pre> | <pre>[<br>  {<br>    "file_path": "/var/log/messages",<br>    "log_stream_name": "{instance_id}/messages"<br>  },<br>  {<br>    "file_path": "/var/log/user-data.log",<br>    "log_stream_name": "{instance_id}/user_data"<br>  },<br>  {<br>    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br>    "log_stream_name": "{instance_id}/runner"<br>  }<br>]</pre> | no |
+| runner\_log\_files | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | <pre>[<br>  {<br>    "file_path": "/var/log/messages",<br>    "log_group_name": "messages",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  },<br>  {<br>    "file_path": "/var/log/user-data.log",<br>    "log_group_name": "user_data",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  },<br>  {<br>    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br>    "log_group_name": "runner",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  }<br>]</pre> | no |
 | runners\_lambda\_s3\_key | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
 | runners\_lambda\_s3\_object\_version | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
 | runners\_lambda\_zip | File location of the lambda zip file for scaling runners. | `string` | `null` | no |
 | runners\_maximum\_count | The maximum number of runners that will be created. | `number` | `3` | no |
-| runners\_scale\_down\_lambda\_timeout | Time out for the scale up lambda in seconds. | `number` | `60` | no |
-| runners\_scale\_up\_lambda\_timeout | Time out for the scale down lambda in seconds. | `number` | `180` | no |
+| runners\_scale\_down\_lambda\_timeout | Time out for the scale down lambda in seconds. | `number` | `60` | no |
+| runners\_scale\_up\_lambda\_timeout | Time out for the scale up lambda in seconds. | `number` | `180` | no |
 | scale\_down\_schedule\_expression | Scheduler expression to check every x for scale down. | `string` | `"cron(*/5 * * * ? *)"` | no |
 | subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes |
 | syncer\_lambda\_s3\_key | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |

modules/runners/README.md

@@ -85,7 +85,7 @@ No requirements.
 | lambda\_timeout\_scale\_down | Time out for the scale down lambda in seconds. | `number` | `60` | no |
 | lambda\_timeout\_scale\_up | Time out for the scale up lambda in seconds. | `number` | `60` | no |
 | lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
-| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
+| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | market\_options | Market options for the action runner instances. | `string` | `"spot"` | no |
 | minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
 | overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` override the `Name` tag spot instances created by the runner agent. | `map(string)` | <pre>{<br>  "name_runner": "",<br>  "name_sg": ""<br>}</pre> | no |
@@ -96,7 +96,7 @@ No requirements.
 | runner\_as\_root | Run the action runner under the root user. | `bool` | `false` | no |
 | runner\_extra\_labels | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
 | runner\_iam\_role\_managed\_policy\_arns | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
-| runner\_log\_files | (optional) List of logfiles to send to cloudwatch. | <pre>list(object({<br>    file_path       = string<br>    log_stream_name = string<br>  }))</pre> | <pre>[<br>  {<br>    "file_path": "/var/log/messages",<br>    "log_stream_name": "{instance_id}/messages"<br>  },<br>  {<br>    "file_path": "/var/log/user-data.log",<br>    "log_stream_name": "{instance_id}/user_data"<br>  },<br>  {<br>    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br>    "log_stream_name": "{instance_id}/runner"<br>  }<br>]</pre> | no |
+| runner\_log\_files | (optional) List of logfiles to send to cloudwatch, will onlybe usded if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: module will prefix the log group with `/github-self-hosted-runners/<var.environment>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | <pre>[<br>  {<br>    "file_path": "/var/log/messages",<br>    "log_group_name": "messages",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  },<br>  {<br>    "file_path": "/var/log/user-data.log",<br>    "log_group_name": "user_data",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  },<br>  {<br>    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",<br>    "log_group_name": "runner",<br>    "log_stream_name": "{instance_id}",<br>    "prefix_log_group": true<br>  }<br>]</pre> | no |
 | runners\_lambda\_s3\_key | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
 | runners\_lambda\_s3\_object\_version | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
 | runners\_maximum\_count | The maximum number of runners that will be created. | `number` | `3` | no |

modules/runners/logging.tf

@@ -1,7 +1,15 @@
 locals {
-  logfiles = var.enable_cloudwatch_agent ? [for l in var.runner_log_files : merge(l, { "log_group_name" : aws_cloudwatch_log_group.runners[0].name })] : []
+  logfiles = var.enable_cloudwatch_agent ? [for l in var.runner_log_files : {
+    "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.environment}/${l.log_group_name}" : "/${l.log_group_name}"
+    "log_stream_name" : l.log_stream_name
+    "file_path" : l.file_path
+  }] : []
+
+  loggroups_names = distinct([for l in local.logfiles : l.log_group_name])
+
 }
 
+
 resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" {
   count = var.enable_cloudwatch_agent ? 1 : 0
   name  = "${var.environment}-cloudwatch_agent_config_runner"
@@ -12,9 +20,9 @@ resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" {
   tags = local.tags
 }
 
-resource "aws_cloudwatch_log_group" "runners" {
-  count             = var.enable_cloudwatch_agent ? 1 : 0
-  name              = "${var.environment}/runners"
+resource "aws_cloudwatch_log_group" "gh_runners" {
+  count             = length(local.loggroups_names)
+  name              = local.loggroups_names[count.index]
   retention_in_days = var.logging_retention_in_days
   tags              = local.tags
 }

modules/runners/variables.tf

@@ -210,7 +210,7 @@ variable "idle_config" {
 variable "logging_retention_in_days" {
   description = "Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653."
   type        = number
-  default     = 7
+  default     = 180
 }
 
 variable "enable_ssm_on_runners" {
@@ -258,23 +258,31 @@ variable "cloudwatch_config" {
 }
 
 variable "runner_log_files" {
-  description = "(optional) List of logfiles to send to cloudwatch."
+  description = "(optional) List of logfiles to send to cloudwatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.environment>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream."
   type = list(object({
-    file_path       = string
-    log_stream_name = string
+    log_group_name   = string
+    prefix_log_group = bool
+    file_path        = string
+    log_stream_name  = string
   }))
   default = [
     {
+      "log_group_name" : "messages",
+      "prefix_log_group" : true,
       "file_path" : "/var/log/messages",
-      "log_stream_name" : "{instance_id}/messages"
+      "log_stream_name" : "{instance_id}"
     },
     {
+      "log_group_name" : "user_data",
+      "prefix_log_group" : true,
       "file_path" : "/var/log/user-data.log",
-      "log_stream_name" : "{instance_id}/user_data"
+      "log_stream_name" : "{instance_id}"
     },
     {
+      "log_group_name" : "runner",
+      "prefix_log_group" : true,
       "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
-      "log_stream_name" : "{instance_id}/runner"
+      "log_stream_name" : "{instance_id}"
     }
   ]
 }

variables.tf

@@ -190,7 +190,7 @@ variable "enable_ssm_on_runners" {
 variable "logging_retention_in_days" {
   description = "Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653."
   type        = number
-  default     = 7
+  default     = 180
 }
 
 variable "runner_allow_prerelease_binaries" {
@@ -276,23 +276,31 @@ variable "cloudwatch_config" {
 }
 
 variable "runner_log_files" {
-  description = "(optional) List of logfiles to send to cloudwatch."
+  description = "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
   type = list(object({
-    file_path       = string
-    log_stream_name = string
+    log_group_name   = string
+    prefix_log_group = bool
+    file_path        = string
+    log_stream_name  = string
   }))
   default = [
     {
+      "log_group_name" : "messages",
+      "prefix_log_group" : true,
       "file_path" : "/var/log/messages",
-      "log_stream_name" : "{instance_id}/messages"
+      "log_stream_name" : "{instance_id}"
     },
     {
+      "log_group_name" : "user_data",
+      "prefix_log_group" : true,
       "file_path" : "/var/log/user-data.log",
-      "log_stream_name" : "{instance_id}/user_data"
+      "log_stream_name" : "{instance_id}"
     },
     {
+      "log_group_name" : "runner",
+      "prefix_log_group" : true,
       "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
-      "log_stream_name" : "{instance_id}/runner"
+      "log_stream_name" : "{instance_id}"
     }
   ]
 }