Skip to content

Commit 2c649a8

Browse files
IIBenIIbberriot
IIBenII
authored and
bberriot
committed
Take feedback and improve monitoring handling for autopilot clusters
1 parent 0ac5cf7 commit 2c649a8

File tree

27 files changed

+46
-135
lines changed

27 files changed

+46
-135
lines changed

‎README.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,6 @@ Then perform the following commands on the root folder:
137137
| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no |
138138
| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no |
139139
| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no |
140-
| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no |
141140
| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no |
142141
| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no |
143142
| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | <pre>[<br> {<br> "key_name": "",<br> "state": "DECRYPTED"<br> }<br>]</pre> | no |

‎autogen/main/cluster.tf.tmpl

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,16 +84,25 @@ resource "google_container_cluster" "primary" {
8484
}
8585
}
8686
{% endif %}
87+
{% if autopilot_cluster != true %}
8788
# only one of logging/monitoring_service or logging/monitoring_config can be specified
89+
{% if beta_cluster %}
8890
logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service
91+
{% else %}
92+
logging_service = local.logmon_config_is_set ? null : var.logging_service
93+
{% endif %}
8994
dynamic "logging_config" {
9095
for_each = length(var.logging_enabled_components) > 0 ? [1] : []
9196

9297
content {
9398
enable_components = var.logging_enabled_components
9499
}
95100
}
101+
{% if beta_cluster %}
96102
monitoring_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.monitoring_service
103+
{% else %}
104+
monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service
105+
{% endif %}
97106
dynamic "monitoring_config" {
98107
for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : []
99108

@@ -109,7 +118,6 @@ resource "google_container_cluster" "primary" {
109118
}
110119
}
111120
}
112-
{% if autopilot_cluster != true %}
113121
cluster_autoscaling {
114122
enabled = var.cluster_autoscaling.enabled
115123
dynamic "auto_provisioning_defaults" {

‎autogen/main/main.tf.tmpl

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,10 @@ locals {
117117
cluster_cloudrun_enabled = var.cloudrun
118118
gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }]
119119
{% endif %}
120-
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
120+
121+
{% if autopilot_cluster != true %}
122+
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
123+
{% endif %}
121124

122125
cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
123126
security_group = var.authenticator_security_group
@@ -223,10 +226,10 @@ locals {
223226
cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled
224227
cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled
225228
confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : []
229+
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
226230

227231
# /BETA features
228232
{% endif %}
229-
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
230233

231234
cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
232235
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]

‎autogen/main/variables.tf.tmpl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,11 +326,13 @@ variable "configure_ip_masq" {
326326
default = false
327327
}
328328

329+
{% if beta_cluster %}
329330
variable "cluster_telemetry_type" {
330331
type = string
331332
description = "Available options include ENABLED, DISABLED, and SYSTEM_ONLY"
332333
default = null
333334
}
335+
{% endif %}
334336

335337
variable "logging_service" {
336338
type = string
@@ -644,6 +646,7 @@ variable "timeouts" {
644646
}
645647
}
646648

649+
{% if autopilot_cluster != true %}
647650
variable "monitoring_enable_managed_prometheus" {
648651
type = bool
649652
description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled."
@@ -661,6 +664,7 @@ variable "logging_enabled_components" {
661664
description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration."
662665
default = []
663666
}
667+
{% endif %}
664668

665669
{% if beta_cluster %}
666670
{% if autopilot_cluster != true %}

‎cluster.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,15 @@ resource "google_container_cluster" "primary" {
6363
min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
6464

6565
# only one of logging/monitoring_service or logging/monitoring_config can be specified
66-
logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service
66+
logging_service = local.logmon_config_is_set ? null : var.logging_service
6767
dynamic "logging_config" {
6868
for_each = length(var.logging_enabled_components) > 0 ? [1] : []
6969

7070
content {
7171
enable_components = var.logging_enabled_components
7272
}
7373
}
74-
monitoring_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.monitoring_service
74+
monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service
7575
dynamic "monitoring_config" {
7676
for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : []
7777

‎main.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ locals {
8383
provider = null
8484
}]
8585
cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
86-
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
86+
87+
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
8788

8889
cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
8990
security_group = var.authenticator_security_group
@@ -156,7 +157,6 @@ locals {
156157
cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{
157158
workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace
158159
}]
159-
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
160160

161161
cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
162162
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]

‎modules/beta-autopilot-private-cluster/README.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,6 @@ Then perform the following commands on the root folder:
102102
| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes |
103103
| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no |
104104
| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no |
105-
| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no |
106105
| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no |
107106
| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no |
108107
| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no |
@@ -111,8 +110,6 @@ Then perform the following commands on the root folder:
111110
| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no |
112111
| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no |
113112
| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no |
114-
| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
115-
| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
116113
| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
117114
| name | The name of the cluster (required) | `string` | n/a | yes |
118115
| network | The VPC network to host the cluster in (required) | `string` | n/a | yes |

‎modules/beta-autopilot-private-cluster/cluster.tf

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -60,31 +60,6 @@ resource "google_container_cluster" "primary" {
6060

6161
min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
6262

63-
# only one of logging/monitoring_service or logging/monitoring_config can be specified
64-
logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service
65-
dynamic "logging_config" {
66-
for_each = length(var.logging_enabled_components) > 0 ? [1] : []
67-
68-
content {
69-
enable_components = var.logging_enabled_components
70-
}
71-
}
72-
monitoring_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.monitoring_service
73-
dynamic "monitoring_config" {
74-
for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : []
75-
76-
content {
77-
enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : []
78-
79-
dynamic "managed_prometheus" {
80-
for_each = var.monitoring_enable_managed_prometheus ? [1] : []
81-
82-
content {
83-
enabled = var.monitoring_enable_managed_prometheus
84-
}
85-
}
86-
}
87-
}
8863
cluster_autoscaling {
8964
dynamic "auto_provisioning_defaults" {
9065
for_each = var.create_service_account ? [1] : []

‎modules/beta-autopilot-private-cluster/main.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ locals {
6262
cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
6363
pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : []
6464

65-
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
65+
6666

6767
cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
6868
security_group = var.authenticator_security_group
@@ -130,9 +130,9 @@ locals {
130130
cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled
131131
cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled
132132
confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : []
133+
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
133134

134135
# /BETA features
135-
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
136136

137137
cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
138138
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]

‎modules/beta-autopilot-private-cluster/variables.tf

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -417,21 +417,4 @@ variable "timeouts" {
417417
}
418418
}
419419

420-
variable "monitoring_enable_managed_prometheus" {
421-
type = bool
422-
description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled."
423-
default = false
424-
}
425-
426-
variable "monitoring_enabled_components" {
427-
type = list(string)
428-
description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
429-
default = []
430-
}
431-
432-
variable "logging_enabled_components" {
433-
type = list(string)
434-
description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration."
435-
default = []
436-
}
437420

‎modules/beta-autopilot-public-cluster/README.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,15 +93,12 @@ Then perform the following commands on the root folder:
9393
| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes |
9494
| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no |
9595
| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no |
96-
| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no |
9796
| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no |
9897
| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no |
9998
| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no |
10099
| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no |
101100
| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no |
102101
| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no |
103-
| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no |
104-
| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no |
105102
| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no |
106103
| name | The name of the cluster (required) | `string` | n/a | yes |
107104
| network | The VPC network to host the cluster in (required) | `string` | n/a | yes |

‎modules/beta-autopilot-public-cluster/cluster.tf

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -60,31 +60,6 @@ resource "google_container_cluster" "primary" {
6060

6161
min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null
6262

63-
# only one of logging/monitoring_service or logging/monitoring_config can be specified
64-
logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service
65-
dynamic "logging_config" {
66-
for_each = length(var.logging_enabled_components) > 0 ? [1] : []
67-
68-
content {
69-
enable_components = var.logging_enabled_components
70-
}
71-
}
72-
monitoring_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.monitoring_service
73-
dynamic "monitoring_config" {
74-
for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : []
75-
76-
content {
77-
enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : []
78-
79-
dynamic "managed_prometheus" {
80-
for_each = var.monitoring_enable_managed_prometheus ? [1] : []
81-
82-
content {
83-
enabled = var.monitoring_enable_managed_prometheus
84-
}
85-
}
86-
}
87-
}
8863
cluster_autoscaling {
8964
dynamic "auto_provisioning_defaults" {
9065
for_each = var.create_service_account ? [1] : []

‎modules/beta-autopilot-public-cluster/main.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ locals {
6262
cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
6363
pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : []
6464

65-
logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
65+
6666

6767
cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{
6868
security_group = var.authenticator_security_group
@@ -129,9 +129,9 @@ locals {
129129
cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled
130130
cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled
131131
confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : []
132+
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
132133

133134
# /BETA features
134-
cluster_telemetry_type_is_set = var.cluster_telemetry_type != null
135135

136136
cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
137137
cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]

‎modules/beta-autopilot-public-cluster/variables.tf

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -387,21 +387,4 @@ variable "timeouts" {
387387
}
388388
}
389389

390-
variable "monitoring_enable_managed_prometheus" {
391-
type = bool
392-
description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled."
393-
default = false
394-
}
395-
396-
variable "monitoring_enabled_components" {
397-
type = list(string)
398-
description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration."
399-
default = []
400-
}
401-
402-
variable "logging_enabled_components" {
403-
type = list(string)
404-
description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration."
405-
default = []
406-
}
407390

0 commit comments

Comments
 (0)