Skip to content

Commit 0411c32

Browse files
wyardleywyardley
committed
update to support autopilot and node_config_defaults
1 parent f121604 commit 0411c32

File tree

32 files changed

+209
-43
lines changed

32 files changed

+209
-43
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ Then perform the following commands on the root folder:
189189
| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
190190
| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
191191
| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
192+
| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
192193
| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
193194
| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
194195
| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

‎autogen/main/cluster.tf.tmpl

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -270,8 +270,12 @@ resource "google_container_cluster" "primary" {
270270
}
271271
{% if autopilot_cluster %}
272272
dynamic "node_pool_auto_config" {
273-
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
273+
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled !="" ? [1] : []
274274
content {
275+
dynamic "kubelet_config" {
276+
for_each = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
277+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
278+
}
275279
network_tags {
276280
tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
277281
}
@@ -660,7 +664,6 @@ resource "google_container_cluster" "primary" {
660664
}
661665
}
662666
}
663-
{% if beta_cluster %}
664667

665668
node_pool_defaults {
666669
node_config_defaults {
@@ -677,12 +680,12 @@ resource "google_container_cluster" "primary" {
677680
gcfs_config {
678681
enabled = var.enable_gcfs
679682
}
683+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
680684
{% endif %}
681685
}
682686
}
683-
{% endif %}
684-
{% if beta_cluster %}
685687

688+
{% if beta_cluster %}
686689
depends_on = [google_project_iam_member.service_agent]
687690
{% endif %}
688691
}
@@ -1047,7 +1050,7 @@ resource "google_container_node_pool" "windows_pools" {
10471050
cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
10481051
cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
10491052
cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
1050-
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
1053+
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
10511054
pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
10521055
}
10531056
}

‎autogen/main/variables.tf.tmpl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ variable "service_external_ips" {
102102
default = false
103103
}
104104

105+
variable "insecure_kubelet_readonly_port_enabled" {
106+
type = string
107+
description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
108+
default = ""
109+
110+
validation {
111+
condition = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
112+
error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
113+
}
114+
}
115+
105116
{% if autopilot_cluster != true %}
106117
variable "datapath_provider" {
107118
type = string

‎cluster.tf

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,16 @@ resource "google_container_cluster" "primary" {
499499
}
500500
}
501501
}
502+
503+
node_pool_defaults {
504+
node_config_defaults {
505+
gcfs_config {
506+
enabled = var.enable_gcfs
507+
}
508+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
509+
}
510+
}
511+
502512
}
503513
/******************************************
504514
Create Container Cluster node pools
@@ -745,7 +755,7 @@ resource "google_container_node_pool" "pools" {
745755
cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
746756
cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
747757
cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
748-
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
758+
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
749759
pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
750760
}
751761
}
@@ -1028,7 +1038,7 @@ resource "google_container_node_pool" "windows_pools" {
10281038
cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
10291039
cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
10301040
cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
1031-
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
1041+
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
10321042
pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
10331043
}
10341044
}

‎examples/node_pool/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ module "gke" {
7979
sandbox_enabled = true
8080
cpu_manager_policy = "static"
8181
cpu_cfs_quota = true
82-
insecure_kubelet_readonly_port_enabled = "TRUE"
82+
insecure_kubelet_readonly_port_enabled = "FALSE"
8383
local_ssd_ephemeral_count = 2
8484
pod_pids_limit = 4096
8585
},

‎examples/node_pool_update_variant/main.tf

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,12 @@ module "gke" {
6161

6262
node_pools = [
6363
{
64-
name = "pool-01"
65-
min_count = 1
66-
max_count = 2
67-
service_account = var.compute_engine_service_account
68-
auto_upgrade = true
64+
name = "pool-01"
65+
min_count = 1
66+
max_count = 2
67+
service_account = var.compute_engine_service_account
68+
auto_upgrade = true
69+
insecure_kubelet_readonly_port_enabled = "FALSE"
6970
},
7071
{
7172
name = "pool-02"

‎examples/simple_autopilot_public/main.tf

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,23 @@ module "gke" {
3636
source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-public-cluster"
3737
version = "~> 33.0"
3838

39-
project_id = var.project_id
40-
name = "${local.cluster_type}-cluster"
41-
regional = true
42-
region = var.region
43-
network = module.gcp-network.network_name
44-
subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
45-
ip_range_pods = local.pods_range_name
46-
ip_range_services = local.svc_range_name
47-
release_channel = "RAPID"
48-
enable_vertical_pod_autoscaling = true
49-
network_tags = [local.cluster_type]
50-
deletion_protection = false
51-
enable_l4_ilb_subsetting = true
52-
gcs_fuse_csi_driver = true
53-
stateful_ha = false
54-
gke_backup_agent_config = false
39+
project_id = var.project_id
40+
name = "${local.cluster_type}-cluster"
41+
regional = true
42+
region = var.region
43+
network = module.gcp-network.network_name
44+
subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
45+
ip_range_pods = local.pods_range_name
46+
ip_range_services = local.svc_range_name
47+
release_channel = "RAPID"
48+
enable_vertical_pod_autoscaling = true
49+
network_tags = [local.cluster_type]
50+
deletion_protection = false
51+
enable_l4_ilb_subsetting = true
52+
gcs_fuse_csi_driver = true
53+
insecure_kubelet_readonly_port_enabled = "FALSE"
54+
stateful_ha = false
55+
gke_backup_agent_config = false
5556
ray_operator_config = {
5657
enabled = true
5758
logging_enabled = true

‎modules/beta-autopilot-private-cluster/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ Then perform the following commands on the root folder:
113113
| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
114114
| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
115115
| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
116+
| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
116117
| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
117118
| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
118119
| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

‎modules/beta-autopilot-private-cluster/cluster.tf

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,12 @@ resource "google_container_cluster" "primary" {
118118
}
119119
}
120120
dynamic "node_pool_auto_config" {
121-
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
121+
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
122122
content {
123+
dynamic "kubelet_config" {
124+
for_each = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
125+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
126+
}
123127
network_tags {
124128
tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
125129
}

‎modules/beta-autopilot-private-cluster/variables.tf

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ variable "service_external_ips" {
102102
default = false
103103
}
104104

105+
variable "insecure_kubelet_readonly_port_enabled" {
106+
type = string
107+
description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
108+
default = ""
109+
110+
validation {
111+
condition = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
112+
error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
113+
}
114+
}
115+
105116
variable "maintenance_start_time" {
106117
type = string
107118
description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"

‎modules/beta-autopilot-public-cluster/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ Then perform the following commands on the root folder:
104104
| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
105105
| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
106106
| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
107+
| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
107108
| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
108109
| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
109110
| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

‎modules/beta-autopilot-public-cluster/cluster.tf

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,12 @@ resource "google_container_cluster" "primary" {
118118
}
119119
}
120120
dynamic "node_pool_auto_config" {
121-
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
121+
for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
122122
content {
123+
dynamic "kubelet_config" {
124+
for_each = var.insecure_kubelet_readonly_port_enabled != "" ? [1] : []
125+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled
126+
}
123127
network_tags {
124128
tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
125129
}

‎modules/beta-autopilot-public-cluster/variables.tf

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ variable "service_external_ips" {
102102
default = false
103103
}
104104

105+
variable "insecure_kubelet_readonly_port_enabled" {
106+
type = string
107+
description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
108+
default = ""
109+
110+
validation {
111+
condition = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
112+
error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
113+
}
114+
}
115+
105116
variable "maintenance_start_time" {
106117
type = string
107118
description = "Time window specified for daily or recurring maintenance operations in RFC3339 format"

‎modules/beta-private-cluster-update-variant/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ Then perform the following commands on the root folder:
232232
| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
233233
| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
234234
| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
235+
| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
235236
| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
236237
| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
237238
| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

‎modules/beta-private-cluster-update-variant/cluster.tf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,7 @@ resource "google_container_cluster" "primary" {
580580
gcfs_config {
581581
enabled = var.enable_gcfs
582582
}
583+
insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null
583584
}
584585
}
585586

@@ -918,7 +919,7 @@ resource "google_container_node_pool" "pools" {
918919
cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
919920
cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
920921
cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
921-
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
922+
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
922923
pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
923924
}
924925
}
@@ -1215,7 +1216,7 @@ resource "google_container_node_pool" "windows_pools" {
12151216
cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static")
12161217
cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null)
12171218
cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null)
1218-
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null)
1219+
insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled != "" ? var.insecure_kubelet_readonly_port_enabled : null)
12191220
pod_pids_limit = lookup(each.value, "pod_pids_limit", null)
12201221
}
12211222
}

‎modules/beta-private-cluster-update-variant/variables.tf

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ variable "service_external_ips" {
102102
default = false
103103
}
104104

105+
variable "insecure_kubelet_readonly_port_enabled" {
106+
type = string
107+
description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`."
108+
default = ""
109+
110+
validation {
111+
condition = contains(["FALSE", "TRUE", ""], var.insecure_kubelet_readonly_port_enabled)
112+
error_message = "The node_metadata value must be one of \"TRUE\", \"FALSE\", or empty string (\"\")."
113+
}
114+
}
115+
105116
variable "datapath_provider" {
106117
type = string
107118
description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature."

‎modules/beta-private-cluster/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ Then perform the following commands on the root folder:
210210
| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
211211
| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no |
212212
| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no |
213+
| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `string` | `""` | no |
213214
| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no |
214215
| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no |
215216
| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes |

0 commit comments

Comments
 (0)