Merge branch 'master' of https://github.com/terraform-google-modules/terraform-google-kubernetes-engine

Author: ericyz

.github/renovate.json

@@ -15,24 +15,24 @@
      ],
      "stabilityDays":0
   },
+  "separateMajorMinor":false,
   "packageRules": [
     {
       "matchPaths": ["examples/**", "test/**", ".github/**"],
       "extends": [":semanticCommitTypeAll(chore)"]
     },
+    {
+      "matchPaths": ["*", "modules/**"],
+      "extends": [":semanticCommitTypeAll(fix)"]
+    },
     {
       "matchDepTypes": ["module"],
-      "groupName": "TF modules",
-      "separateMajorMinor":false,
-      "major": {
-        "semanticCommitType": "feat!"
-      }
+      "groupName": "TF modules"
     },
     {
       "matchDepTypes": ["require"],
       "groupName": "GO modules",
-      "postUpdateOptions": ["gomodTidy"],
-      "separateMajorMinor":false
+      "postUpdateOptions": ["gomodTidy"]
     },
     {
       "matchPackageNames": ["go"],
@@ -41,8 +41,7 @@
     },
     {
       "matchPackageNames": ["google", "google-beta"],
-      "groupName": "terraform googles",
-      "separateMajorMinor": false
+      "groupName": "terraform googles"
     }
   ]
 }

.github/workflows/stale.yml

@@ -21,7 +21,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v6
+    - uses: actions/stale@v7
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days'

README.md

@@ -155,6 +155,7 @@ Then perform the following commands on the root folder:
 | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no |
 | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | <pre>[<br>  "8443",<br>  "9443",<br>  "15017"<br>]</pre> | no |
 | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no |
+| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no |
 | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no |
@@ -183,6 +184,7 @@ Then perform the following commands on the root folder:
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
+| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | <pre>[<br>  "10.0.0.0/8",<br>  "172.16.0.0/12",<br>  "192.168.0.0/16"<br>]</pre> | no |
@@ -278,6 +280,7 @@ The node_pools variable takes the following parameters:
 | tags | The list of instance tags applied to all nodes | | Required |
 | value | The value for the taint | | Required |
 | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional |
+| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional |
 
 ## windows_node_pools variable
 The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created.

autogen/main/README.md

@@ -232,6 +232,7 @@ The node_pools variable takes the following parameters:
 | tags | The list of instance tags applied to all nodes | | Required |
 | value | The value for the taint | | Required |
 | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional |
+| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional |
 
 ## windows_node_pools variable
 The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created.

autogen/main/cluster.tf.tmpl

@@ -150,6 +150,17 @@ resource "google_container_cluster" "primary" {
     }
   }
   {% endif %}
+  {% if autopilot_cluster == true %}
+  cluster_autoscaling {
+    dynamic "auto_provisioning_defaults" {
+      for_each = var.create_service_account ? [1] : []
+
+      content {
+        service_account = local.service_account
+      }
+    }
+  }
+  {% endif %}
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }
@@ -224,7 +235,6 @@ resource "google_container_cluster" "primary" {
       disabled = !var.horizontal_pod_autoscaling
     }
 
-
     {% if autopilot_cluster != true %}
     network_policy_config {
       disabled = !var.network_policy
@@ -237,6 +247,14 @@ resource "google_container_cluster" "primary" {
     gcp_filestore_csi_driver_config {
       enabled = var.filestore_csi_driver
     }
+
+    dynamic "gce_persistent_disk_csi_driver_config" {
+      for_each = local.cluster_gce_pd_csi_config
+
+      content {
+        enabled = gce_persistent_disk_csi_driver_config.value.enabled
+      }
+    }
     {% endif %}
     {% if beta_cluster and autopilot_cluster != true %}
 
@@ -253,14 +271,6 @@ resource "google_container_cluster" "primary" {
       }
     }
 
-    dynamic "gce_persistent_disk_csi_driver_config" {
-      for_each = local.cluster_gce_pd_csi_config
-
-      content {
-        enabled = gce_persistent_disk_csi_driver_config.value.enabled
-      }
-    }
-
     kalm_config {
       enabled = var.kalm_config
     }
@@ -686,6 +696,10 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_labels["all"],
       local.node_pools_labels[each.value["name"]],
     )
+    resource_labels = merge(
+      local.node_pools_resource_labels["all"],
+      local.node_pools_resource_labels[each.value["name"]],
+    )
     metadata = merge(
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {},

autogen/main/main.tf.tmpl

@@ -95,6 +95,7 @@ locals {
     enabled  = false
     provider = null
   }]
+  cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
 {% endif %}
 {% if beta_cluster and autopilot_cluster != true %}
   cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? {
@@ -109,7 +110,6 @@ locals {
     )
   ] : []
   cluster_cloudrun_enabled  = var.cloudrun
-  cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }]
   gke_backup_agent_config   = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }]
   logmon_config_is_set      = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus
 {% endif %}

autogen/main/variables.tf.tmpl

@@ -171,6 +171,16 @@ variable "node_pools_labels" {
   }
 }
 
+variable "node_pools_resource_labels" {
+  type        = map(map(string))
+  description = "Map of maps containing resource labels by node-pool name"
+
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "node_pools_metadata" {
   type        = map(map(string))
   description = "Map of maps containing node metadata by node-pool name"
@@ -605,6 +615,12 @@ variable "cluster_dns_domain" {
   default     = ""
 }
 
+variable "gce_pd_csi_driver" {
+  type        = bool
+  description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver."
+  default     = true
+}
+
 {% endif %}
 variable "timeouts" {
   type        = map(string)
@@ -713,11 +729,5 @@ variable "enable_identity_service" {
   description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API."
   default     = false
 }
-
-variable "gce_pd_csi_driver" {
-  type        = bool
-  description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver."
-  default     = false
-}
   {% endif %}
 {% endif %}

autogen/main/variables_defaults.tf.tmpl

@@ -35,6 +35,20 @@ locals {
     var.node_pools_labels
   )
 
+  node_pools_resource_labels = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    zipmap(
+      [for node_pool in var.windows_node_pools : node_pool["name"]],
+      [for node_pool in var.windows_node_pools : {}]
+    ),
+    var.node_pools_resource_labels
+  )
+
   node_pools_metadata = merge(
     { all = {} },
     { default-node-pool = {} },

autogen/main/versions.tf.tmpl

@@ -24,7 +24,7 @@ terraform {
   required_providers {
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 4.42.0, < 5.0"
+      version = ">= 4.45.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
@@ -38,7 +38,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.36.0, < 5.0"
+      version = ">= 4.45.0, < 5.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

autogen/safer-cluster/main.tf.tmpl

@@ -86,12 +86,13 @@ module "gke" {
   // If removing the default node pool, initial_node_count should be at least 1.
   initial_node_count = (var.initial_node_count == 0) ? 1 : var.initial_node_count
 
-  node_pools          = var.node_pools
-  windows_node_pools  = var.windows_node_pools
-  node_pools_labels   = var.node_pools_labels
-  node_pools_metadata = var.node_pools_metadata
-  node_pools_taints   = var.node_pools_taints
-  node_pools_tags     = var.node_pools_tags
+  node_pools                 = var.node_pools
+  windows_node_pools         = var.windows_node_pools
+  node_pools_labels          = var.node_pools_labels
+  node_pools_resource_labels = var.node_pools_resource_labels
+  node_pools_metadata        = var.node_pools_metadata
+  node_pools_taints          = var.node_pools_taints
+  node_pools_tags            = var.node_pools_tags
 
   node_pools_oauth_scopes = var.node_pools_oauth_scopes
 

autogen/safer-cluster/variables.tf.tmpl

@@ -168,6 +168,16 @@ variable "node_pools_labels" {
   }
 }
 
+variable "node_pools_resource_labels" {
+  type        = map(map(string))
+  description = "Map of maps containing resource labels by node-pool name"
+
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "node_pools_metadata" {
   type        = map(map(string))
   description = "Map of maps containing node metadata by node-pool name"

build/int.cloudbuild.yaml

@@ -401,9 +401,26 @@ steps:
     - verify private-zonal-with-networking
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration']
-
-
-
+- id: init simple-autopilot-private-non-default-sa
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage init --verbose']
+- id: apply simple-autopilot-private-non-default-sa
+  waitFor:
+    - init simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage apply --verbose']
+- id: verify simple-autopilot-private-non-default-sa
+  waitFor:
+    - apply simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage verify --verbose']
+- id: teardown simple-autopilot-private-non-default-sa
+  waitFor:
+    - verify simple-autopilot-private-non-default-sa
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

cluster.tf

@@ -131,7 +131,6 @@ resource "google_container_cluster" "primary" {
       disabled = !var.horizontal_pod_autoscaling
     }
 
-
     network_policy_config {
       disabled = !var.network_policy
     }
@@ -143,6 +142,14 @@ resource "google_container_cluster" "primary" {
     gcp_filestore_csi_driver_config {
       enabled = var.filestore_csi_driver
     }
+
+    dynamic "gce_persistent_disk_csi_driver_config" {
+      for_each = local.cluster_gce_pd_csi_config
+
+      content {
+        enabled = gce_persistent_disk_csi_driver_config.value.enabled
+      }
+    }
   }
 
   datapath_provider = var.datapath_provider
@@ -377,6 +384,10 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_labels["all"],
       local.node_pools_labels[each.value["name"]],
     )
+    resource_labels = merge(
+      local.node_pools_resource_labels["all"],
+      local.node_pools_resource_labels[each.value["name"]],
+    )
     metadata = merge(
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {},
@@ -531,6 +542,10 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_labels["all"],
       local.node_pools_labels[each.value["name"]],
     )
+    resource_labels = merge(
+      local.node_pools_resource_labels["all"],
+      local.node_pools_resource_labels[each.value["name"]],
+    )
     metadata = merge(
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
       lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {},