Skip to content

Commit 62d79fa

Browse files
morgantemorgante
authored
Merge pull request #256 from asproul/create-before-destroy-nodepools
Allow a node pool to be created before it is destroyed
2 parents c696058 + e80b9c2 commit 62d79fa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+4347
-0
lines changed

‎README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,22 @@ Then perform the following commands on the root folder:
108108
- `terraform apply` to apply the infrastructure build
109109
- `terraform destroy` to destroy the built infrastructure
110110

111+
## Upgrade to v3.0.0
112+
113+
v3.0.0 is a breaking release. Refer to the
114+
[Upgrading to v3.0 guide][upgrading-to-v3.0] for details.
115+
116+
## Upgrade to v2.0.0
117+
118+
v2.0.0 is a breaking release. Refer to the
119+
[Upgrading to v2.0 guide][upgrading-to-v2.0] for details.
120+
121+
## Upgrade to v1.0.0
122+
123+
Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths.
124+
125+
In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster.
126+
111127
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
112128
## Inputs
113129

‎autogen/cluster.tf

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,92 @@ resource "google_container_cluster" "primary" {
219219
/******************************************
220220
Create Container Cluster node pools
221221
*****************************************/
222+
{% if update_variant %}
223+
locals {
224+
force_node_pool_recreation_resources = [
225+
"disk_size_gb",
226+
"disk_type",
227+
"accelerator_count",
228+
"accelerator_type",
229+
"local_ssd_count",
230+
"machine_type",
231+
"preemptible",
232+
"service_account",
233+
]
234+
}
235+
236+
# This keepers list is based on the terraform google provider schemaNodeConfig
237+
# resources where "ForceNew" is "true". schemaNodeConfig can be found in node_config.go at
238+
# https://github.com/terraform-providers/terraform-provider-google/blob/master/google/node_config.go#L22
239+
resource "random_id" "name" {
240+
count = length(var.node_pools)
241+
byte_length = 2
242+
prefix = format("%s-", lookup(var.node_pools[count.index], "name"))
243+
keepers = merge(
244+
zipmap(
245+
local.force_node_pool_recreation_resources,
246+
[for keeper in local.force_node_pool_recreation_resources : lookup(var.node_pools[count.index], keeper, "")]
247+
),
248+
{
249+
labels = join(",",
250+
sort(
251+
concat(
252+
keys(var.node_pools_labels["all"]),
253+
values(var.node_pools_labels["all"]),
254+
keys(var.node_pools_labels[var.node_pools[count.index]["name"]]),
255+
values(var.node_pools_labels[var.node_pools[count.index]["name"]])
256+
)
257+
)
258+
)
259+
},
260+
{
261+
metadata = join(",",
262+
sort(
263+
concat(
264+
keys(var.node_pools_metadata["all"]),
265+
values(var.node_pools_metadata["all"]),
266+
keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]),
267+
values(var.node_pools_metadata[var.node_pools[count.index]["name"]])
268+
)
269+
)
270+
)
271+
},
272+
{
273+
oauth_scopes = join(",",
274+
sort(
275+
concat(
276+
var.node_pools_oauth_scopes["all"],
277+
var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]]
278+
)
279+
)
280+
)
281+
},
282+
{
283+
tags = join(",",
284+
sort(
285+
concat(
286+
var.node_pools_tags["all"],
287+
var.node_pools_tags[var.node_pools[count.index]["name"]]
288+
)
289+
)
290+
)
291+
}
292+
)
293+
}
294+
295+
{% endif %}
222296
resource "google_container_node_pool" "pools" {
223297
{% if beta_cluster %}
224298
provider = google-beta
225299
{% else %}
226300
provider = google
227301
{% endif %}
228302
count = length(var.node_pools)
303+
{% if update_variant %}
304+
name = random_id.name.*.hex[count.index]
305+
{% else %}
229306
name = var.node_pools[count.index]["name"]
307+
{% endif %}
230308
project = var.project_id
231309
location = local.location
232310
cluster = google_container_cluster.primary.name
@@ -342,6 +420,9 @@ resource "google_container_node_pool" "pools" {
342420

343421
lifecycle {
344422
ignore_changes = [initial_node_count]
423+
{% if update_variant %}
424+
create_before_destroy = true
425+
{% endif %}
345426
}
346427

347428
timeouts {

‎examples/node_pool_update_variant/README.md

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Node Pool Cluster
2+
3+
This example illustrates how to create a cluster with multiple custom node-pool configurations with node labels, taints, and network tags.
4+
5+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
6+
## Inputs
7+
8+
| Name | Description | Type | Default | Required |
9+
|------|-------------|:----:|:-----:|:-----:|
10+
| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
11+
| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
12+
| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
13+
| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
14+
| network | The VPC network to host the cluster in | string | n/a | yes |
15+
| project\_id | The project ID to host the cluster in | string | n/a | yes |
16+
| region | The region to host the cluster in | string | n/a | yes |
17+
| subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
18+
| zones | The zone to host the cluster in (required if is a zonal cluster) | list(string) | n/a | yes |
19+
20+
## Outputs
21+
22+
| Name | Description |
23+
|------|-------------|
24+
| ca\_certificate | |
25+
| client\_token | |
26+
| cluster\_name | Cluster name |
27+
| ip\_range\_pods | The secondary IP range used for pods |
28+
| ip\_range\_services | The secondary IP range used for services |
29+
| kubernetes\_endpoint | |
30+
| location | |
31+
| master\_kubernetes\_version | The master Kubernetes version |
32+
| network | |
33+
| project\_id | |
34+
| region | |
35+
| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
36+
| subnetwork | |
37+
| zones | List of zones in which the cluster resides |
38+
39+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
40+
41+
To provision this example, run the following from within this directory:
42+
- `terraform init` to get the plugins
43+
- `terraform plan` to see the infrastructure plan
44+
- `terraform apply` to apply the infrastructure build
45+
- `terraform destroy` to destroy the built infrastructure

‎examples/node_pool_update_variant/data/shutdown-script.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash -e
2+
3+
# Copyright 2018 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
kubectl --kubeconfig=/var/lib/kubelet/kubeconfig drain --force=true --ignore-daemonsets=true --delete-local-data "$HOSTNAME"

‎examples/node_pool_update_variant/main.tf

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
locals {
18+
cluster_type = "node-pool-update-variant"
19+
}
20+
21+
provider "google" {
22+
version = "~> 2.12.0"
23+
region = var.region
24+
}
25+
26+
data "google_compute_subnetwork" "subnetwork" {
27+
name = var.subnetwork
28+
project = var.project_id
29+
region = var.region
30+
}
31+
32+
module "gke" {
33+
source = "../../modules/private-cluster-update-variant"
34+
project_id = var.project_id
35+
name = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
36+
regional = false
37+
region = var.region
38+
zones = var.zones
39+
network = var.network
40+
subnetwork = var.subnetwork
41+
ip_range_pods = var.ip_range_pods
42+
ip_range_services = var.ip_range_services
43+
create_service_account = false
44+
service_account = var.compute_engine_service_account
45+
enable_private_endpoint = true
46+
enable_private_nodes = true
47+
master_ipv4_cidr_block = "172.16.0.0/28"
48+
49+
master_authorized_networks_config = [
50+
{
51+
cidr_blocks = [
52+
{
53+
cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range
54+
display_name = "VPC"
55+
},
56+
]
57+
},
58+
]
59+
60+
node_pools = [
61+
{
62+
name = "pool-01"
63+
min_count = 1
64+
max_count = 2
65+
service_account = var.compute_engine_service_account
66+
auto_upgrade = true
67+
},
68+
{
69+
name = "pool-02"
70+
machine_type = "n1-standard-2"
71+
min_count = 1
72+
max_count = 2
73+
disk_size_gb = 30
74+
disk_type = "pd-standard"
75+
accelerator_count = 1
76+
accelerator_type = "nvidia-tesla-p4"
77+
image_type = "COS"
78+
auto_repair = false
79+
service_account = var.compute_engine_service_account
80+
},
81+
]
82+
83+
node_pools_oauth_scopes = {
84+
all = []
85+
pool-01 = []
86+
pool-02 = []
87+
}
88+
89+
node_pools_metadata = {
90+
all = {}
91+
pool-01 = {
92+
shutdown-script = file("${path.module}/data/shutdown-script.sh")
93+
}
94+
pool-02 = {}
95+
}
96+
97+
node_pools_labels = {
98+
all = {
99+
all-pools-example = true
100+
}
101+
pool-01 = {
102+
pool-01-example = true
103+
}
104+
pool-02 = {}
105+
}
106+
107+
node_pools_tags = {
108+
all = [
109+
"all-node-example",
110+
]
111+
pool-01 = [
112+
"pool-01-example",
113+
]
114+
pool-02 = []
115+
}
116+
}
117+
118+
data "google_client_config" "default" {
119+
}

‎examples/node_pool_update_variant/outputs.tf

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
output "kubernetes_endpoint" {
18+
sensitive = true
19+
value = module.gke.endpoint
20+
}
21+
22+
output "client_token" {
23+
sensitive = true
24+
value = base64encode(data.google_client_config.default.access_token)
25+
}
26+
27+
output "ca_certificate" {
28+
value = module.gke.ca_certificate
29+
}
30+
31+
output "service_account" {
32+
description = "The service account to default running nodes as if not overridden in `node_pools`."
33+
value = module.gke.service_account
34+
}
35+

0 commit comments

Comments
 (0)