Skip to content

Commit bcd5e03

Browse files
splichybharathkkb
splichy
and
bharathkkb
authored
feat: add all pod_ranges to cluster firewall rules and add missing shadow rules (#1480)
Co-authored-by: Bharath KKB <[email protected]>
1 parent e7566c5 commit bcd5e03

File tree

41 files changed

+910
-107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+910
-107
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ Then perform the following commands on the root folder:
198198
| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
199199
| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
200200
| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no |
201+
| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | <pre>object({<br> metadata = string<br> })</pre> | <pre>{<br> "metadata": "INCLUDE_ALL_METADATA"<br>}</pre> | no |
201202
| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
202203
| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
203204
| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |

‎autogen/main/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ The node_pools variable takes the following parameters:
218218
| name | The name of the node pool | | Required |
219219
{% if beta_cluster %}
220220
| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional |
221-
| pod_range | The ID of the secondary range for pod IPs. | | Optional |
221+
| pod_range | The name of the secondary range for pod IPs. | | Optional |
222222
{% endif %}
223223
| node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required |
224224
| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional |

‎autogen/main/firewall.tf.tmpl

Lines changed: 73 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = [
37+
destination_ranges = concat([
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
40-
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
]
40+
],
41+
local.pod_all_ip_ranges
42+
)
4243

4344
# Allow all possible protocols
4445
allow { protocol = "tcp" }
@@ -143,7 +144,7 @@ resource "google_compute_firewall" "shadow_allow_pods" {
143144
priority = var.shadow_firewall_rules_priority
144145
direction = "INGRESS"
145146

146-
source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
147+
source_ranges = local.pod_all_ip_ranges
147148
target_tags = [local.cluster_network_tag]
148149

149150
# Allow all possible protocols
@@ -154,8 +155,11 @@ resource "google_compute_firewall" "shadow_allow_pods" {
154155
allow { protocol = "esp" }
155156
allow { protocol = "ah" }
156157

157-
log_config {
158-
metadata = "INCLUDE_ALL_METADATA"
158+
dynamic "log_config" {
159+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
160+
content {
161+
metadata = log_config.value.metadata
162+
}
159163
}
160164
}
161165

@@ -177,8 +181,11 @@ resource "google_compute_firewall" "shadow_allow_master" {
177181
ports = ["10250", "443"]
178182
}
179183

180-
log_config {
181-
metadata = "INCLUDE_ALL_METADATA"
184+
dynamic "log_config" {
185+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
186+
content {
187+
metadata = log_config.value.metadata
188+
}
182189
}
183190
}
184191

@@ -209,7 +216,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" {
209216
ports = ["1-65535"]
210217
}
211218

212-
log_config {
213-
metadata = "INCLUDE_ALL_METADATA"
219+
dynamic "log_config" {
220+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
221+
content {
222+
metadata = log_config.value.metadata
223+
}
224+
}
225+
}
226+
227+
resource "google_compute_firewall" "shadow_allow_inkubelet" {
228+
count = var.add_shadow_firewall_rules ? 1 : 0
229+
230+
name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet"
231+
description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet."
232+
project = local.network_project_id
233+
network = var.network
234+
priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999
235+
direction = "INGRESS"
236+
237+
source_ranges = local.pod_all_ip_ranges
238+
source_tags = [local.cluster_network_tag]
239+
target_tags = [local.cluster_network_tag]
240+
241+
allow {
242+
protocol = "tcp"
243+
ports = ["10255"]
244+
}
245+
246+
dynamic "log_config" {
247+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
248+
content {
249+
metadata = log_config.value.metadata
250+
}
251+
}
252+
}
253+
254+
resource "google_compute_firewall" "shadow_deny_exkubelet" {
255+
count = var.add_shadow_firewall_rules ? 1 : 0
256+
257+
name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet"
258+
description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet."
259+
project = local.network_project_id
260+
network = var.network
261+
priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000
262+
direction = "INGRESS"
263+
264+
source_ranges = ["0.0.0.0/0"]
265+
target_tags = [local.cluster_network_tag]
266+
267+
deny {
268+
protocol = "tcp"
269+
ports = ["10255"]
270+
}
271+
272+
dynamic "log_config" {
273+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
274+
content {
275+
metadata = log_config.value.metadata
276+
}
214277
}
215278
}

‎autogen/main/main.tf.tmpl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ locals {
8686

8787
cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
8888
cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
89+
{% if autopilot_cluster != true %}
90+
pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools): local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0] )) : []
91+
{% else %}
92+
pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : []
93+
{% endif %}
8994

9095
{% if autopilot_cluster != true %}
9196
cluster_network_policy = var.network_policy ? [{

‎autogen/main/variables.tf.tmpl

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -485,9 +485,23 @@ variable "add_shadow_firewall_rules" {
485485
}
486486

487487
variable "shadow_firewall_rules_priority" {
488-
type = number
488+
type = number
489489
description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000."
490-
default = 999
490+
default = 999
491+
validation {
492+
condition = var.shadow_firewall_rules_priority < 1000
493+
error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)."
494+
}
495+
}
496+
497+
variable "shadow_firewall_rules_log_config" {
498+
type = object({
499+
metadata = string
500+
})
501+
description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging."
502+
default = {
503+
metadata = "INCLUDE_ALL_METADATA"
504+
}
491505
}
492506

493507
{% if beta_cluster %}

‎docs/private_clusters.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,16 @@ If you are using these features with a private cluster, you will need to either:
2020

2121
If you are going to isolate your GKE private clusters from internet access you could check [this guide](https://medium.com/google-cloud/completely-private-gke-clusters-with-no-internet-connectivity-945fffae1ccd) and the associated [repo](https://github.com/andreyk-code/no-inet-gke-cluster).
2222

23+
## Discontiguous multi-Pod CIDR
24+
If you are going to use [discontiguous multi-Pod CIDR](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr) it can happen that GKE robot will not update `gke-[cluster-name]-[cluster-hash]-all` and other firewall rules automatically when you add a new node pool (as stated in [documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr#modified_firewall_rule)). You can prevent this from happening, by using a workaround with shadow firewall rules:
25+
```
26+
module "gke" {
27+
...
28+
add_shadow_firewall_rules = true
29+
shadow_firewall_rules_log_config = null # to save some $ on logs
30+
}
31+
```
32+
2333
## Troubleshooting
2434

2535
### Master Authorized Network

‎firewall.tf

Lines changed: 73 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = [
37+
destination_ranges = concat([
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
40-
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
]
40+
],
41+
local.pod_all_ip_ranges
42+
)
4243

4344
# Allow all possible protocols
4445
allow { protocol = "tcp" }
@@ -99,7 +100,7 @@ resource "google_compute_firewall" "shadow_allow_pods" {
99100
priority = var.shadow_firewall_rules_priority
100101
direction = "INGRESS"
101102

102-
source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
103+
source_ranges = local.pod_all_ip_ranges
103104
target_tags = [local.cluster_network_tag]
104105

105106
# Allow all possible protocols
@@ -110,8 +111,11 @@ resource "google_compute_firewall" "shadow_allow_pods" {
110111
allow { protocol = "esp" }
111112
allow { protocol = "ah" }
112113

113-
log_config {
114-
metadata = "INCLUDE_ALL_METADATA"
114+
dynamic "log_config" {
115+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
116+
content {
117+
metadata = log_config.value.metadata
118+
}
115119
}
116120
}
117121

@@ -133,8 +137,11 @@ resource "google_compute_firewall" "shadow_allow_master" {
133137
ports = ["10250", "443"]
134138
}
135139

136-
log_config {
137-
metadata = "INCLUDE_ALL_METADATA"
140+
dynamic "log_config" {
141+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
142+
content {
143+
metadata = log_config.value.metadata
144+
}
138145
}
139146
}
140147

@@ -165,7 +172,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" {
165172
ports = ["1-65535"]
166173
}
167174

168-
log_config {
169-
metadata = "INCLUDE_ALL_METADATA"
175+
dynamic "log_config" {
176+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
177+
content {
178+
metadata = log_config.value.metadata
179+
}
180+
}
181+
}
182+
183+
resource "google_compute_firewall" "shadow_allow_inkubelet" {
184+
count = var.add_shadow_firewall_rules ? 1 : 0
185+
186+
name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet"
187+
description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet."
188+
project = local.network_project_id
189+
network = var.network
190+
priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999
191+
direction = "INGRESS"
192+
193+
source_ranges = local.pod_all_ip_ranges
194+
source_tags = [local.cluster_network_tag]
195+
target_tags = [local.cluster_network_tag]
196+
197+
allow {
198+
protocol = "tcp"
199+
ports = ["10255"]
200+
}
201+
202+
dynamic "log_config" {
203+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
204+
content {
205+
metadata = log_config.value.metadata
206+
}
207+
}
208+
}
209+
210+
resource "google_compute_firewall" "shadow_deny_exkubelet" {
211+
count = var.add_shadow_firewall_rules ? 1 : 0
212+
213+
name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet"
214+
description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet."
215+
project = local.network_project_id
216+
network = var.network
217+
priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000
218+
direction = "INGRESS"
219+
220+
source_ranges = ["0.0.0.0/0"]
221+
target_tags = [local.cluster_network_tag]
222+
223+
deny {
224+
protocol = "tcp"
225+
ports = ["10255"]
226+
}
227+
228+
dynamic "log_config" {
229+
for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config]
230+
content {
231+
metadata = log_config.value.metadata
232+
}
170233
}
171234
}

‎main.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ locals {
7373

7474
cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
7575
cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
76+
pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : []
7677

7778
cluster_network_policy = var.network_policy ? [{
7879
enabled = true

‎modules/beta-autopilot-private-cluster/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ Then perform the following commands on the root folder:
125125
| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
126126
| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
127127
| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no |
128+
| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | <pre>object({<br> metadata = string<br> })</pre> | <pre>{<br> "metadata": "INCLUDE_ALL_METADATA"<br>}</pre> | no |
128129
| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
129130
| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
130131
| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |

0 commit comments

Comments
 (0)