Skip to content

Commit 99cfd98

Browse files
tomasgareautomasgareau
authored
fix: Create separate firewall rule for egress to TPUs (#1126)
BREAKING CHANGE: TPU firewall rule split into a separate resource
1 parent b8b8547 commit 99cfd98

File tree

5 files changed

+185
-20
lines changed

5 files changed

+185
-20
lines changed

‎autogen/main/firewall.tf.tmpl

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,20 +34,50 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
{% if beta_cluster %}
38-
destination_ranges = compact([
39-
local.cluster_endpoint_for_nodes,
40-
local.cluster_subnet_cidr,
41-
local.cluster_alias_ranges_cidr[var.ip_range_pods],
42-
google_container_cluster.primary.tpu_ipv4_cidr_block,
43-
])
44-
{% else %}
4537
destination_ranges = [
4638
local.cluster_endpoint_for_nodes,
4739
local.cluster_subnet_cidr,
4840
local.cluster_alias_ranges_cidr[var.ip_range_pods],
4941
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
{% if not private_cluster %}
52+
depends_on = [
53+
google_container_cluster.primary,
54+
]
5055
{% endif %}
56+
}
57+
58+
59+
{% if beta_cluster %}
60+
/******************************************
61+
Allow egress to the TPU IPv4 CIDR block
62+
63+
This rule is defined separately from the
64+
intra_egress rule above since it requires
65+
an output from the google_container_cluster
66+
resource.
67+
68+
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
69+
*****************************************/
70+
resource "google_compute_firewall" "tpu_egress" {
71+
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
72+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
73+
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
74+
project = local.network_project_id
75+
network = var.network
76+
priority = var.firewall_priority
77+
direction = "EGRESS"
78+
79+
target_tags = [local.cluster_network_tag]
80+
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]
5181

5282
# Allow all possible protocols
5383
allow { protocol = "tcp" }
@@ -65,6 +95,7 @@ resource "google_compute_firewall" "intra_egress" {
6595
}
6696

6797

98+
{% endif %}
6899
/******************************************
69100
Allow GKE master to hit non 443 ports for
70101
Webhooks/Admission Controllers

‎modules/beta-private-cluster-update-variant/firewall.tf

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,44 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = compact([
37+
destination_ranges = [
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
4040
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
google_container_cluster.primary.tpu_ipv4_cidr_block,
42-
])
41+
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
}
52+
53+
54+
/******************************************
55+
Allow egress to the TPU IPv4 CIDR block
56+
57+
This rule is defined separately from the
58+
intra_egress rule above since it requires
59+
an output from the google_container_cluster
60+
resource.
61+
62+
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
63+
*****************************************/
64+
resource "google_compute_firewall" "tpu_egress" {
65+
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
66+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
67+
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
68+
project = local.network_project_id
69+
network = var.network
70+
priority = var.firewall_priority
71+
direction = "EGRESS"
72+
73+
target_tags = [local.cluster_network_tag]
74+
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]
4375

4476
# Allow all possible protocols
4577
allow { protocol = "tcp" }

‎modules/beta-private-cluster/firewall.tf

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,44 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = compact([
37+
destination_ranges = [
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
4040
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
google_container_cluster.primary.tpu_ipv4_cidr_block,
42-
])
41+
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
}
52+
53+
54+
/******************************************
55+
Allow egress to the TPU IPv4 CIDR block
56+
57+
This rule is defined separately from the
58+
intra_egress rule above since it requires
59+
an output from the google_container_cluster
60+
resource.
61+
62+
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
63+
*****************************************/
64+
resource "google_compute_firewall" "tpu_egress" {
65+
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
66+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
67+
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
68+
project = local.network_project_id
69+
network = var.network
70+
priority = var.firewall_priority
71+
direction = "EGRESS"
72+
73+
target_tags = [local.cluster_network_tag]
74+
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]
4375

4476
# Allow all possible protocols
4577
allow { protocol = "tcp" }

‎modules/beta-public-cluster-update-variant/firewall.tf

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,47 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = compact([
37+
destination_ranges = [
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
4040
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
google_container_cluster.primary.tpu_ipv4_cidr_block,
42-
])
41+
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
depends_on = [
52+
google_container_cluster.primary,
53+
]
54+
}
55+
56+
57+
/******************************************
58+
Allow egress to the TPU IPv4 CIDR block
59+
60+
This rule is defined separately from the
61+
intra_egress rule above since it requires
62+
an output from the google_container_cluster
63+
resource.
64+
65+
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
66+
*****************************************/
67+
resource "google_compute_firewall" "tpu_egress" {
68+
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
69+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
70+
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
71+
project = local.network_project_id
72+
network = var.network
73+
priority = var.firewall_priority
74+
direction = "EGRESS"
75+
76+
target_tags = [local.cluster_network_tag]
77+
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]
4378

4479
# Allow all possible protocols
4580
allow { protocol = "tcp" }

‎modules/beta-public-cluster/firewall.tf

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,47 @@ resource "google_compute_firewall" "intra_egress" {
3434
direction = "EGRESS"
3535

3636
target_tags = [local.cluster_network_tag]
37-
destination_ranges = compact([
37+
destination_ranges = [
3838
local.cluster_endpoint_for_nodes,
3939
local.cluster_subnet_cidr,
4040
local.cluster_alias_ranges_cidr[var.ip_range_pods],
41-
google_container_cluster.primary.tpu_ipv4_cidr_block,
42-
])
41+
]
42+
43+
# Allow all possible protocols
44+
allow { protocol = "tcp" }
45+
allow { protocol = "udp" }
46+
allow { protocol = "icmp" }
47+
allow { protocol = "sctp" }
48+
allow { protocol = "esp" }
49+
allow { protocol = "ah" }
50+
51+
depends_on = [
52+
google_container_cluster.primary,
53+
]
54+
}
55+
56+
57+
/******************************************
58+
Allow egress to the TPU IPv4 CIDR block
59+
60+
This rule is defined separately from the
61+
intra_egress rule above since it requires
62+
an output from the google_container_cluster
63+
resource.
64+
65+
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
66+
*****************************************/
67+
resource "google_compute_firewall" "tpu_egress" {
68+
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
69+
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
70+
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
71+
project = local.network_project_id
72+
network = var.network
73+
priority = var.firewall_priority
74+
direction = "EGRESS"
75+
76+
target_tags = [local.cluster_network_tag]
77+
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]
4378

4479
# Allow all possible protocols
4580
allow { protocol = "tcp" }

0 commit comments

Comments
 (0)