Skip to content

Commit 4d58cad

Browse files
nvnmandadhinvnmandadhi
committed
add island-cluster submodule and an example
1 parent 19e9c0d commit 4d58cad

File tree

10 files changed

+479
-0
lines changed

10 files changed

+479
-0
lines changed

‎examples/island_cluster_with_vm_router/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# GKE island cluster using VM as router
2+
3+
This uses `island-cluster` submodule to provison a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project.

‎examples/island_cluster_with_vm_router/main.tf

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
module "gke-island-cluster" {
2+
source = "../../modules/island-cluster"
3+
4+
project_id = var.project_id
5+
region = var.region
6+
random_suffix = "b8701ba0"
7+
node_locations = var.node_locations
8+
subnet_cidr = var.subnet_cidr
9+
secondary_ranges = var.secondary_ranges
10+
master_authorized_networks = var.master_authorized_networks
11+
router_machine_type = var.router_machine_type
12+
primary_subnet = var.primary_subnet
13+
primary_net_cidrs = var.primary_net_cidrs
14+
psc_subnet_cidr = var.psc_subnet_cidr
15+
proxy_subnet_cidr = var.proxy_subnet_cidr
16+
}

‎examples/island_cluster_with_vm_router/terraform.tfvars

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
project_id = "<project_id>"
2+
region = "us-central1"
3+
node_locations = [
4+
"us-central1-a",
5+
"us-central1-b",
6+
"us-central1-f"
7+
]
8+
subnet_cidr = "100.64.0.0/20"
9+
router_machine_type = "n2-highcpu-4"
10+
primary_subnet = "projects/<project_id>/regions/<region>/subnetworks/<subnet>"
11+
secondary_ranges = {
12+
pods = "100.64.64.0/18"
13+
services = "100.64.128.0/20"
14+
master_cidr = "100.64.144.0/28"
15+
}
16+
proxy_subnet_cidr = "100.64.168.0/24"
17+
psc_subnet_cidr = "100.64.192.0/24"
18+
master_authorized_networks = [
19+
{
20+
cidr_block = "100.64.0.0/10"
21+
display_name = "cluster net"
22+
}
23+
]
24+
primary_net_cidrs = [
25+
"10.0.0.0/8",
26+
"192.168.0.0/16",
27+
"172.16.0.0/12"
28+
]

‎examples/island_cluster_with_vm_router/vars.tf

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
variable "project_id" {
2+
type = string
3+
}
4+
5+
variable "region" {
6+
type = string
7+
}
8+
9+
variable "node_locations" {
10+
type = list(string)
11+
}
12+
13+
variable "primary_subnet" {
14+
type = string
15+
description = "subnet of the existing network for the router VM NIC"
16+
}
17+
18+
variable "subnet_cidr" {
19+
type = string
20+
}
21+
22+
variable "psc_subnet_cidr" {
23+
type = string
24+
}
25+
26+
variable "proxy_subnet_cidr" {
27+
type = string
28+
}
29+
30+
variable "secondary_ranges" {
31+
type = map(string)
32+
}
33+
34+
variable "master_authorized_networks" {
35+
type = list(object({ cidr_block = string, display_name = string }))
36+
description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)."
37+
}
38+
39+
variable "primary_net_cidrs" {
40+
type = list(string)
41+
}
42+
43+
variable "router_machine_type" {
44+
type = string
45+
}

‎modules/island-cluster/README.md

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# GKE island cluster using VM as router
2+
3+
This submodule provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project.
4+
5+
1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network.
6+
2. Outbound connections will go through the router.
7+
3. For inbound connections, use Private Service Connect.
8+
9+
## Requirements
10+
11+
| Name | Version |
12+
|------|---------|
13+
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.6 |
14+
15+
## Providers
16+
17+
| Name | Version |
18+
|------|---------|
19+
| <a name="provider_google"></a> [google](#provider\_google) | n/a |
20+
| <a name="provider_google-beta"></a> [google-beta](#provider\_google-beta) | n/a |
21+
22+
## Modules
23+
24+
| Name | Source | Version |
25+
|------|--------|---------|
26+
| <a name="module_gke"></a> [gke](#module\_gke) | terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster | ~> 30.0 |
27+
| <a name="module_net"></a> [net](#module\_net) | terraform-google-modules/network/google | ~> 9.0 |
28+
29+
## Resources
30+
31+
| Name | Type |
32+
|------|------|
33+
| [google-beta_google_gke_hub_membership.primary](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_gke_hub_membership) | resource |
34+
| [google_compute_firewall.proxy_snet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
35+
| [google_compute_firewall.psc_snet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
36+
| [google_compute_instance.vm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource |
37+
| [google_project_iam_member.gke-dev](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
38+
| [google_service_account.gke-access-sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
39+
| [google_service_account.gke-sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
40+
41+
## Inputs
42+
43+
| Name | Description | Type | Default | Required |
44+
|------|-------------|------|---------|:--------:|
45+
| <a name="input_master_authorized_networks"></a> [master\_authorized\_networks](#input\_master\_authorized\_networks) | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes |
46+
| <a name="input_node_locations"></a> [node\_locations](#input\_node\_locations) | n/a | `list(string)` | n/a | yes |
47+
| <a name="input_primary_net_cidrs"></a> [primary\_net\_cidrs](#input\_primary\_net\_cidrs) | n/a | `list(string)` | n/a | yes |
48+
| <a name="input_primary_subnet"></a> [primary\_subnet](#input\_primary\_subnet) | n/a | `string` | n/a | yes |
49+
| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes |
50+
| <a name="input_proxy_subnet_cidr"></a> [proxy\_subnet\_cidr](#input\_proxy\_subnet\_cidr) | n/a | `string` | n/a | yes |
51+
| <a name="input_psc_subnet_cidr"></a> [psc\_subnet\_cidr](#input\_psc\_subnet\_cidr) | n/a | `string` | n/a | yes |
52+
| <a name="input_random_suffix"></a> [random\_suffix](#input\_random\_suffix) | n/a | `string` | n/a | yes |
53+
| <a name="input_region"></a> [region](#input\_region) | n/a | `string` | n/a | yes |
54+
| <a name="input_router_machine_type"></a> [router\_machine\_type](#input\_router\_machine\_type) | n/a | `string` | n/a | yes |
55+
| <a name="input_secondary_ranges"></a> [secondary\_ranges](#input\_secondary\_ranges) | n/a | `map(string)` | n/a | yes |
56+
| <a name="input_subnet_cidr"></a> [subnet\_cidr](#input\_subnet\_cidr) | n/a | `string` | n/a | yes |
57+
58+
## Outputs
59+
60+
| Name | Description |
61+
|------|-------------|
62+
| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | n/a |

‎modules/island-cluster/main.tf

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
resource "google_service_account" "gke-sa" {
2+
account_id = "gke-sa-${var.random_suffix}"
3+
}
4+
5+
resource "google_service_account" "gke-access-sa" {
6+
account_id = "gke-access-sa-${var.random_suffix}"
7+
}
8+
9+
resource "google_project_iam_member" "gke-dev" {
10+
project = var.project_id
11+
member = "serviceAccount:${google_service_account.gke-access-sa.email}"
12+
role = "roles/container.developer"
13+
}
14+
15+
module "net" {
16+
source = "terraform-google-modules/network/google"
17+
version = "~> 9.0"
18+
19+
network_name = "gke-net-${var.random_suffix}"
20+
routing_mode = "GLOBAL"
21+
project_id = var.project_id
22+
delete_default_internet_gateway_routes = true
23+
24+
subnets = [
25+
{
26+
subnet_name = "${var.region}-snet-${var.random_suffix}"
27+
subnet_ip = var.subnet_cidr
28+
subnet_region = var.region
29+
subnet_private_access = "true"
30+
},
31+
{
32+
subnet_name = "${var.region}-proxy-snet-${var.random_suffix}"
33+
subnet_ip = var.proxy_subnet_cidr
34+
subnet_region = var.region
35+
purpose = "REGIONAL_MANAGED_PROXY"
36+
role = "ACTIVE"
37+
},
38+
{
39+
subnet_name = "${var.region}-psc-snet-${var.random_suffix}"
40+
subnet_ip = var.psc_subnet_cidr
41+
subnet_region = var.region
42+
subnet_private_access = "true"
43+
purpose = "PRIVATE_SERVICE_CONNECT"
44+
}
45+
]
46+
47+
secondary_ranges = {
48+
"${var.region}-snet-${var.random_suffix}" = [
49+
{
50+
range_name = "${var.region}-snet-pods-${var.random_suffix}"
51+
ip_cidr_range = var.secondary_ranges["pods"]
52+
},
53+
{
54+
range_name = "${var.region}-snet-services-${var.random_suffix}"
55+
ip_cidr_range = var.secondary_ranges["services"]
56+
},
57+
]
58+
}
59+
60+
routes = flatten([
61+
[for k, v in var.primary_net_cidrs :
62+
{
63+
name = "egress-gke-${k}-${var.random_suffix}"
64+
description = "egress through the router for range ${v}"
65+
destination_range = v
66+
tags = "gke-${var.random_suffix}"
67+
next_hop_instance = google_compute_instance.vm.self_link
68+
priority = 100
69+
}
70+
],
71+
[
72+
{
73+
name = "default-igw-${var.random_suffix}"
74+
description = "internet through the router"
75+
destination_range = "0.0.0.0/0"
76+
tags = "gke-${var.random_suffix}"
77+
next_hop_instance = google_compute_instance.vm.self_link
78+
priority = 100
79+
}
80+
]
81+
])
82+
83+
firewall_rules = [
84+
{
85+
name = "iap-fw-${var.random_suffix}"
86+
direction = "INGRESS"
87+
allow = [
88+
{
89+
protocol = "TCP"
90+
ports = ["22"]
91+
}
92+
]
93+
ranges = ["35.235.240.0/20"]
94+
},
95+
{
96+
name = "tcp-primary-fw-${var.random_suffix}"
97+
direction = "INGRESS"
98+
allow = [
99+
{
100+
protocol = "TCP"
101+
}
102+
]
103+
ranges = [
104+
var.subnet_cidr,
105+
var.secondary_ranges["pods"]
106+
]
107+
},
108+
]
109+
}
110+
111+
module "gke" {
112+
source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
113+
version = "~> 30.0"
114+
115+
depends_on = [google_compute_instance.vm]
116+
117+
name = "gke-test-${var.random_suffix}"
118+
project_id = var.project_id
119+
region = var.region
120+
release_channel = "RAPID"
121+
zones = var.node_locations
122+
network = module.net.network_name
123+
subnetwork = "${var.region}-snet-${var.random_suffix}"
124+
ip_range_pods = "${var.region}-snet-pods-${var.random_suffix}"
125+
ip_range_services = "${var.region}-snet-services-${var.random_suffix}"
126+
http_load_balancing = true
127+
horizontal_pod_autoscaling = true
128+
enable_private_endpoint = true
129+
enable_private_nodes = true
130+
datapath_provider = "ADVANCED_DATAPATH"
131+
monitoring_enable_managed_prometheus = false
132+
enable_shielded_nodes = true
133+
master_global_access_enabled = false
134+
master_ipv4_cidr_block = var.secondary_ranges["master_cidr"]
135+
master_authorized_networks = var.master_authorized_networks
136+
deletion_protection = false
137+
remove_default_node_pool = true
138+
disable_default_snat = true
139+
gateway_api_channel = "CHANNEL_STANDARD"
140+
141+
node_pools = [
142+
{
143+
name = "default-${var.random_suffix}"
144+
machine_type = "e2-highcpu-2"
145+
min_count = 1
146+
max_count = 100
147+
local_ssd_count = 0
148+
spot = true
149+
local_ssd_ephemeral_count = 0
150+
disk_size_gb = 100
151+
disk_type = "pd-standard"
152+
image_type = "COS_CONTAINERD"
153+
logging_variant = "DEFAULT"
154+
auto_repair = true
155+
auto_upgrade = true
156+
service_account = google_service_account.gke-sa.email
157+
initial_node_count = 1
158+
enable_secure_boot = true
159+
},
160+
]
161+
162+
node_pools_tags = {
163+
all = ["gke-${var.random_suffix}"]
164+
}
165+
166+
node_pools_oauth_scopes = {
167+
all = [
168+
"https://www.googleapis.com/auth/logging.write",
169+
"https://www.googleapis.com/auth/monitoring",
170+
]
171+
}
172+
173+
timeouts = {
174+
create = "10m"
175+
update = "10m"
176+
delete = "10m"
177+
}
178+
}
179+
180+
resource "google_gke_hub_membership" "primary" {
181+
provider = google-beta
182+
183+
project = var.project_id
184+
membership_id = "${var.project_id}-${module.gke.name}"
185+
location = var.region
186+
187+
endpoint {
188+
gke_cluster {
189+
resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
190+
}
191+
}
192+
authority {
193+
issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}"
194+
}
195+
}
196+
197+
resource "google_compute_firewall" "psc_snet" {
198+
name = "allow-psc-${var.random_suffix}"
199+
direction = "INGRESS"
200+
allow {
201+
protocol = "tcp"
202+
}
203+
source_ranges = [var.psc_subnet_cidr]
204+
target_service_accounts = [google_service_account.gke-sa.email]
205+
network = module.net.network_id
206+
project = var.project_id
207+
}
208+
209+
resource "google_compute_firewall" "proxy_snet" {
210+
name = "allow-proxy-${var.random_suffix}"
211+
direction = "INGRESS"
212+
allow {
213+
protocol = "tcp"
214+
}
215+
source_ranges = [var.proxy_subnet_cidr]
216+
target_service_accounts = [google_service_account.gke-sa.email]
217+
network = module.net.network_id
218+
project = var.project_id
219+
}

‎modules/island-cluster/outputs.tf

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
output "cluster_id" {
2+
value = module.gke.cluster_id
3+
}

0 commit comments

Comments
 (0)