chore: add island-cluster example (#1918)

nvnmandadhi · bharathkkb · web-flow · commit e711adbdfd01 · 2024-04-04T15:12:04.000-05:00
Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
diff --git a/examples/island_cluster_with_vm_router/README.md b/examples/island_cluster_with_vm_router/README.md
@@ -0,0 +1,38 @@
+# GKE island cluster using VM as router
+
+This example provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project.
+
+1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network.
+1. Outbound connections will go through the router.
+1. For inbound connections, use Private Service Connect.
+
+## Deploy
+
+1. Update `project_id`, `cluster_name` and `primary_subnet` values in `terraform.tfvars`, and update other variables as needed.
+1. Run `terraform apply`.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| cluster\_name | n/a | `string` | n/a | yes |
+| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes |
+| node\_locations | n/a | `list(string)` | n/a | yes |
+| primary\_net\_cidrs | n/a | `list(string)` | n/a | yes |
+| primary\_subnet | n/a | `string` | n/a | yes |
+| project\_id | n/a | `string` | n/a | yes |
+| proxy\_subnet\_cidr | n/a | `string` | n/a | yes |
+| psc\_subnet\_cidr | n/a | `string` | n/a | yes |
+| region | n/a | `string` | n/a | yes |
+| router\_machine\_type | n/a | `string` | n/a | yes |
+| secondary\_ranges | n/a | `map(string)` | n/a | yes |
+| subnet\_cidr | n/a | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cluster\_id | n/a |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/island_cluster_with_vm_router/main.tf b/examples/island_cluster_with_vm_router/main.tf
@@ -0,0 +1,226 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_id" "rand" {
+  byte_length = 4
+}
+
+resource "google_service_account" "gke-sa" {
+  account_id = "gke-sa-${random_id.rand.hex}"
+  project    = var.project_id
+}
+
+module "net" {
+  source  = "terraform-google-modules/network/google"
+  version = "~> 9.0"
+
+  network_name                           = "gke-net-${random_id.rand.hex}"
+  routing_mode                           = "GLOBAL"
+  project_id                             = var.project_id
+  delete_default_internet_gateway_routes = true
+
+  subnets = [
+    {
+      subnet_name           = "${var.cluster_name}-${var.region}-snet"
+      subnet_ip             = var.subnet_cidr
+      subnet_region         = var.region
+      subnet_private_access = "true"
+    },
+    {
+      subnet_name   = "${var.cluster_name}-${var.region}-proxy-snet"
+      subnet_ip     = var.proxy_subnet_cidr
+      subnet_region = var.region
+      purpose       = "REGIONAL_MANAGED_PROXY"
+      role          = "ACTIVE"
+    },
+    {
+      subnet_name           = "${var.cluster_name}-${var.region}-psc-snet"
+      subnet_ip             = var.psc_subnet_cidr
+      subnet_region         = var.region
+      subnet_private_access = "true"
+      purpose               = "PRIVATE_SERVICE_CONNECT"
+    }
+  ]
+
+  secondary_ranges = {
+    "${var.cluster_name}-${var.region}-snet" = [
+      {
+        range_name    = "${var.cluster_name}-${var.region}-snet-pods"
+        ip_cidr_range = var.secondary_ranges["pods"]
+      },
+      {
+        range_name    = "${var.cluster_name}-${var.region}-snet-services"
+        ip_cidr_range = var.secondary_ranges["services"]
+      },
+    ]
+  }
+
+  routes = flatten([
+    [for k, v in var.primary_net_cidrs :
+      {
+        name              = "${var.cluster_name}-egress-gke-${k}"
+        description       = "egress through the router for range ${v}"
+        destination_range = v
+        tags              = "gke-${random_id.rand.hex}"
+        next_hop_instance = google_compute_instance.vm.self_link
+        priority          = 100
+      }
+    ],
+    [
+      {
+        name              = "${var.cluster_name}-default-igw"
+        description       = "internet through the router"
+        destination_range = "0.0.0.0/0"
+        tags              = "gke-${random_id.rand.hex}"
+        next_hop_instance = google_compute_instance.vm.self_link
+        priority          = 100
+      }
+    ]
+  ])
+
+  firewall_rules = [
+    {
+      name      = "${var.cluster_name}-iap"
+      direction = "INGRESS"
+      allow = [
+        {
+          protocol = "TCP"
+          ports    = ["22"]
+        }
+      ]
+      ranges = ["35.235.240.0/20"]
+    },
+    {
+      name      = "${var.cluster_name}-tcp-primary"
+      direction = "INGRESS"
+      allow = [
+        {
+          protocol = "TCP"
+        }
+      ]
+      ranges = [
+        var.subnet_cidr,
+        var.secondary_ranges["pods"]
+      ]
+    },
+    {
+      name      = "${var.cluster_name}-allow-psc"
+      direction = "INGRESS"
+      allow = [
+        {
+          protocol = "TCP"
+        }
+      ]
+      ranges                  = [var.psc_subnet_cidr]
+      target_service_accounts = [google_service_account.gke-sa.email]
+    },
+    {
+      name      = "${var.cluster_name}-allow-proxy"
+      direction = "INGRESS"
+      allow = [
+        {
+          protocol = "TCP"
+        }
+      ]
+      ranges                  = [var.proxy_subnet_cidr]
+      target_service_accounts = [google_service_account.gke-sa.email]
+    },
+  ]
+}
+
+module "gke" {
+  source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
+  version = "~> 30.0"
+
+  depends_on = [google_compute_instance.vm]
+
+  name                                 = var.cluster_name
+  project_id                           = var.project_id
+  region                               = var.region
+  release_channel                      = "RAPID"
+  zones                                = var.node_locations
+  network                              = module.net.network_name
+  subnetwork                           = "${var.cluster_name}-${var.region}-snet"
+  ip_range_pods                        = "${var.cluster_name}-${var.region}-snet-pods"
+  ip_range_services                    = "${var.cluster_name}-${var.region}-snet-services"
+  enable_private_endpoint              = true
+  enable_private_nodes                 = true
+  datapath_provider                    = "ADVANCED_DATAPATH"
+  monitoring_enable_managed_prometheus = false
+  enable_shielded_nodes                = true
+  master_global_access_enabled         = false
+  master_ipv4_cidr_block               = var.secondary_ranges["master_cidr"]
+  master_authorized_networks           = var.master_authorized_networks
+  deletion_protection                  = false
+  remove_default_node_pool             = true
+  disable_default_snat                 = true
+  gateway_api_channel                  = "CHANNEL_STANDARD"
+
+  node_pools = [
+    {
+      name                      = "default"
+      machine_type              = "e2-highcpu-2"
+      min_count                 = 1
+      max_count                 = 100
+      local_ssd_count           = 0
+      spot                      = true
+      local_ssd_ephemeral_count = 0
+      disk_size_gb              = 100
+      disk_type                 = "pd-standard"
+      image_type                = "COS_CONTAINERD"
+      logging_variant           = "DEFAULT"
+      auto_repair               = true
+      auto_upgrade              = true
+      service_account           = google_service_account.gke-sa.email
+      initial_node_count        = 1
+      enable_secure_boot        = true
+    },
+  ]
+
+  node_pools_tags = {
+    all = ["gke-${random_id.rand.hex}"]
+  }
+
+  node_pools_oauth_scopes = {
+    all = [
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+    ]
+  }
+
+  timeouts = {
+    create = "15m"
+    update = "15m"
+    delete = "15m"
+  }
+}
+
+resource "google_gke_hub_membership" "primary" {
+  provider = google-beta
+
+  project       = var.project_id
+  membership_id = "${var.project_id}-${module.gke.name}"
+  location      = var.region
+
+  endpoint {
+    gke_cluster {
+      resource_link = "//container.googleapis.com/${module.gke.cluster_id}"
+    }
+  }
+  authority {
+    issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}"
+  }
+}
diff --git a/examples/island_cluster_with_vm_router/outputs.tf b/examples/island_cluster_with_vm_router/outputs.tf
@@ -0,0 +1,19 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cluster_id" {
+  value = module.gke.cluster_id
+}
diff --git a/examples/island_cluster_with_vm_router/router.tf b/examples/island_cluster_with_vm_router/router.tf
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_compute_instance" "vm" {
+  project                   = var.project_id
+  zone                      = var.node_locations[0]
+  name                      = "${var.cluster_name}-router-${random_id.rand.hex}"
+  machine_type              = var.router_machine_type
+  allow_stopping_for_update = true
+  boot_disk {
+    initialize_params {
+      image = "debian-cloud/debian-12"
+    }
+  }
+  can_ip_forward = true
+  shielded_instance_config {
+    enable_secure_boot = true
+  }
+  network_interface {
+    subnetwork = var.primary_subnet
+  }
+  network_interface {
+    subnetwork = module.net.subnets["${var.region}/${var.cluster_name}-${var.region}-snet"]["self_link"]
+  }
+  metadata_startup_script = <<-EOT
+    #!/bin/bash
+    set -ex
+    sudo apt-get update
+    echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf
+    sudo sysctl -p
+    sudo iptables -A FORWARD -i ens5 -o ens4 -j ACCEPT
+    sudo iptables -A FORWARD -i ens4 -o ens5 -m state --state ESTABLISHED,RELATED -j ACCEPT
+    GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway"
+    GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google")
+    sudo ip route add ${var.secondary_ranges["pods"]} via $${GWY_IP} dev ens5
+    sudo iptables -t nat -A POSTROUTING -o ens4 -s 0.0.0.0/0 -j MASQUERADE
+  EOT
+}
diff --git a/examples/island_cluster_with_vm_router/terraform.tfvars b/examples/island_cluster_with_vm_router/terraform.tfvars
@@ -0,0 +1,29 @@
+project_id   = "<project_id>"
+region       = "us-central1"
+cluster_name = "gke-island-cluster-test"
+node_locations = [
+  "us-central1-a",
+  "us-central1-b",
+  "us-central1-f"
+]
+subnet_cidr         = "100.64.0.0/20"
+router_machine_type = "n2-highcpu-4"
+primary_subnet      = "projects/<project_id>/regions/<region>/subnetworks/<subnet>"
+secondary_ranges = {
+  pods        = "100.64.64.0/18"
+  services    = "100.64.128.0/20"
+  master_cidr = "100.64.144.0/28"
+}
+proxy_subnet_cidr = "100.64.168.0/24"
+psc_subnet_cidr   = "100.64.192.0/24"
+master_authorized_networks = [
+  {
+    cidr_block   = "100.64.0.0/10"
+    display_name = "cluster net"
+  }
+]
+primary_net_cidrs = [
+  "10.0.0.0/8",
+  "192.168.0.0/16",
+  "172.16.0.0/12"
+]
diff --git a/examples/island_cluster_with_vm_router/vars.tf b/examples/island_cluster_with_vm_router/vars.tf
diff --git a/examples/island_cluster_with_vm_router/versions.tf b/examples/island_cluster_with_vm_router/versions.tf
