feat: Add support for gVisor per node pool (#1001)

LukaszCzarnotaSabre · bharathkkb · bharathkkb · commit 73f39dfe6a9b · 2021-10-18T13:41:09.000-05:00
* feat: Add support for gVisor per node pool

* fix image type, add test

Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -449,6 +449,9 @@ resource "google_container_node_pool" "pools" {
   {% endif %}
 
   node_config {
+    {% if beta_cluster %}
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
+    {% else %}
     image_type   = lookup(each.value, "image_type", "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
@@ -519,8 +522,7 @@ resource "google_container_node_pool" "pools" {
     }
     {% if beta_cluster %}
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -62,13 +62,15 @@ module "gke" {
     },
     {
       name            = "pool-03"
+      machine_type    = "n1-standard-2"
       node_locations  = "${var.region}-b,${var.region}-c"
       autoscaling     = false
       node_count      = 2
       disk_type       = "pd-standard"
-      image_type      = "COS"
       auto_upgrade    = true
       service_account = var.compute_engine_service_account
+      pod_range       = "test"
+      sandbox_enabled = true
     },
   ]
 
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -406,7 +406,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -475,8 +475,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
@@ -334,7 +334,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -403,8 +403,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -387,7 +387,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -456,8 +456,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
@@ -315,7 +315,7 @@ resource "google_container_node_pool" "pools" {
   }
 
   node_config {
-    image_type   = lookup(each.value, "image_type", "COS")
+    image_type   = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")
     machine_type = lookup(each.value, "machine_type", "e2-medium")
     labels = merge(
       lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},
@@ -384,8 +384,7 @@ resource "google_container_node_pool" "pools" {
       }
     }
     dynamic "sandbox_config" {
-      for_each = local.cluster_sandbox_enabled
-
+      for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []
       content {
         sandbox_type = sandbox_config.value
       }
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
@@ -319,7 +319,7 @@
             including(
               "name" => "pool-03",
               "config" => including(
-                "machineType" => "e2-medium",
+                "machineType" => "n1-standard-2",
               ),
             )
           )
@@ -376,6 +376,7 @@
                   "all-pools-example" => "true",
                   "cluster_name" => cluster_name,
                   "node_pool" => "pool-03",
+                  "sandbox.gke.io/runtime"=>"gvisor"
                 },
               ),
             )
@@ -396,6 +397,44 @@
             )
           )
         end
+
+        it "has the expected pod range" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "networkConfig" => including(
+                "podIpv4CidrBlock" => "172.16.0.0/18",
+                "podRange" => "test"
+              )
+            )
+          )
+        end
+
+        it "has the expected image" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "imageType" => "COS_CONTAINERD",
+              ),
+            )
+          )
+        end
+
+        it "has the expected linux node config sysctls" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "linuxNodeConfig" => including(
+                  "sysctls" => including(
+                    "net.core.netdev_max_backlog" => "20000"
+                  )
+                )
+              )
+            )
+          )
+        end
       end
     end
   end

Original file line number	Diff line number	Diff line change
`@@ -406,7 +406,7 @@ resource "google_container_node_pool" "pools" {`
`406`	`406`	`}`
`407`	`407`
`408`	`408`	`node_config {`
`409`		`- image_type = lookup(each.value, "image_type", "COS")`
	`409`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`410`	`410`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`411`	`411`	`labels = merge(`
`412`	`412`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -475,8 +475,7 @@ resource "google_container_node_pool" "pools" {`
`475`	`475`	`}`
`476`	`476`	`}`
`477`	`477`	`dynamic "sandbox_config" {`
`478`		`- for_each = local.cluster_sandbox_enabled`
`479`		`-`
	`478`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`480`	`479`	`content {`
`481`	`480`	`sandbox_type = sandbox_config.value`
`482`	`481`	`}`
Original file line number	Diff line number	Diff line change
`@@ -334,7 +334,7 @@ resource "google_container_node_pool" "pools" {`
`334`	`334`	`}`
`335`	`335`
`336`	`336`	`node_config {`
`337`		`- image_type = lookup(each.value, "image_type", "COS")`
	`337`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`338`	`338`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`339`	`339`	`labels = merge(`
`340`	`340`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -403,8 +403,7 @@ resource "google_container_node_pool" "pools" {`
`403`	`403`	`}`
`404`	`404`	`}`
`405`	`405`	`dynamic "sandbox_config" {`
`406`		`- for_each = local.cluster_sandbox_enabled`
`407`		`-`
	`406`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`408`	`407`	`content {`
`409`	`408`	`sandbox_type = sandbox_config.value`
`410`	`409`	`}`
Original file line number	Diff line number	Diff line change
`@@ -387,7 +387,7 @@ resource "google_container_node_pool" "pools" {`
`387`	`387`	`}`
`388`	`388`
`389`	`389`	`node_config {`
`390`		`- image_type = lookup(each.value, "image_type", "COS")`
	`390`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`391`	`391`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`392`	`392`	`labels = merge(`
`393`	`393`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -456,8 +456,7 @@ resource "google_container_node_pool" "pools" {`
`456`	`456`	`}`
`457`	`457`	`}`
`458`	`458`	`dynamic "sandbox_config" {`
`459`		`- for_each = local.cluster_sandbox_enabled`
`460`		`-`
	`459`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`461`	`460`	`content {`
`462`	`461`	`sandbox_type = sandbox_config.value`
`463`	`462`	`}`
Original file line number	Diff line number	Diff line change
`@@ -315,7 +315,7 @@ resource "google_container_node_pool" "pools" {`
`315`	`315`	`}`
`316`	`316`
`317`	`317`	`node_config {`
`318`		`- image_type = lookup(each.value, "image_type", "COS")`
	`318`	`+ image_type = lookup(each.value, "image_type", lookup(each.value, "sandbox_enabled", var.sandbox_enabled) ? "COS_CONTAINERD" : "COS")`
`319`	`319`	`machine_type = lookup(each.value, "machine_type", "e2-medium")`
`320`	`320`	`labels = merge(`
`321`	`321`	`lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {},`
`@@ -384,8 +384,7 @@ resource "google_container_node_pool" "pools" {`
`384`	`384`	`}`
`385`	`385`	`}`
`386`	`386`	`dynamic "sandbox_config" {`
`387`		`- for_each = local.cluster_sandbox_enabled`
`388`		`-`
	`387`	`+ for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : []`
`389`	`388`	`content {`
`390`	`389`	`sandbox_type = sandbox_config.value`
`391`	`390`	`}`