add binary checks, add tests

Author: bharathkkb

.gitignore

@@ -50,3 +50,7 @@ credentials.json
 
 # File to populate env vars used by Docker test runs
 .envrc
+
+# ignore generated ASM yamls in /workspace/test/fixtures/simple_regional_with_asm as it is a test
+# in a production scenario these files are expected to be checked in
+/test/fixtures/simple_regional_with_asm/asm-dir

.kitchen.yml

@@ -221,3 +221,13 @@ suites:
       systems:
         - name: sandbox_enabled
           backend: local
+  - name: "simple_regional_with_asm"
+    driver:
+      root_module_directory: test/fixtures/simple_regional_with_asm
+    verifier:
+      systems:
+        - name: simple_regional_with_asm
+          backend: local
+          controls:
+            - gcloud
+            - kubectl

build/int.cloudbuild.yaml

@@ -384,6 +384,26 @@ steps:
     - verify workload-identity-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-identity-local']
+- id: create simple-regional-with-asm-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-with-asm-local']
+- id: converge simple-regional-with-asm-local
+  waitFor:
+    - create simple-regional-with-asm-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-with-asm-local']
+- id: verify simple-regional-with-asm-local
+  waitFor:
+    - converge simple-regional-with-asm-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-with-asm-local']
+- id: destroy simple-regional-with-asm-local
+  waitFor:
+    - verify simple-regional-with-asm-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-asm-local']
 tags:
 - 'ci'
 - 'integration'

examples/simple_regional_with_asm/main.tf

@@ -32,7 +32,6 @@ module "gke" {
   project_id = var.project_id
   name       = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
   regional   = true
-  #TESTING
   release_channel         = "REGULAR"
   region                  = var.region
   network                 = var.network
@@ -44,15 +43,19 @@ module "gke" {
   node_pools = [
     {
       name         = "asm-node-pool"
+      autoscaling  = false
+      # ASM requires minimum 4 nodes and n1-standard-4
+      # As this is a regional cluster we have node_count * 3 = 6 nodes
+      node_count    = 2
       machine_type = "n1-standard-4"
-      min_count    = 6
     },
   ]
 }
 
 module "asm" {
   source       = "../../modules/asm"
   cluster_name = module.gke.name
+  cluster_endpoint = module.gke.endpoint
   project_id   = var.project_id
   location     = module.gke.location
 }

modules/asm/main.tf

@@ -14,21 +14,31 @@
  * limitations under the License.
  */
 
+data "google_container_cluster" "primary" {
+  name     = var.cluster_name
+  project  = var.project_id
+  location = var.location
+}
+
+data "google_client_config" "default" {
+}
+
 module "asm_install" {
   source  = "terraform-google-modules/gcloud/google"
   version = "~> 1.0"
+  module_depends_on = [var.cluster_endpoint]
 
   platform                          = "linux"
-  gcloud_sdk_version                = "292.0.0"
+  gcloud_sdk_version                = "293.0.0"
   skip_download                     = var.skip_gcloud_download
   upgrade                           = false
   use_tf_google_credentials_env_var = true
   additional_components             = ["kubectl", "kpt", "anthoscli", "alpha"]
 
   create_cmd_entrypoint  = "${path.module}/scripts/install_asm.sh"
-  create_cmd_body        = "${var.project_id} ${var.cluster_name} ${var.location} ${var.asm_release_channel}"
-  destroy_cmd_entrypoint = "gcloud"
-  destroy_cmd_body       = "version"
+  create_cmd_body        = "${var.project_id} ${var.cluster_name} ${var.location}"
+  destroy_cmd_entrypoint = "${path.module}/scripts/kubectl_wrapper.sh"
+  destroy_cmd_body       = "https://${var.cluster_endpoint} ${data.google_client_config.default.access_token} ${data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate} kubectl delete ns istio-system"
 }
 
 resource "google_service_account" "gke_hub_sa" {
@@ -51,7 +61,7 @@ module "gke_hub_registration" {
   version = "~> 1.0"
 
   platform                          = "linux"
-  gcloud_sdk_version                = "292.0.0"
+  gcloud_sdk_version                = "293.0.0"
   skip_download                     = var.skip_gcloud_download
   upgrade                           = false
   use_tf_google_credentials_env_var = true

modules/asm/scripts/gke_hub_registration.sh

@@ -27,7 +27,8 @@ SERVICE_ACCOUNT_KEY=$4
 
 #write temp key, cleanup at exit
 tmp_file=$(mktemp)
+# shellcheck disable=SC2064
 trap "rm -rf $tmp_file" EXIT
-echo ${SERVICE_ACCOUNT_KEY} | base64 --decode > $tmp_file
+echo "${SERVICE_ACCOUNT_KEY}" | base64 --decode > "$tmp_file"
 
-gcloud container hub memberships register ${MEMBERSHIP_NAME} --gke-cluster=${CLUSTER_LOCATION}/${CLUSTER_NAME} --service-account-key-file=${tmp_file} --quiet
+gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --service-account-key-file="${tmp_file}" --quiet

modules/asm/scripts/install_asm.sh

@@ -15,34 +15,52 @@
 
 set -e
 
-if [ "$#" -lt 4 ]; then
+if [ "$#" -lt 3 ]; then
     >&2 echo "Not all expected arguments set."
     exit 1
 fi
 
 PROJECT_ID=$1
 CLUSTER_NAME=$2
 CLUSTER_LOCATION=$3
-ASM_CHANNEL=$4
 ASM_RESOURCES="asm-dir"
 BASE_DIR="asm-base-dir"
+# check for needed binaries
+# kustomize is a requirement for installing ASM and is not available via gcloud. Safely exit if not available.
+if [[ -z $(command -v kustomize) ]]; then
+  echo "kustomize is unavailable. Skipping ASM installation. Please install kustomize, add to PATH and rerun terraform apply."
+  exit 1
+fi
+# check docker which is optionally used for validating asm yaml using gcr.io/kustomize-functions/validate-asm:v0.1.0
+if [[ $(command -v docker) ]]; then
+  echo "Docker is available. ASM yaml validation will be performed."
+else
+  echo "ASM yaml validation will be skipped as Docker is unavailable"
+  SKIP_ASM_VALIDATION=true
+fi
 mkdir -p $ASM_RESOURCES
 pushd $ASM_RESOURCES
-gcloud config set project ${PROJECT_ID}
+gcloud config set project "${PROJECT_ID}"
 if [[ -d ./asm-patch ]]; then
     echo "ASM patch directory exists. Skipping download..."
 else
     echo "Downloading ASM patch"
     kpt pkg get https://github.com/GoogleCloudPlatform/anthos-service-mesh-packages.git/[email protected] .
 fi
-anthoscli export -c ${CLUSTER_NAME} -o ${BASE_DIR} -p ${PROJECT_ID} -l ${CLUSTER_LOCATION}
+anthoscli export -c "${CLUSTER_NAME}" -o ${BASE_DIR} -p "${PROJECT_ID}" -l "${CLUSTER_LOCATION}"
 kpt cfg set asm-patch/ base-dir ../${BASE_DIR}
-kpt cfg set asm-patch/ gcloud.core.project ${PROJECT_ID}
-kpt cfg set asm-patch/ gcloud.container.cluster ${CLUSTER_NAME}
-kpt cfg set asm-patch/ gcloud.compute.location ${CLUSTER_LOCATION}
+kpt cfg set asm-patch/ gcloud.core.project "${PROJECT_ID}"
+kpt cfg set asm-patch/ gcloud.container.cluster "${CLUSTER_NAME}"
+kpt cfg set asm-patch/ gcloud.compute.location "${CLUSTER_LOCATION}"
 kpt cfg list-setters asm-patch/
-pushd ${BASE_DIR} && kustomize create --autodetect --namespace ${PROJECT_ID} && popd
+pushd ${BASE_DIR} && kustomize create --autodetect --namespace "${PROJECT_ID}" && popd
 pushd asm-patch && kustomize build -o ../${BASE_DIR}/all.yaml && popd
-kpt fn source ${BASE_DIR} | kpt fn run --image gcr.io/kustomize-functions/validate-asm:v0.1.0
+# skip validate as we should investigate if we can check this without having to resort to dind
+if [[ ${SKIP_ASM_VALIDATION} ]]; then
+ echo "Skipping ASM validation..."
+else
+ echo "Running ASM validation..."
+ kpt fn source ${BASE_DIR} | kpt fn run --image gcr.io/kustomize-functions/validate-asm:v0.1.0
+fi
 anthoscli apply -f ${BASE_DIR}
 kubectl wait --for=condition=available --timeout=600s deployment --all -n istio-system

modules/asm/scripts/kubectl_wrapper.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -e
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+HOST=$1
+TOKEN=$2
+CA_CERTIFICATE=$3
+
+shift 3
+
+RANDOM_ID="${RANDOM}_${RANDOM}"
+export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}"
+
+function cleanup {
+    rm -rf "${TMPDIR}"
+}
+trap cleanup EXIT
+
+mkdir "${TMPDIR}"
+
+export KUBECONFIG="${TMPDIR}/config"
+
+# shellcheck disable=SC1117
+base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d"
+echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate"
+
+kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null
+rm -f "${TMPDIR}/ca_certificate"
+kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null
+kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null
+kubectl config use-context kubectl-wrapper 1>/dev/null
+kubectl version 1>/dev/null
+
+"$@"

modules/asm/variables.tf

@@ -19,6 +19,11 @@ variable "cluster_name" {
   type        = string
 }
 
+variable "cluster_endpoint" {
+  description = "The GKE cluster endpoint."
+  type        = string
+}
+
 variable "project_id" {
   description = "The project in which the resource belongs."
   type        = string
@@ -35,12 +40,6 @@ variable "skip_gcloud_download" {
   default     = true
 }
 
-variable "asm_release_channel" {
-  description = "ASM Release Channel (REGULAR/RAPID/STABLE)"
-  type        = string
-  default     = "REGULAR"
-}
-
 variable "enable_gke_hub_registration" {
   description = "Enables GKE Hub Registration when set to true"
   type        = bool

test/fixtures/simple_regional_with_asm/example.tf

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+data "google_project" "project" {
+  project_id = var.project_ids[2]
+}
+
 module "example" {
   source = "../../../examples/simple_regional_with_asm"
 

test/fixtures/simple_regional_with_asm/outputs.tf

@@ -78,3 +78,8 @@ output "service_account" {
   description = "The service account to default running nodes as if not overridden in `node_pools`."
   value       = module.example.service_account
 }
+
+output "project_number" {
+  description = "The project number of the ASM project"
+  value       = data.google_project.project.number
+}

test/integration/simple_regional_with_asm/controls/gcloud.rb

@@ -0,0 +1,58 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project_id = attribute('project_id')
+project_number = attribute('project_number')
+location = attribute('location')
+cluster_name = attribute('cluster_name')
+k8s_service_account_email = attribute('k8s_service_account_email')
+k8s_service_account_name = attribute('k8s_service_account_name')
+
+control "gcloud" do
+  title "Google Compute Engine GKE configuration"
+  describe command("gcloud beta --project=#{project_id} container clusters --region=#{location} describe #{cluster_name} --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let!(:data) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+
+    describe "mesh id" do
+      it "is correct" do
+        expect(data['resourceLabels']["mesh_id"]).to eq "proj-#{project_number}"
+      end
+    end
+  end
+
+  describe command("gcloud container hub memberships describe gke-asm-membership --format=json") do
+    its(:exit_status) { should eq 0 }
+    its(:stderr) { should eq '' }
+
+    let!(:hub) do
+      if subject.exit_status == 0
+        JSON.parse(subject.stdout)
+      else
+        {}
+      end
+    end
+    it "membership has expected gke cluster" do
+      expect(hub['endpoint']["gkeCluster"]["resourceLink"]).to eq "//container.googleapis.com/projects/#{project_id}/locations/#{location}/clusters/#{cluster_name}"
+    end
+  end
+end