Updated dev tools docker image tag to 0.4.3

 * Fixed lint tests
 * Updated Makefile

Author: paulpalamarchuk

Makefile

@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.3
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
@@ -27,7 +27,7 @@ REGISTRY_URL := gcr.io/cloud-foundation-cicd
 docker_run:
 	docker run --rm -it \
 		-e SERVICE_ACCOUNT_JSON \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/bin/bash
 
@@ -39,7 +39,7 @@ docker_test_prepare:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh prepare_environment
 
@@ -51,7 +51,7 @@ docker_test_cleanup:
 		-e TF_VAR_org_id \
 		-e TF_VAR_folder_id \
 		-e TF_VAR_billing_account \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/execute_with_credentials.sh cleanup_environment
 
@@ -60,31 +60,31 @@ docker_test_cleanup:
 docker_test_integration:
 	docker run --rm -it \
 		-e SERVICE_ACCOUNT_JSON \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_integration.sh
 
 # Execute lint tests within the docker container
 .PHONY: docker_test_lint
 docker_test_lint:
 	docker run --rm -it \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_lint.sh
 
 # Generate documentation
 .PHONY: docker_generate_docs
 docker_generate_docs:
 	docker run --rm -it \
-		-v $(CURDIR):/workspace \
+		-v "$(CURDIR)":/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs'
 
 # Generate files from autogen
 .PHONY: docker_generate
 docker_generate:
 	docker run --rm -it \
-                -v $(CURDIR):/workspace \
+                -v "$(CURDIR)":/workspace \
                 $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
                 /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate'
 

autogen/README.md

@@ -195,8 +195,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |

autogen/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

build/int.cloudbuild.yaml

@@ -38,4 +38,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'

build/lint.cloudbuild.yaml

@@ -24,4 +24,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.3'

modules/beta-private-cluster-update-variant/README.md

@@ -188,8 +188,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
@@ -258,6 +259,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 

modules/beta-private-cluster-update-variant/main.tf

@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google-beta
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version

modules/beta-private-cluster-update-variant/networks.tf

@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google-beta
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }

modules/beta-private-cluster-update-variant/sa.tf

@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }

modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

modules/beta-private-cluster-update-variant/variables.tf

@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -267,6 +268,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."

modules/beta-private-cluster/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

modules/beta-public-cluster/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

modules/private-cluster-update-variant/README.md

@@ -174,8 +174,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |
-| region | The region to host the cluster in (required) | string | n/a | yes |
+| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
+| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `<map>` | no |
@@ -237,6 +238,9 @@ following project roles:
 - roles/iam.serviceAccountUser
 - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`)
 
+Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project:
+- roles/resourcemanager.projectIamAdmin
+
 ### Enable APIs
 In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created:
 

modules/private-cluster-update-variant/main.tf

@@ -23,7 +23,7 @@ data "google_compute_zones" "available" {
   provider = google
 
   project = var.project_id
-  region  = var.region
+  region  = local.region
 }
 
 resource "random_shuffle" "available_zones" {
@@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" {
 locals {
   // location
   location = var.regional ? var.region : var.zones[0]
+  region   = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region
   // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted
   node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones))
   // kuberentes version

modules/private-cluster-update-variant/networks.tf

@@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" {
   provider = google
 
   name    = var.subnetwork
-  region  = var.region
+  region  = local.region
   project = local.network_project_id
 }

modules/private-cluster-update-variant/sa.tf

@@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer"
 
 resource "google_project_iam_member" "cluster_service_account-gcr" {
   count   = var.create_service_account && var.grant_registry_access ? 1 : 0
-  project = var.project_id
+  project = var.registry_project_id == "" ? var.project_id : var.registry_project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.cluster_service_account[0].email}"
 }

modules/private-cluster-update-variant/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

modules/private-cluster-update-variant/variables.tf

@@ -40,7 +40,8 @@ variable "regional" {
 
 variable "region" {
   type        = string
-  description = "The region to host the cluster in (required)"
+  description = "The region to host the cluster in (optional if zonal cluster / required if regional)"
+  default     = null
 }
 
 variable "zones" {
@@ -257,6 +258,12 @@ variable "grant_registry_access" {
   default     = false
 }
 
+variable "registry_project_id" {
+  type        = string
+  description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project."
+  default     = ""
+}
+
 variable "service_account" {
   type        = string
   description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created."

modules/private-cluster/scripts/wait-for-cluster.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,11 @@
 
 set -e
 
+# shellcheck disable=SC2034
+if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+
 PROJECT=$1
 CLUSTER_NAME=$2
 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json"

scripts/wait-for-cluster.sh

@@ -15,6 +15,7 @@
 
 set -e
 
+# shellcheck disable=SC2034
 if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
     CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS}
 fi

test/task_helper_functions.sh

@@ -32,7 +32,7 @@ function check_generate() {
     --exclude '*/.kitchen' \
     --exclude '*/.git' \
     /workspace "${tempdir}" >/dev/null 2>/dev/null
-  cd "${tempdir}" || exit 1
+  cd "${tempdir}/workspace" || exit 1
   generate >/dev/null 2>/dev/null
   diff -r \
     --exclude=".terraform" \
@@ -49,16 +49,3 @@ function check_generate() {
   rm -Rf "${tempdir}"
   return $((rval))
 }
-
-find_files() {
-  local pth="$1"
-  shift
-    find "${pth}" '(' \
-    -path '*/.git' -o \
-    -path '*/.terraform' -o \
-    -path '*/.kitchen' -o \
-    -path './autogen' -o \
-    -path './test/fixtures/all_examples' -o \
-    -path './test/fixtures/shared' ')' \
-    -prune -o -type f "$@"
-}