feat!: Add multi-repo support for Config Sync (#872)

unlimitedbits · morgante · bharathkkb · web-flow · commit 23da1035cf55 · 2021-05-16T23:39:32.000-05:00
* Add multi-repo support for Config Sync

* Run make build

* increase wait time for configsync api

* Update modules/k8s-operator-crd-support/main.tf

Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;

* Enhance wait_for_configsync_api step

Co-authored-by: Morgante Pell &lt;morgante.pell@morgante.net&gt;
Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
Co-authored-by: Morgante Pell &lt;morgantep@google.com&gt;
diff --git a/modules/acm/README.md b/modules/acm/README.md
@@ -49,8 +49,9 @@ By default, this module will attempt to download the ACM operator from Google di
 | cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes |
 | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no |
 | enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no |
+| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no |
 | enable\_policy\_controller | Whether to enable the ACM Policy Controller on the cluster | `bool` | `true` | no |
-| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map` | `null` | no |
+| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details | `map(any)` | `null` | no |
 | install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no |
 | location | GCP location used to reach cluster. | `string` | n/a | yes |
 | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no |
@@ -62,6 +63,7 @@ By default, this module will attempt to download the ACM operator from Google di
 | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |
 | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no |
 | sync\_repo | ACM Git repo address | `string` | n/a | yes |
+| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no |
 | use\_existing\_context | Use existing kubecontext to auth kube-api. | `bool` | `false` | no |
 
 ## Outputs
diff --git a/modules/acm/main.tf b/modules/acm/main.tf
@@ -37,8 +37,10 @@ module "acm_operator" {
   project_id               = var.project_id
   location                 = var.location
   operator_path            = var.operator_path
+  enable_multi_repo        = var.enable_multi_repo
   sync_repo                = var.sync_repo
   sync_branch              = var.sync_branch
+  sync_revision            = var.sync_revision
   policy_dir               = var.policy_dir
   cluster_endpoint         = var.cluster_endpoint
   create_ssh_key           = var.create_ssh_key
@@ -56,4 +58,6 @@ module "acm_operator" {
   operator_cr_template_path     = "${path.module}/templates/acm-config.yml.tpl"
   operator_credential_namespace = "config-management-system"
   operator_credential_name      = "git-creds"
+
+  rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl"
 }
diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl
@@ -9,10 +9,15 @@ spec:
     enabled: ${enable_policy_controller}
     templateLibraryInstalled: ${install_template_library}
     logDeniesEnabled: ${enable_log_denies}
+%{ if enable_multi_repo ~}
+  enableMultiRepo: true
+%{ else ~}
   git:
     syncRepo: ${sync_repo}
     secretType: ${secret_type}
     ${policy_dir_node}
     ${sync_branch_node}
+    ${sync_revision_node}
   ${source_format_node}
+%{ endif ~}
   ${hierarchy_controller_map_node}
diff --git a/modules/acm/templates/root-sync.yml.tpl b/modules/acm/templates/root-sync.yml.tpl
@@ -0,0 +1,14 @@
+apiVersion: configsync.gke.io/v1beta1
+kind: RootSync
+metadata:
+  name: root-sync
+  namespace: config-management-system
+spec:
+  ${source_format_node}
+  git:
+    repo: ${sync_repo}
+    auth: ${secret_type}
+    ${policy_dir_node}
+    ${sync_branch_node}
+    ${sync_revision_node}
+    ${secret_ref_node}
diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf
@@ -35,6 +35,12 @@ variable "operator_path" {
   default     = null
 }
 
+variable "enable_multi_repo" {
+  description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)."
+  type        = bool
+  default     = false
+}
+
 variable "sync_repo" {
   description = "ACM Git repo address"
   type        = string
@@ -46,6 +52,12 @@ variable "sync_branch" {
   default     = ""
 }
 
+variable "sync_revision" {
+  description = "ACM repo Git revision. If un-set, uses Config Management default."
+  type        = string
+  default     = ""
+}
+
 variable "policy_dir" {
   description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default."
   type        = string
@@ -95,7 +107,7 @@ variable "source_format" {
 
 variable "hierarchy_controller" {
   description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/anthos-config-management/docs/how-to/installing-hierarchy-controller) for more details"
-  type        = map
+  type        = map(any)
   default     = null
 }
 
diff --git a/modules/config-sync/README.md b/modules/config-sync/README.md
@@ -50,7 +50,8 @@ To deploy this config:
 | cluster\_endpoint | Kubernetes cluster endpoint. | `string` | n/a | yes |
 | cluster\_name | GCP cluster name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes |
 | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no |
-| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map` | `null` | no |
+| enable\_multi\_repo | Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo). | `bool` | `false` | no |
+| hierarchy\_controller | Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details. | `map(any)` | `null` | no |
 | location | GCP location used to reach cluster. | `string` | n/a | yes |
 | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | `string` | `null` | no |
 | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no |
@@ -60,6 +61,7 @@ To deploy this config:
 | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no |
 | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no |
 | sync\_repo | ACM Git repo address | `string` | n/a | yes |
+| sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no |
 
 ## Outputs
 
diff --git a/modules/config-sync/main.tf b/modules/config-sync/main.tf
@@ -22,8 +22,10 @@ module "configsync_operator" {
   project_id           = var.project_id
   location             = var.location
   operator_path        = var.operator_path
+  enable_multi_repo    = var.enable_multi_repo
   sync_repo            = var.sync_repo
   sync_branch          = var.sync_branch
+  sync_revision        = var.sync_revision
   policy_dir           = var.policy_dir
   cluster_endpoint     = var.cluster_endpoint
   create_ssh_key       = var.create_ssh_key
@@ -36,4 +38,6 @@ module "configsync_operator" {
   operator_cr_template_path     = "${path.module}/templates/config-sync-config.yml.tpl"
   operator_credential_namespace = "config-management-system"
   operator_credential_name      = "git-creds"
+
+  rootsync_cr_template_path = "${path.module}/templates/root-sync.yml.tpl"
 }
diff --git a/modules/config-sync/templates/root-sync.yml.tpl b/modules/config-sync/templates/root-sync.yml.tpl
@@ -0,0 +1,14 @@
+apiVersion: configsync.gke.io/v1beta1
+kind: RootSync
+metadata:
+  name: root-sync
+  namespace: config-management-system
+spec:
+  ${source_format_node}
+  git:
+    repo: ${sync_repo}
+    auth: ${secret_type}
+    ${policy_dir_node}
+    ${sync_branch_node}
+    ${sync_revision_node}
+    ${secret_ref_node}
diff --git a/modules/config-sync/variables.tf b/modules/config-sync/variables.tf
@@ -35,6 +35,12 @@ variable "operator_path" {
   default     = null
 }
 
+variable "enable_multi_repo" {
+  description = "Whether to use ACM Config Sync [multi-repo mode](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/multi-repo)."
+  type        = bool
+  default     = false
+}
+
 variable "sync_repo" {
   description = "ACM Git repo address"
   type        = string
@@ -46,6 +52,12 @@ variable "sync_branch" {
   default     = ""
 }
 
+variable "sync_revision" {
+  description = "ACM repo Git revision. If un-set, uses Config Management default."
+  type        = string
+  default     = ""
+}
+
 variable "policy_dir" {
   description = "Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default."
   type        = string
@@ -82,6 +94,6 @@ variable "source_format" {
 
 variable "hierarchy_controller" {
   description = "Configurations for Hierarchy Controller. See [Hierarchy Controller docs](https://cloud.google.com/kubernetes-engine/docs/add-on/config-sync/how-to/installing-hierarchy-controller) for more details."
-  type        = map
+  type        = map(any)
   default     = null
 }
diff --git a/modules/k8s-operator-crd-support/main.tf b/modules/k8s-operator-crd-support/main.tf
@@ -15,16 +15,21 @@
  */
 
 locals {
-  cluster_endpoint                               = "https://${var.cluster_endpoint}"
-  private_key                                    = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key
-  k8sop_creds_secret_key                         = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type
-  should_download_manifest                       = var.operator_path == null ? true : false
-  manifest_path                                  = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path
-  sync_branch_node                               = var.sync_branch != "" ? format("syncBranch: %s", var.sync_branch) : ""
-  policy_dir_node                                = var.policy_dir != "" ? format("policyDir: %s", var.policy_dir) : ""
-  hierarchy_controller_map_node                  = var.hierarchy_controller == null ? "" : format("hierarchyController:\n    %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")))
-  source_format_node                             = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : ""
-  append_arg_use_existing_context_for_gatekeeper = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : ""
+  cluster_endpoint                = "https://${var.cluster_endpoint}"
+  private_key                     = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.k8sop_creds[0].private_key_pem : var.ssh_auth_key
+  k8sop_creds_secret_key          = var.secret_type == "cookiefile" ? "cookie_file" : var.secret_type
+  should_download_manifest        = var.operator_path == null ? true : false
+  manifest_path                   = local.should_download_manifest ? "${path.root}/.terraform/tmp/${var.project_id}-${var.cluster_name}/config-management-operator.yaml" : var.operator_path
+  sync_branch_property            = var.enable_multi_repo ? "branch" : "syncBranch"
+  sync_branch_node                = var.sync_branch != "" ? format("%s: %s", local.sync_branch_property, var.sync_branch) : ""
+  sync_revision_property          = var.enable_multi_repo ? "revision" : "syncRev"
+  sync_revision_node              = var.sync_revision != "" ? format("%s: %s", local.sync_revision_property, var.sync_revision) : ""
+  policy_dir_property             = var.enable_multi_repo ? "dir" : "policyDir"
+  policy_dir_node                 = var.policy_dir != "" ? format("%s: %s", local.policy_dir_property, var.policy_dir) : ""
+  secret_ref_node                 = var.create_ssh_key == true || var.ssh_auth_key != null ? format("secretRef:\n      name: %s", var.operator_credential_name) : ""
+  hierarchy_controller_map_node   = var.hierarchy_controller == null ? "" : format("hierarchyController:\n    %s", indent(4, replace(yamlencode(var.hierarchy_controller), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")))
+  source_format_node              = var.source_format != "" ? format("sourceFormat: %s", var.source_format) : ""
+  append_arg_use_existing_context = var.use_existing_context ? "USE_EXISTING_CONTEXT_ARG" : ""
 }
 
 module "k8sop_manifest" {
@@ -82,8 +87,10 @@ data "template_file" "k8sop_config" {
   template = file(var.operator_cr_template_path)
   vars = {
     cluster_name                  = var.cluster_name
+    enable_multi_repo             = var.enable_multi_repo
     sync_repo                     = var.sync_repo
     sync_branch_node              = local.sync_branch_node
+    sync_revision_node            = local.sync_revision_node
     policy_dir_node               = local.policy_dir_node
     secret_type                   = var.create_ssh_key ? "ssh" : var.secret_type
     enable_policy_controller      = var.enable_policy_controller ? "true" : "false"
@@ -109,6 +116,58 @@ module "k8sop_config" {
   kubectl_destroy_command = "kubectl delete -f - <<EOF\n${data.template_file.k8sop_config.rendered}EOF"
 }
 
+
+data "template_file" "rootsync_config" {
+
+  template = file(var.rootsync_cr_template_path)
+  vars = {
+    sync_repo          = var.sync_repo
+    sync_branch_node   = local.sync_branch_node
+    sync_revision_node = local.sync_revision_node
+    policy_dir_node    = local.policy_dir_node
+    secret_ref_node    = local.secret_ref_node
+    secret_type        = var.create_ssh_key ? "ssh" : var.secret_type
+    source_format_node = local.source_format_node
+  }
+}
+
+module "wait_for_configsync_api" {
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version = "~> 2.0.2"
+  enabled = var.enable_multi_repo
+
+  module_depends_on = [module.k8sop_config.wait]
+  cluster_name      = var.cluster_name
+  project_id        = var.project_id
+  cluster_location  = var.location
+  create_cmd_triggers = {
+    rootsync    = data.template_file.rootsync_config.rendered,
+    script_sha1 = sha1(file("${path.module}/scripts/wait_for_configsync.sh"))
+  }
+  service_account_key_file = var.service_account_key_file
+  use_existing_context     = var.use_existing_context
+
+  kubectl_create_command  = "${path.module}/scripts/wait_for_configsync.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context}"
+  kubectl_destroy_command = ""
+}
+
+module "rootsync_config" {
+  source  = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version = "~> 2.0.2"
+  enabled = var.enable_multi_repo
+
+  module_depends_on        = [module.wait_for_configsync_api.wait]
+  cluster_name             = var.cluster_name
+  project_id               = var.project_id
+  cluster_location         = var.location
+  create_cmd_triggers      = { rootsync = data.template_file.rootsync_config.rendered }
+  service_account_key_file = var.service_account_key_file
+  use_existing_context     = var.use_existing_context
+
+  kubectl_create_command  = "kubectl apply -f - <<EOF\n${data.template_file.rootsync_config.rendered}EOF"
+  kubectl_destroy_command = "kubectl delete -f - <<EOF\n${data.template_file.rootsync_config.rendered}EOF"
+}
+
 module "wait_for_gatekeeper" {
   source                   = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
   version                  = "~> 2.0.2"
@@ -121,6 +180,6 @@ module "wait_for_gatekeeper" {
   service_account_key_file = var.service_account_key_file
   use_existing_context     = var.use_existing_context
 
-  kubectl_create_command  = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context_for_gatekeeper}"
+  kubectl_create_command  = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location} ${local.append_arg_use_existing_context}"
   kubectl_destroy_command = ""
 }
diff --git a/modules/k8s-operator-crd-support/scripts/wait_for_configsync.sh b/modules/k8s-operator-crd-support/scripts/wait_for_configsync.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+is_configsync_ready() {
+    # Check if config-management-system namespace exists
+    kubectl --context "$1" get namespace config-management-system &> /dev/null
+    export exit_code=$?
+    while [ ! " ${exit_code} " -eq 0 ]
+        do
+            sleep 5
+            echo -e "Waiting for namespace config-mangement-system in cluster $1 to be created..."
+            kubectl --context "$1" get namespace config-management-system &> /dev/null
+            export exit_code=$?
+        done
+    echo -e "Namespace config-management-system in cluster $1 created."
+
+    # Once namespace is created, check if config-managment pods are ready
+    kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null
+    export exit_code=$?
+
+    while [ ! " ${exit_code} " -eq 0 ]
+        do
+            sleep 5
+            echo -e "Waiting for config-management pods in cluster $1 to become ready..."
+            kubectl --context "$1" -n config-management-system wait --timeout 60s --for=condition=Ready pod --all &> /dev/null
+            export exit_code=$?
+        done
+
+    echo -e "Config-management pods in cluster $1 are ready."
+    return
+}
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+PROJECT_ID=$1
+CLUSTER_NAME=$2
+CLUSTER_LOCATION=$3
+USE_EXISTING_CONTEXT=$4
+
+# Check if we need to use the current context
+if [ -z ${USE_EXISTING_CONTEXT+x} ]; then
+    # GKE Cluster. Use the GKE cluster context
+    is_configsync_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}"
+else
+    echo "USE_EXISTING_CONTEXT variable is set. Using current context to wait for deployment to be ready."
+    # Get the current context. This can be used for non GKE Clusters
+    CURRENT_CONTEXT=$(kubectl config current-context)
+    is_configsync_ready "${CURRENT_CONTEXT}"
+fi
diff --git a/modules/k8s-operator-crd-support/variables.tf b/modules/k8s-operator-crd-support/variables.tf
