2023.1: Various fixes & improvements to HCP Vault

doc/source/configuration/vault.rst

@@ -84,47 +84,6 @@ Setup Vault on the seed node
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.key
 
-Setup HAProxy config for Vault
-------------------------------
-
-1. Create the HAProxy config to reverse proxy the Vault HA container
-
-   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
-
-   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
-
-   .. code-block::
-
-      # Delete "verify none" if not using self-signed/unknown issuer
-      {% raw %}
-      frontend vault_front
-         mode tcp
-         option tcplog
-         bind {{ kolla_internal_vip_address }}:8200
-         default_backend vault_back
-
-      backend vault_back
-         mode tcp
-         option httpchk GET /v1/sys/health
-         # https://www.vaultproject.io/api-docs/system/health
-         # 200: initialized, unsealed, and active
-         # 501: not initialised (required for bootstrapping)
-         # 503: sealed (required for bootstrapping)
-         http-check expect rstatus (200|501|503)
-
-      {% for host in groups['control'] %}
-      {% set host_name = hostvars[host].ansible_facts.hostname %}
-      {% set host_ip = 'api' | kolla_address(host) %}
-         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
-      {% endfor %}
-      {% endraw %}
-
-2. Deploy HAProxy with the new Vault service configuration:
-
-   .. code-block::
-
-      kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
-
 Setup Vault HA on the overcloud hosts
 -------------------------------------
 
@@ -215,6 +174,55 @@ Create the backend TLS and RabbitMQ TLS certificates
 
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/<controller>-key.pem
 
+.. _vault-haproxy:
+
+HAProxy integration
+===================
+
+It is possible to expose the overcloud Vault service via the Kolla Ansible HAProxy load balancer.
+This provides a single highly available API endpoint, as well as monitoring of the Vault backends when combined with Prometheus.
+HAProxy integration is no longer required for generating OpenStack control plane certificates, making it possible to deploy Vault and generate certificates before any containers have been deployed by Kolla Ansible.
+
+1. Create the HAProxy config to reverse proxy the Vault HA container
+
+   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
+
+   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
+
+   .. code-block::
+
+      # Delete "verify none" if not using self-signed/unknown issuer
+      {% raw %}
+      frontend vault_front
+         mode tcp
+         option tcplog
+         bind {{ kolla_internal_vip_address }}:8200
+         default_backend vault_back
+
+      backend vault_back
+         mode tcp
+         option httpchk GET /v1/sys/health
+         # https://www.vaultproject.io/api-docs/system/health
+         # 200: initialized, unsealed, and active
+         # 429: standby
+         http-check expect rstatus (200|429)
+
+      {% for host in groups['control'] %}
+      {% set host_name = hostvars[host].ansible_facts.hostname %}
+      {% set host_ip = 'api' | kolla_address(host) %}
+         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
+      {% endfor %}
+      {% endraw %}
+
+2. If HAProxy has not yet been deployed, continue to :ref:`certificates deployment <vault-certificates>`.
+   If HAProxy has been deployed, it may be redeployed with the new Vault service configuration:
+
+   .. code-block::
+
+      kayobe overcloud service deploy -kt haproxy
+
+.. _vault-certificates:
+
 Certificates deployment
 =======================
 
@@ -231,6 +239,7 @@ Enable the required TLS variables in kayobe and kolla
 
       # Whether TLS is enabled for the external API endpoints. Default is 'no'.
       kolla_enable_tls_external: yes
+      kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
 
    See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest.
 
@@ -240,6 +249,7 @@ Enable the required TLS variables in kayobe and kolla
 
       # Whether TLS is enabled for the internal API endpoints. Default is 'no'.
       kolla_enable_tls_internal: yes
+      kolla_admin_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
 
 3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
 
@@ -289,6 +299,8 @@ Enable the required TLS variables in kayobe and kolla
 Barbican integration
 ====================
 
+Barbican integration depends on :ref:`HAProxy integration <vault-haproxy>`.
+
 Enable Barbican in kayobe
 -------------------------
 
@@ -339,7 +351,7 @@ Configure Barbican
       enabled_secretstore_plugins=vault_plugin
 
       [vault_plugin]
-      vault_url = https://{{ kolla_internal_vip_address }}:8200
+      vault_url = https://{{ kolla_internal_fqdn }}:8200
       use_ssl = True
       {% raw %}
       ssl_ca_crt_file = {{ openstack_cacert }}

etc/kayobe/ansible/requirements.yml

@@ -9,7 +9,7 @@ collections:
   - name: stackhpc.pulp
     version: 0.5.5
   - name: stackhpc.hashicorp
-    version: 2.4.0
+    version: 2.5.0
   - name: stackhpc.kayobe_workflows
     version: 1.0.3
 roles:

etc/kayobe/ansible/vault-deploy-barbican.yml

@@ -4,7 +4,7 @@
   gather_facts: True
   hosts: controllers[0]
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
   tasks:
     - name: Assert that secrets_barbican_approle_secret_id is defined
@@ -25,79 +25,82 @@
         extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
         virtualenv: "{{ virtualenv_path }}/kayobe"
 
-    - name: Enable AppRole auth module
-      hashivault_auth_method:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        method_type: approle
-        state: enabled
+    - environment:
+        https_proxy: ''
+      block:
+        - name: Enable AppRole auth module
+          hashivault_auth_method:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            method_type: approle
+            state: enabled
 
-    - name: Enable barbican kv store
-      hashivault_secret_engine:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: barbican
-        backend: kv
-        description: "Barbican kv store"
+        - name: Enable barbican kv store
+          hashivault_secret_engine:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: barbican
+            backend: kv
+            description: "Barbican kv store"
 
-    - name: Ensure barbican policy is defined
-      hashivault_policy:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: "barbican-policy"
-        state: present
-        rules: |
-          path "barbican/*" {
-            capabilities = ["create", "read", "update", "delete", "list"]
-          }
+        - name: Ensure barbican policy is defined
+          hashivault_policy:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: "barbican-policy"
+            state: present
+            rules: |
+              path "barbican/*" {
+                capabilities = ["create", "read", "update", "delete", "list"]
+              }
 
-    - name: Ensure barbican AppRole is defined
-      hashivault_approle_role:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        bind_secret_id: true
-        secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}"
-        secret_id_ttl: 0
-        token_policies: barbican-policy
-        name: barbican
+        - name: Ensure barbican AppRole is defined
+          hashivault_approle_role:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            bind_secret_id: true
+            secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}"
+            secret_id_ttl: 0
+            token_policies: barbican-policy
+            name: barbican
 
-    - name: Get barbican Approle ID
-      hashivault_approle_role_id:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: barbican
-      register: barbican_role_id
+        - name: Get barbican Approle ID
+          hashivault_approle_role_id:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: barbican
+          register: barbican_role_id
 
-    - name: Print barbican Approle ID
-      debug:
-        msg: "barbican role id is {{ barbican_role_id.id }}"
+        - name: Print barbican Approle ID
+          debug:
+            msg: "barbican role id is {{ barbican_role_id.id }}"
 
-    - name: Write barbican Approle ID to file if requested
-      delegate_to: localhost
-      copy:
-        content: "{{ barbican_role_id.id }}"
-        dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}"
-      when: stackhpc_write_barbican_role_id_to_file | default(false) | bool
+        - name: Write barbican Approle ID to file if requested
+          delegate_to: localhost
+          copy:
+            content: "{{ barbican_role_id.id }}"
+            dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}"
+          when: stackhpc_write_barbican_role_id_to_file | default(false) | bool
 
-    - name: Check if barbican Approle Secret ID is defined
-      hashivault_approle_role_secret_get:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        secret: "{{ secrets_barbican_approle_secret_id }}"
-        name: barbican
-      register: barbican_approle_secret_get
+        - name: Check if barbican Approle Secret ID is defined
+          hashivault_approle_role_secret_get:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            secret: "{{ secrets_barbican_approle_secret_id }}"
+            name: barbican
+          register: barbican_approle_secret_get
 
-    - name: Ensure barbican AppRole Secret ID is defined
-      hashivault_approle_role_secret:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        secret: "{{ secrets_barbican_approle_secret_id }}"
-        name: barbican
-      when: barbican_approle_secret_get.status == "absent"
+        - name: Ensure barbican AppRole Secret ID is defined
+          hashivault_approle_role_secret:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            secret: "{{ secrets_barbican_approle_secret_id }}"
+            name: barbican
+          when: barbican_approle_secret_get.status == "absent"

etc/kayobe/ansible/vault-deploy-overcloud.yml

@@ -85,9 +85,12 @@
     - import_role:
         name: stackhpc.hashicorp.vault_unseal
       vars:
-        # NOTE: Need to unseal each backend, so don't use the VIP
-        vault_api_addr: "http://localhost:8200"
+        vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
+        vault_unseal_token: "{{ vault_keys.root_token }}"
+        vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
+      environment:
+        https_proxy: ''
 
 - name: Configure PKI
   any_errors_fatal: true
@@ -108,3 +111,5 @@
         vault_pki_intermediate_roles: "{{ overcloud_vault_pki_roles }}"
         vault_pki_write_certificate_files: true
         vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/vault"
+      environment:
+        https_proxy: ''

etc/kayobe/ansible/vault-generate-backend-tls.yml

@@ -18,7 +18,7 @@
 - name: Generate backend API certificates
   hosts: controllers:network
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
     - name: Set a fact about the virtualenv on the remote system
@@ -53,6 +53,8 @@
         extra_params:
           ip_sans: "{{ internal_net_name | net_ip }}"
       register: backend_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:

etc/kayobe/ansible/vault-generate-internal-tls.yml

@@ -3,7 +3,7 @@
   hosts: controllers
   run_once: true
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
     - name: Include Vault keys
@@ -22,6 +22,8 @@
         extra_params:
           ip_sans: "{{ kolla_internal_vip_address }}"
       register: internal_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:

etc/kayobe/ansible/vault-generate-test-external-tls.yml

@@ -3,7 +3,7 @@
   hosts: controllers
   run_once: true
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     # NOTE: Using the same CA as internal TLS.
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
@@ -23,6 +23,8 @@
         extra_params:
           ip_sans: "{{ kolla_external_vip_address }}"
       register: external_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:

etc/kayobe/ansible/vault-unseal-overcloud.yml

@@ -28,6 +28,9 @@
     - import_role:
         name: stackhpc.hashicorp.vault_unseal
       vars:
-        # NOTE: Need to unseal each backend, so don't use the VIP
-        vault_api_addr: "http://localhost:8200"
+        vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
+        vault_unseal_token: "{{ vault_keys.root_token }}"
+        vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
+      environment:
+        https_proxy: ''