Merge pull request #1001 from stackhpc/zed-yoga-merge

Merge yoga to zed

Author: markgoddard

doc/source/configuration/monitoring.rst

@@ -141,58 +141,41 @@ OpenStack Capacity
 ==================
 
 OpenStack Capacity allows you to see how much space you have available
-in your cloud. StackHPC Kayobe Config includes a playbook for manual
-deployment, and it's necessary that some variables are set before
-running this playbook.
+in your cloud. StackHPC Kayobe Config will deploy OpenStack Capacity
+by default on a service deploy, this can be disabled by setting
+``stackhpc_enable_os_capacity`` to false.
 
-To successfully deploy OpenStack Capacity, you are required to specify
-the OpenStack application credentials in ``kayobe/secrets.yml`` as:
+OpenStack Capacity is deployed automatically using a service deploy hook
+with the generated kolla-ansible admin credentials, you can override these
+by setting the authentication url, username, password, project name and
+project domain name in ``stackhpc-monitoring.yml``:
 
 .. code-block:: yaml
 
-    secrets_os_capacity_credential_id: <some_credential_id>
-    secrets_os_capacity_credential_secret: <some_credential_secret>
+    stackhpc_os_capacity_auth_url: <keystone_auth_url>
+    stackhpc_os_capacity_username: <openstack_username>
+    stackhpc_os_capacity_password: <openstack_password_encrypted_with_vault>
+    stackhpc_os_capacity_project_name: <openstack_project_name>
+    stackhpc_os_capacity_domain_name: <openstack_project_domain_name>
+    stackhpc_os_capacity_openstack_region_name: <openstack_region_name>
 
-The Keystone authentication URL and OpenStack region can be changed
-from their defaults in ``stackhpc-monitoring.yml`` should you need to
-set a different OpenStack region for your cloud. The authentication
-URL is set to use ``kolla_internal_fqdn`` by default:
-
-.. code-block:: yaml
-
-    stackhpc_os_capacity_auth_url: <some_authentication_url>
-    stackhpc_os_capacity_openstack_region_name: <some_openstack_region>
-
-Additionally, you are required to enable a conditional flag to allow
-HAProxy and Prometheus configuration to be templated during deployment.
-
-.. code-block:: yaml
-
-    stackhpc_enable_os_capacity: true
-
-If you are deploying in a cloud with internal TLS, you may be required
+Additionally, you should ensure these credentials have the correct permissions
+for the exporter. If you are deploying in a cloud with internal TLS, you may be required
 to disable certificate verification for the OpenStack Capacity exporter
 if your certificate is not signed by a trusted CA.
 
 .. code-block:: yaml
 
     stackhpc_os_capacity_openstack_verify: false
 
-After defining your credentials, you may deploy OpenStack Capacity
-using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook
+If you've modified your credentials, you will need to re-deploy OpenStack Capacity
+using the ``deploy-os-capacity-exporter.yml`` Ansible playbook
 via Kayobe.
 
 .. code-block:: console
 
     kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml
 
-It is required that you re-configure the Prometheus, Grafana and HAProxy
-services following deployment, to do this run the following Kayobe command.
-
-.. code-block:: console
-
-    kayobe overcloud service reconfigure -kt grafana,prometheus,loadbalancer
-
 If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus
 alerts after deployment it's likely the os_exporter secrets have not been
 set correctly, double check you have entered the correct authentication

etc/kayobe/ansible/deploy-os-capacity-exporter.yml

@@ -17,14 +17,33 @@
       ansible.builtin.file:
         path: /opt/kayobe/os-capacity/
         state: directory
+      when: stackhpc_enable_os_capacity
+
+    - name: Read admin-openrc credential file
+      ansible.builtin.command:
+        cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh"
+      delegate_to: localhost
+      register: credential
+      when: stackhpc_enable_os_capacity
+
+    - name: Set facts for admin credentials
+      ansible.builtin.set_fact:
+        stackhpc_os_capacity_auth_url: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_project_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_domain_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_openstack_region_name: "{{ credential.stdout_lines | select('match', '.*OS_REGION_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_username: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_password: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\",'') }}"
+      when: stackhpc_enable_os_capacity
 
     - name: Template clouds.yml
       ansible.builtin.template:
         src: templates/os_capacity-clouds.yml.j2
         dest: /opt/kayobe/os-capacity/clouds.yaml
+      when: stackhpc_enable_os_capacity
 
     - name: Ensure os_capacity container is running
-      docker_container:
+      community.docker.docker_container:
         name: os_capacity
         image: ghcr.io/stackhpc/os-capacity:master
         env:
@@ -37,3 +56,4 @@
         network_mode: host
         restart_policy: unless-stopped
       become: true
+      when: stackhpc_enable_os_capacity

etc/kayobe/ansible/hotfix-containers.yml

@@ -30,13 +30,13 @@
 
     - name: Set fact for containers list
       set_fact:
-        containers_list: host_containers.stdout
+        containers_list: "{{ host_containers.stdout }}"
 
     - name: Fail if no containers match given regex
       vars:
         hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}"
       fail:
-        msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers | split('\n') }}"
+        msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers.stdout_lines }}"
       when: hotfix_containers == ""
 
     - name: Ensure hotfix-files directory exists on the remote host

etc/kayobe/ansible/run-container-hotfix.yml

@@ -20,3 +20,4 @@
 
 - name: Run container_hotfix_command
   command: "{{ kolla_container_engine | default('docker')}} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}"
+  when: container_hotfix_command

etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2

@@ -2,12 +2,14 @@ clouds:
   openstack:
     auth:
       auth_url: "{{ stackhpc_os_capacity_auth_url }}"
-      application_credential_id: "{{ secrets_os_capacity_credential_id }}"
-      application_credential_secret: "{{ secrets_os_capacity_credential_secret }}"
+      project_name: "{{ stackhpc_os_capacity_project_name }}"
+      domain_name: "{{ stackhpc_os_capacity_domain_name }}"
+      username: "{{ stackhpc_os_capacity_username }}"
+      password: "{{ stackhpc_os_capacity_password }}"
     region_name: "{{ stackhpc_os_capacity_openstack_region_name }}"
     interface: "internal"
     identity_api_version: 3
-    auth_type: "v3applicationcredential"
+    auth_type: "password"
 {% if not stackhpc_os_capacity_openstack_verify | bool %}
     verify: False
 {% endif %}

etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml

@@ -0,0 +1 @@
+../../../ansible/deploy-os-capacity-exporter.yml

etc/kayobe/kolla/kolla-build.conf

@@ -9,3 +9,8 @@ base_tag = jammy-20231004
 base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
 {% endif %}
 build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }}
+
+[openstack-base]
+type = git
+location = https://github.com/stackhpc/requirements
+reference = stackhpc/{{ openstack_release }}

etc/kayobe/stackhpc-monitoring.yml

@@ -14,13 +14,7 @@ alertmanager_low_memory_threshold_gib: 5
 # Whether the OpenStack Capacity exporter is enabled.
 # Enabling this flag will result in HAProxy configuration and Prometheus scrape
 # targets being templated during deployment.
-stackhpc_enable_os_capacity: false
-
-# Keystone authentication URL for OpenStack Capacity
-stackhpc_os_capacity_auth_url: "http{% if kolla_enable_tls_internal | bool %}s{% endif %}://{{ kolla_internal_fqdn }}:5000"
-
-# OpenStack region for OpenStack Capacity
-stackhpc_os_capacity_openstack_region_name: "RegionOne"
+stackhpc_enable_os_capacity: true
 
 # Whether TLS certificate verification is enabled for the OpenStack Capacity
 # exporter during Keystone authentication.

releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Automatic deployment for OpenStack Capacity via a Kayobe service
+    deploy hook using kolla admin credentials.
+upgrade:
+  - |
+    OpenStack Capacity no longer uses application credentials. Please
+    delete any previously generated application credentials.