Merge branch 'stackhpc/2024.1' into caracal-odds-and-ends

Author: markgoddard

.automation

@@ -264,6 +264,28 @@ jobs:
         run: |
           docker image pull $KAYOBE_IMAGE
 
+      # Rocky 9 OVN deployments will fail when the hostname contains a '.'
+      - name: Fix hostname
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/fix-hostname.yml
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      # Reboot to Apply hostname change
+      - name: Reboot
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml -e reboot_with_bootstrap_user=true
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Run growroot
         run: |
           docker run -t --rm \
@@ -304,10 +326,29 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Change rabbit queues from HA to Quorum
+        run: |
+          sed -i -e 's/om_enable_rabbitmq_high_availability: true/om_enable_rabbitmq_high_availability: false/' \
+                 -e 's/om_enable_rabbitmq_quorum_queues: false/om_enable_rabbitmq_quorum_queues: true/' \
+                 etc/kayobe/environments/ci-aio/kolla/globals.yml
+        if: inputs.upgrade
+
+      - name: Migrate RabbitMQ queues
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/script-run.sh tools/rabbitmq-quorum-migration.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       # If testing upgrade, checkout the current release branch
       # Stash changes to tracked files, and set clean=false to avoid removing untracked files.
+      # Revert changes to RabbitMQ Queue types to avoid a merge conflict
       - name: Stash config changes
-        run: git stash
+        run: git restore etc/kayobe/environments/ci-aio/kolla/globals.yml && git stash
         if: inputs.upgrade
 
       - name: Checkout current release config

.github/workflows/stackhpc-all-in-one.yml

@@ -345,6 +345,10 @@ should be used in the Kolla Manila configuration e.g.:
 RADOS Gateways
 --------------
 
+RADOS Gateway integration is described in the :kolla-ansible-doc:`Kolla Ansible
+documentation
+<https://docs.openstack.org/kolla-ansible/latest/reference/storage/external-ceph-guide.html#radosgw>`.
+
 RADOS Gateways (RGWs) are defined with the following:
 
 .. code:: yaml
@@ -375,7 +379,7 @@ The set of commands below configure all of these.
   - "config set client.rgw rgw_enable_apis 's3, swift, swift_auth, admin'"
   - "config set client.rgw rgw_enforce_swift_acls true"
   - "config set client.rgw rgw_keystone_accepted_admin_roles 'admin'"
-  - "config set client.rgw rgw_keystone_accepted_roles 'member, Member, _member_, admin'"
+  - "config set client.rgw rgw_keystone_accepted_roles 'member, admin'"
   - "config set client.rgw rgw_keystone_admin_domain Default"
   - "config set client.rgw rgw_keystone_admin_password {{ secrets_ceph_rgw_keystone_password }}"
   - "config set client.rgw rgw_keystone_admin_project service"
@@ -391,6 +395,12 @@ The set of commands below configure all of these.
   - "config set client.rgw rgw_swift_account_in_url true"
   - "config set client.rgw rgw_swift_versioning_enabled true"
 
+Enable the Kolla Ansible RADOS Gateway integration in ``kolla.yml``:
+
+.. code:: yaml
+
+   kolla_enable_ceph_rgw: true
+
 As we have configured Ceph to respond to Swift APIs, you will need to tell
 Kolla to account for this when registering Swift endpoints with Keystone. Also,
 when ``rgw_swift_account_in_url`` is set, the equivalent Kolla variable should
@@ -412,6 +422,11 @@ before deploying the RADOS gateways. If you are using the Kolla load balancer
 
   kayobe overcloud service deploy -kt ceph-rgw,keystone,haproxy,loadbalancer
 
+There are two options for load balancing RADOS Gateway:
+
+1. HA with Ceph Ingress services
+2. RGWs with hyper-converged Ceph (using the Kolla Ansible deployed HAProxy
+   load balancer)
 
 .. _RGWs-with-hyper-converged-Ceph:
 

doc/source/_static/images/release-train.svg

@@ -60,12 +60,12 @@ To deploy the CAPI management cluster using this site-specific environment, run
 
 .. code-block:: bash
 
-    # Activate the environment
-    ./bin/activate <site-specific-name>
-
     # Install or update the local Ansible Python venv
     ./bin/ensure-venv
 
+    # Activate the environment
+    source bin/activate <site-specific-name>
+
     # Install or update Ansible dependencies
     ansible-galaxy install -f -r ./requirements.yml
 
@@ -103,12 +103,7 @@ To configure the Magnum service with the Cluster API driver enabled, first ensur
 
 Next, copy the CAPI management cluster's kubeconfig file into your stackhpc-kayobe-config environment (e.g. ``<your-skc-environment>/kolla/config/magnum/kubeconfig``). This file must be Ansible vault encrypted.
 
-The following config should also be set in your stackhpc-kayobe-config environment:
-
-.. code-block:: yaml
-    :caption: kolla/globals.yml
-
-    magnum_capi_helm_driver_enabled: true
+The presence of a kubeconfig file in the Magnum config directory is used by Kolla to determine whether the CAPI Helm driver should be enabled.
 
 To apply the configuration, run ``kayobe overcloud service reconfigure -kt magnum``.
 

doc/source/configuration/cephadm.rst

@@ -12,7 +12,6 @@ The short version
    particular the defaults assume that the ``provision_oc_net`` network will be
    used.
 #. Generate secrets: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml``
-#. Encrypt the secrets: ``ansible-vault encrypt --vault-password-file ~/vault.password  $KAYOBE_CONFIG_PATH/environments/ci-multinode/wazuh-secrets.yml``
 #. Deploy the Wazuh manager: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml``
 #. Deploy the Wazuh agents: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml``
 
@@ -250,7 +249,6 @@ It will be used by wazuh secrets playbook to generate wazuh secrets vault file.
 .. code-block:: console
 
   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml
-  ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/wazuh-secrets.yml
 
 Configure Wazuh Dashboard's Server Host
 ---------------------------------------

doc/source/configuration/magnum-capi.rst

@@ -63,7 +63,7 @@ Place the host or batch of hosts into maintenance mode:
 
 .. code-block:: console
 
-   sudo cephadm shell -- ceph orch host maintenance enter <host>
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml -l <host>
 
 To update all eligible packages, use ``*``, escaping if necessary:
 
@@ -72,7 +72,8 @@ To update all eligible packages, use ``*``, escaping if necessary:
    kayobe overcloud host package update --packages "*" --limit <host>
 
 If the kernel has been upgraded, reboot the host or batch of hosts to pick up
-the change:
+the change. While running this playbook, consider setting ``ANSIBLE_SERIAL`` to
+the maximum number of hosts that can safely reboot concurrently.
 
 .. code-block:: console
 
@@ -82,7 +83,7 @@ Remove the host or batch of hosts from maintenance mode:
 
 .. code-block:: console
 
-   sudo cephadm shell -- ceph orch host maintenance exit <host>
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml -l <host>
 
 Wait for Ceph health to return to ``HEALTH_OK``:
 

doc/source/configuration/wazuh.rst

@@ -0,0 +1,13 @@
+---
+- name: Ensure a Ceph host has entered maintenance
+  gather_facts: true
+  any_errors_fatal: true
+  # We need to check whether it is OK to stop hosts after previous hosts have
+  # entered maintenance.
+  serial: 1
+  hosts: ceph
+  become: true
+  tasks:
+    - name: Ensure a Ceph host has entered maintenance
+      ansible.builtin.import_role:
+        name: stackhpc.cephadm.enter_maintenance

doc/source/operations/upgrading-ceph.rst

@@ -0,0 +1,12 @@
+---
+- name: Ensure a Ceph host has exited maintenance
+  gather_facts: true
+  any_errors_fatal: true
+  hosts: ceph
+  # The role currently requires hosts to exit maintenance serially.
+  serial: 1
+  become: true
+  tasks:
+    - name: Ensure a Ceph host has exited maintenance
+      ansible.builtin.import_role:
+        name: stackhpc.cephadm.exit_maintenance

etc/kayobe/ansible/ceph-enter-maintenance.yml

@@ -35,9 +35,7 @@
     - include_role:
         name: ansible-lockdown.rhel9_cis
       when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '9'
-      tags: always
 
     - include_role:
         name: ansible-lockdown.ubuntu22_cis
       when: ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_major_version == '22'
-      tags: always

etc/kayobe/ansible/ceph-exit-maintenance.yml

@@ -1,6 +1,6 @@
 ---
-- name: Fix hostname on storage nodes for cephadm
-  hosts: storage
+- name: Ensure hostnames match inventory hostnames
+  hosts: fix-hostname
   gather_facts: false
   vars:
     ansible_user: "{{ bootstrap_user }}"

etc/kayobe/ansible/cis.yml

@@ -1,3 +1,4 @@
+---
 - name: Prometheus friendly network names
   hosts: overcloud
   gather_facts: no

etc/kayobe/ansible/fix-hostname.yml

@@ -2,9 +2,26 @@
 - name: Reboot the host
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}"
+  gather_facts: false
+  vars:
+    reboot_timeout_s: "{{ 20 * 60 }}"
+    reboot_with_bootstrap_user: false
+    ansible_user: "{{ bootstrap_user if reboot_with_bootstrap_user | bool else kayobe_ansible_user }}"
+    ansible_ssh_common_args: "{{ '-o StrictHostKeyChecking=no' if reboot_with_bootstrap_user | bool else '' }}"
+    ansible_python_interpreter: "/usr/bin/python3"
   tags:
     - reboot
   tasks:
     - name: Reboot and wait
       become: true
       reboot:
+        reboot_timeout: "{{ reboot_timeout_s }}"
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"

etc/kayobe/ansible/prometheus-network-names.yml

@@ -1,15 +1,15 @@
 ---
 collections:
   - name: stackhpc.cephadm
-    version: 1.15.1
+    version: 1.18.0
   # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the
   # pulp_glue Python library being installed.
   - name: pulp.squeezer
     version: 0.0.13
   - name: stackhpc.pulp
     version: 0.5.5
   - name: stackhpc.hashicorp
-    version: 2.5.0
+    version: 2.5.1
   - name: stackhpc.kayobe_workflows
     version: 1.0.3
 roles:

etc/kayobe/ansible/reboot.yml

@@ -31,7 +31,7 @@
             depth: 1
             single_branch: true
 
-        - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
+        - name: Ensure the latest versions of pip and setuptools are installed  # noqa package-latest
           ansible.builtin.pip:
             name: "{{ item.name }}"
             state: latest

etc/kayobe/ansible/requirements.yml

@@ -7,7 +7,7 @@ secrets_wazuh:
   # Strengthen default wazuh api user pass
   wazuh_api_users:
     - username: "wazuh"
-      password: "{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup('community.general.random_string', min_lower=1, min_upper=1, min_special=1, min_numeric=1, length=30)) }}"
+      password: "{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup('community.general.random_string', min_lower=1, min_upper=1, min_special=1, min_numeric=1, length=30, override_special=override_special_characters)) }}"
   # OpenSearch 'admin' user pass
   opendistro_admin_password: "{{ secrets_wazuh.opendistro_admin_password | default(lookup('password', '/dev/null'), true) }}"
   # OpenSearch 'kibanaserver' user pass

etc/kayobe/ansible/stackhpc-openstack-tests.yml

@@ -40,6 +40,15 @@
       reboot:
         reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"
       become: true
       when: file_status.stat.exists
 
@@ -101,6 +110,15 @@
       reboot:
         reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"
       become: true
 
     - name: Update distribution facts

etc/kayobe/ansible/templates/wazuh-secrets.yml.j2

@@ -3,6 +3,7 @@
   gather_facts: false
   vars:
     wazuh_secrets_path: "{{ kayobe_env_config_path }}/wazuh-secrets.yml"
+    override_special_characters: '"#$%&()*+,-./:;<=>?@[\]^_{|}~'
   tasks:
     - name: install passlib[bcrypt]
       pip:

etc/kayobe/ansible/ubuntu-upgrade.yml

@@ -74,15 +74,15 @@ stackhpc_apt_repositories:
 # Do not replace apt configuration for non-overcloud hosts. This can result in
 # errors if apt reconfiguration is performed before local repository mirrors
 # are deployed.
-apt_repositories: "{{ stackhpc_apt_repositories | selectattr('required') | list if 'overcloud' in group_names else [] }}"
+apt_repositories: "{{ stackhpc_apt_repositories | selectattr('required') | list if stackhpc_repos_enabled | bool else [] }}"
 
 # Whether to disable repositories in /etc/apt/sources.list. This may be used
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
 # Do not disable the default apt configuration for non-overcloud hosts. This
 # can result in errors if apt reconfiguration is performed before local
 # repository mirrors are deployed.
-apt_disable_sources_list: "{{ 'overcloud' in group_names }}"
+apt_disable_sources_list: "{{ stackhpc_repos_enabled | bool }}"
 
 # Apt auth configuration for accessing the package repository mirror.
 stackhpc_apt_auth:
@@ -98,7 +98,7 @@ stackhpc_apt_auth:
 # * filename: Name of a file in which to store the auth configuration. The
 #   extension should be '.conf'.
 # Default is an empty list.
-apt_auth: "{{ stackhpc_apt_auth if 'overcloud' in group_names and stackhpc_repo_mirror_username is truthy else [] }}"
+apt_auth: "{{ stackhpc_apt_auth if stackhpc_repos_enabled | bool and stackhpc_repo_mirror_username is truthy else [] }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/ansible/wazuh-secrets.yml

@@ -41,10 +41,10 @@
 #     file: myrepo
 #     gpgkey: http://gpgkey
 #     gpgcheck: yes
-#dnf_custom_repos:
+dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [] }}"
 
 # A dict of custom repositories that point to the local Pulp server.
-# To use these repos, set dnf_custom_repos to the value of stackhpc_dnf_repos.
+# To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
 stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"

etc/kayobe/apt.yml

@@ -25,4 +25,4 @@ ansible-galaxy collection install -p ansible/collections -r requirements.yml
 source $BASE_PATH/src/kayobe-config/etc/kolla/public-openrc.sh
 
 # Run script to configure openstack cloud
-tools/openstack-config 
+tools/openstack-config

etc/kayobe/dnf.yml

@@ -26,4 +26,4 @@ storage-ceph
 # Monitoring groups
 
 [monitoring:children]
-controllers
+controllers

etc/kayobe/environments/aufn-ceph/configure-openstack.sh

@@ -3,3 +3,6 @@
 [container-image-builders:children]
 # Build container images on the all-in-one controller.
 controllers
+
+[fix-hostname:children]
+controllers