Merge remote-tracking branch 'origin/stackhpc/2023.1' into sync-caracal-antelope

Author: Alex-Welsh

doc/source/operations/upgrading-ceph.rst

@@ -63,7 +63,7 @@ Place the host or batch of hosts into maintenance mode:
 
 .. code-block:: console
 
-   sudo cephadm shell -- ceph orch host maintenance enter <host>
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml -l <host>
 
 To update all eligible packages, use ``*``, escaping if necessary:
 
@@ -72,7 +72,8 @@ To update all eligible packages, use ``*``, escaping if necessary:
    kayobe overcloud host package update --packages "*" --limit <host>
 
 If the kernel has been upgraded, reboot the host or batch of hosts to pick up
-the change:
+the change. While running this playbook, consider setting ``ANSIBLE_SERIAL`` to
+the maximum number of hosts that can safely reboot concurrently.
 
 .. code-block:: console
 
@@ -82,7 +83,7 @@ Remove the host or batch of hosts from maintenance mode:
 
 .. code-block:: console
 
-   sudo cephadm shell -- ceph orch host maintenance exit <host>
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml -l <host>
 
 Wait for Ceph health to return to ``HEALTH_OK``:
 

etc/kayobe/ansible/ceph-enter-maintenance.yml

@@ -0,0 +1,13 @@
+---
+- name: Ensure a Ceph host has entered maintenance
+  gather_facts: true
+  any_errors_fatal: true
+  # We need to check whether it is OK to stop hosts after previous hosts have
+  # entered maintenance.
+  serial: 1
+  hosts: ceph
+  become: true
+  tasks:
+    - name: Ensure a Ceph host has entered maintenance
+      ansible.builtin.import_role:
+        name: stackhpc.cephadm.enter_maintenance

etc/kayobe/ansible/ceph-exit-maintenance.yml

@@ -0,0 +1,12 @@
+---
+- name: Ensure a Ceph host has exited maintenance
+  gather_facts: true
+  any_errors_fatal: true
+  hosts: ceph
+  # The role currently requires hosts to exit maintenance serially.
+  serial: 1
+  become: true
+  tasks:
+    - name: Ensure a Ceph host has exited maintenance
+      ansible.builtin.import_role:
+        name: stackhpc.cephadm.exit_maintenance

etc/kayobe/ansible/cis.yml

@@ -35,9 +35,7 @@
     - include_role:
         name: ansible-lockdown.rhel9_cis
       when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '9'
-      tags: always
 
     - include_role:
         name: ansible-lockdown.ubuntu22_cis
       when: ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_major_version == '22'
-      tags: always

etc/kayobe/ansible/prometheus-network-names.yml

@@ -1,3 +1,4 @@
+---
 - name: Prometheus friendly network names
   hosts: overcloud
   gather_facts: no

etc/kayobe/ansible/reboot.yml

@@ -2,9 +2,26 @@
 - name: Reboot the host
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}"
+  gather_facts: false
+  vars:
+    reboot_timeout_s: "{{ 20 * 60 }}"
+    reboot_with_bootstrap_user: false
+    ansible_user: "{{ bootstrap_user if reboot_with_bootstrap_user | bool else kayobe_ansible_user }}"
+    ansible_ssh_common_args: "{{ '-o StrictHostKeyChecking=no' if reboot_with_bootstrap_user | bool else '' }}"
+    ansible_python_interpreter: "/usr/bin/python3"
   tags:
     - reboot
   tasks:
     - name: Reboot and wait
       become: true
       reboot:
+        reboot_timeout: "{{ reboot_timeout_s }}"
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"

etc/kayobe/ansible/requirements.yml

@@ -1,7 +1,7 @@
 ---
 collections:
   - name: stackhpc.cephadm
-    version: 1.15.1
+    version: 1.18.0
   # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the
   # pulp_glue Python library being installed.
   - name: pulp.squeezer

etc/kayobe/ansible/stackhpc-openstack-tests.yml

@@ -31,7 +31,7 @@
             depth: 1
             single_branch: true
 
-        - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
+        - name: Ensure the latest versions of pip and setuptools are installed  # noqa package-latest
           ansible.builtin.pip:
             name: "{{ item.name }}"
             state: latest

etc/kayobe/ansible/templates/wazuh-secrets.yml.j2

@@ -7,7 +7,7 @@ secrets_wazuh:
   # Strengthen default wazuh api user pass
   wazuh_api_users:
     - username: "wazuh"
-      password: "{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup('community.general.random_string', min_lower=1, min_upper=1, min_special=1, min_numeric=1, length=30)) }}"
+      password: "{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup('community.general.random_string', min_lower=1, min_upper=1, min_special=1, min_numeric=1, length=30, override_special=override_special_characters)) }}"
   # OpenSearch 'admin' user pass
   opendistro_admin_password: "{{ secrets_wazuh.opendistro_admin_password | default(lookup('password', '/dev/null'), true) }}"
   # OpenSearch 'kibanaserver' user pass

etc/kayobe/ansible/ubuntu-upgrade.yml

@@ -40,6 +40,15 @@
       reboot:
         reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"
       become: true
       when: file_status.stat.exists
 
@@ -101,6 +110,15 @@
       reboot:
         reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
+        search_paths:
+          # Systems running molly-guard hang waiting for confirmation before rebooting without this.
+          - "/lib/molly-guard"
+          # Default list:
+          - "/sbin"
+          - "/bin"
+          - "/usr/sbin"
+          - "/usr/bin"
+          - "/usr/local/sbin"
       become: true
 
     - name: Update distribution facts

etc/kayobe/ansible/wazuh-secrets.yml

@@ -3,6 +3,7 @@
   gather_facts: false
   vars:
     wazuh_secrets_path: "{{ kayobe_env_config_path }}/wazuh-secrets.yml"
+    override_special_characters: '"#$%&()*+,-./:;<=>?@[\]^_{|}~'
   tasks:
     - name: install passlib[bcrypt]
       pip:

releasenotes/notes/ceph-maintenance-4c4eb0a4f7149665.yaml

@@ -0,0 +1,15 @@
+---
+features:
+  - |
+    Adds two new custom playbooks for placing Ceph hosts into and removing them
+    from maintenance:
+
+    - ``ceph-enter-maintenance.yml``
+    - ``ceph-exit-maintenance.yml``
+upgrade:
+  - |
+    Updates the ``stackhpc.cephadm`` collection to version ``1.18.0``.
+fixes:
+  - |
+    Fixes an issue with idempotency in the ``stackhpc.ceph.cephadm_keys``
+    plugin.

releasenotes/notes/rabbit-migration-script-4721c2e37ac9d3c0.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Added a script to automate RabbitMQ quorum queue migrations.

terraform/aio/vm.tf

@@ -35,7 +35,7 @@ variable "aio_vm_subnet" {
 
 variable "aio_vm_volume_size" {
   type = number
-  default = 35
+  default = 40
 }
 
 variable "aio_vm_tags" {

tools/rabbitmq-quorum-migration.sh

@@ -0,0 +1,60 @@
+#! /usr/bin/bash
+
+set -ex
+
+RABBITMQ_SERVICES_TO_RESTART=barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia
+RABBITMQ_CONTAINER_NAME=rabbitmq
+
+if [[ ! $KAYOBE_CONFIG_PATH ]]; then
+    echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined"
+    echo "Ensure your environment is set up to run kayobe commands"
+    exit 2
+fi
+
+if [[ ! "$1" = "--skip-checks" ]]; then
+    # Fail if clocks are not synced
+    if ! kayobe overcloud host command run -l controllers -b --command "timedatectl status | grep 'synchronized: yes'"; then
+        echo "Failed precheck: Time not synced on controllers"
+        echo "Use 'timedatectl status' to check sync state"
+        echo "Either wait for sync or use 'chronyc makestep'"
+        exit 1
+    fi
+    kayobe overcloud service configuration generate --node-config-dir /tmp/rabbit-migration --kolla-tags none
+    # Fail if HA is set or quorum is not
+    if ! grep 'om_enable_rabbitmq_quorum_queues: true' $KOLLA_CONFIG_PATH/globals.yml || grep 'om_enable_rabbitmq_high_availability: true' $KOLLA_CONFIG_PATH/globals.yml; then
+        echo "Failed precheck: om_enable_rabbitmq_quorum_queues must be enabled, om_enable_rabbitmq_high_availability must be disabled"
+        exit 1
+    fi
+fi
+
+# Generate new config, stop services using rabbit, and reset rabbit state
+kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck
+kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt $RABBITMQ_SERVICES_TO_RESTART
+kayobe kolla ansible run rabbitmq-reset-state
+
+if [[ ! "$1" = "--skip-checks" ]]; then
+    # Fail if any queues still exist
+    sleep 20
+    if kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues name --silent | grep -v '^$'"; then
+        echo "Failed check: RabbitMQ has not stopped properly, queues still exist"
+        exit 1
+    fi
+    # Fail if any exchanges still exist (excluding those starting with 'amq.')
+    if kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_exchanges name --silent | grep -v '^$' | grep -v '^amq.'"; then
+        echo "Failed check: RabbitMQ has not stopped properly, exchanges still exist"
+        exit 1
+    fi
+fi
+
+# Redeploy with quorum queues enabled
+kayobe kolla ansible run deploy-containers -kt $RABBITMQ_SERVICES_TO_RESTART
+
+if [[ ! "$1" = "--skip-checks" ]]; then
+    sleep 20
+    # Assert that at least one quorum queue exists on each controller
+    if kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues type | grep quorum"; then
+        echo "Queues migrated successfully" 
+    else
+        echo "Failed post-check: A controller does not have any quorum queues"
+    fi
+fi