Use Quorum queues by default in RabbitMQ

Author: Alex-Welsh

.github/workflows/stackhpc-all-in-one.yml

@@ -344,6 +344,17 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
         if: inputs.upgrade
 
+      - name: Migrate RabbitMQ queues
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml --tags rabbit-queue-migration
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       - name: Service upgrade
         run: |
           docker run -t --rm \

etc/kayobe/ansible/migrate-rabbitmq-queues.yml

@@ -0,0 +1,84 @@
+---
+# Migrate RabbitMQ queues from HA to Quorum
+# This is primarily used in CI workflows
+
+- name: Migrate RabbitMQ queues
+  hosts: localhost
+  gather_facts: no
+  vars:
+    - rabbitmq_container_name: "rabbitmq"
+    - services_to_restart: "barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia"
+  tasks:
+    - name: Checking timedatectl status
+      command: timedatectl status
+      register: timedatectl_status
+      changed_when: false
+
+    - name: Fail if the clock is not synchronized
+      assert:
+        that:
+          - "'synchronized: yes' not in timedatectl_status.stdout"
+        fail_msg: >
+          timedatectl sees the system clock as unsynchronized.
+          You may need to force synchronisation using `chronyc makestep`.
+          Otherwise, please wait for synchronization.
+
+    - name: Inspect the {{ rabbitmq_container_name }} container
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: inspection
+      become: true
+
+    - name: Ensure the {{ rabbitmq_container_name }} container is running
+      command: "systemctl start kolla-{{ rabbitmq_container_name }}-container.service"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Wait for the {{ rabbitmq_container_name }} container to reach state 'Running'
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: result
+      until: result.stdout == 'true'
+      retries: 10
+      delay: 6
+      become: true
+
+    - name: Wait for the rabbitmq node to automatically start on container start
+      command: "docker exec {{ rabbitmq_container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Migrate RabbitMQ queues
+      shell:
+        cmd: >
+          kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run rabbitmq-reset-state -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run deploy -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      tags: rabbit-queue-migration
+
+    - name: Inspect RabbitMQ queues
+      shell:
+        cmd: "docker exec {{ rabbitmq_container_name }} rabbitmqctl list_queues type"
+      run_once: true
+      delegate_to: localhost
+      register: queues
+      become: true
+
+    - name: Assert that queues have been migrated
+      assert:
+        that: "{{ 'quorum' in queues.stdout }}"
+        fail_msg: Queue migration has failed. Run the migration manually.
+      run_once: true
+      delegate_to: localhost
+
+    - name: Print warning message
+      debug:
+        msg: >
+          Queues have been migrated, now ensure
+          om_enable_rabbitmq_quorum_queues is set to true in kolla/globals.yml
+      run_once: true
+      delegate_to: localhost

etc/kayobe/kolla/globals.yml

@@ -22,11 +22,6 @@ kolla_image_tags:
 # Variables defining which tag to use for each container's image.
 {{ lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tag-vars') }}
 
-#############################################################################
-# RabbitMQ
-
-om_enable_rabbitmq_high_availability: true
-
 #############################################################################
 # Monitoring and alerting related settings
 

releasenotes/notes/rabbit-disable-ha-9b150bcc9e4d4f54.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    RabbitMQ will now use quorum queues by default, rather than traditional HA.
+upgrade:
+  - |
+    RabbitMQ will now use quorum queues by default. See the `kolla-ansible
+    documentation
+    <https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability>`__
+    for migration advice.