Add RabbitMQ Quorum queue migration playbook

Author: Alex-Welsh

etc/kayobe/ansible/migrate-rabbitmq-queues.yml

@@ -0,0 +1,114 @@
+---
+# Migrate RabbitMQ queues from HA to Quorum
+# This is primarily used in CI workflows
+
+- name: Migrate RabbitMQ queues
+  hosts: localhost
+  gather_facts: no
+  vars:
+    - rabbitmq_container_name: "rabbitmq"
+    - services_to_restart: "barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia"
+  tasks:
+    - name: Checking timedatectl status
+      command: timedatectl status
+      register: timedatectl_status
+      changed_when: false
+
+    - name: Generate kolla configuration
+      shell:
+        cmd: >
+          kayobe overcloud service configuration generate --node-config-dir /tmp/rabbit-migration --kolla-tags none
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Fail if HA is set or Quorum is unset
+      shell:
+        cmd: >
+          grep 'om_enable_rabbitmq_quorum_queues: true' $KOLLA_CONFIG_PATH/globals.yml &&
+          ! grep 'om_enable_rabbitmq_high_availability: true' $KOLLA_CONFIG_PATH/globals.yml
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Fail if the clock is not synchronized
+      assert:
+        that:
+          - "'synchronized: yes' not in timedatectl_status.stdout"
+        fail_msg: >
+          timedatectl sees the system clock as unsynchronized.
+          You may need to force synchronisation using `chronyc makestep`.
+          Otherwise, please wait for synchronization.
+
+    - name: Inspect the {{ rabbitmq_container_name }} container
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: inspection
+      become: true
+
+    - name: Ensure the {{ rabbitmq_container_name }} container is running
+      command: "systemctl start kolla-{{ rabbitmq_container_name }}-container.service"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Wait for the {{ rabbitmq_container_name }} container to reach state 'Running'
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: result
+      until: result.stdout == 'true'
+      retries: 10
+      delay: 6
+      become: true
+
+    - name: Wait for the rabbitmq node to automatically start on container start
+      command: "docker exec {{ rabbitmq_container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Generate new configuration and stop services
+      shell:
+        cmd: >
+          kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck &&
+          kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt {{ services_to_restart }} &&
+          kayobe kolla ansible run rabbitmq-reset-state
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      tags: rabbit-queue-migration
+
+    - name: Ensure that no queues exist
+      shell:
+        cmd: >
+          docker exec rabbitmq rabbitmqctl list_queues name --silent &&
+          docker exec rabbitmq rabbitmqctl list_exchanges name --silent
+          | grep -v '^$'
+          | (! grep -v 'amq.')
+        executable: /bin/bash
+      become: true
+      register: queues
+
+    - name: Redeploy services with quorum queues
+      shell:
+        cmd: >
+          kayobe kolla ansible run deploy -kt {{ services_to_restart }}
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      tags: rabbit-queue-migration
+
+    - name: Inspect RabbitMQ queues
+      shell:
+        cmd: "docker exec {{ rabbitmq_container_name }} rabbitmqctl list_queues type"
+      run_once: true
+      delegate_to: localhost
+      register: queues
+      become: true
+
+    - name: Assert that queues have been migrated
+      assert:
+        that: "{{ 'quorum' in queues.stdout }}"
+        fail_msg: Queue migration has failed. Run the migration manually.
+      run_once: true
+      delegate_to: localhost

releasenotes/notes/rabbit-migration-playbook-4721c2e37ac9d3c0.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Added a new playbook to automate RabbitMQ Quorum queue migrations.