Merge pull request #661 from stackhpc/wallaby-ovn-fix-chassis-priorities-playbook

Add a custom playbook to fix OVN chassis priorities

Author: markgoddard

etc/kayobe/ansible/ovn-fix-chassis-priorities.yml

@@ -0,0 +1,69 @@
+---
+# Sometimes, typically after restarting OVN services, the priorities of entries
+# in the ha_chassis and gateway_chassis tables in the OVN northbound database
+# can become misaligned. This results in broken routing for external (bare
+# metal/SR-IOV) ports.
+
+# This playbook can be used to fix the issue by realigning the priorities of
+# the table entries. It does so by assigning the highest priority to the
+# "first" (sorted alphabetically) OVN NB DB host. This results in all gateways
+# being scheduled to a single host, but is less complicated than trying to
+# balance them (and it's also not clear to me how to map between individual
+# ha_chassis and gateway_chassis entries).
+
+# The playbook can be run as follows:
+# kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ovn-fix-chassis-priorities.yml
+
+# If the 'controllers' group does not align with the group used to deploy the
+# OVN NB DB, this can be overridden by passing the following:
+# '-e ovn_nb_db_group=some_other_group'
+
+- name: Find OVN DB DB Leader
+  hosts: "{{ ovn_nb_db_group | default('controllers') }}"
+  tasks:
+    - name: Find the OVN NB DB leader
+      command: docker exec -it ovn_nb_db ovn-nbctl get-connection
+      changed_when: false
+      failed_when: false
+      register: ovn_check_result
+      check_mode: no
+
+    - name: Group hosts by leader/follower role
+      group_by:
+        key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}"
+      changed_when: false
+
+    - name: Assert one leader exists
+      assert:
+        that:
+          - groups['ovn_nb_leader'] | default([]) | length == 1
+
+- name: Fix OVN chassis priorities
+  hosts: ovn_nb_leader
+  vars:
+    ovn_nb_db_group: controllers
+    ovn_nb_db_hosts_sorted: "{{ query('inventory_hostnames', ovn_nb_db_group) | sort | list }}"
+    ha_chassis_max_priority: 32767
+    gateway_chassis_max_priority: "{{ ovn_nb_db_hosts_sorted | length }}"
+  tasks:
+    - name: Fix ha_chassis priorities
+      command: >-
+        docker exec -it ovn_nb_db
+        bash -c '
+        ovn-nbctl find ha_chassis chassis_name={{ item }} |
+        awk '\''$1 == "_uuid" { print $3 }'\'' |
+        while read uuid; do ovn-nbctl set ha_chassis $uuid priority={{ priority }}; done'
+      loop: "{{ ovn_nb_db_hosts_sorted }}"
+      vars:
+        priority: "{{ ha_chassis_max_priority | int - ovn_nb_db_hosts_sorted.index(item) }}"
+
+    - name: Fix gateway_chassis priorities
+      command: >-
+        docker exec -it ovn_nb_db
+        bash -c '
+        ovn-nbctl find gateway_chassis chassis_name={{ item }} |
+        awk '\''$1 == "_uuid" { print $3 }'\'' |
+        while read uuid; do ovn-nbctl set gateway_chassis $uuid priority={{ priority }}; done'
+      loop: "{{ ovn_nb_db_hosts_sorted }}"
+      vars:
+        priority: "{{ gateway_chassis_max_priority | int - ovn_nb_db_hosts_sorted.index(item) }}"