CI: Collect diagnostic information at end of aio jobs

Use the new diagnostics.yml playbook.

(cherry picked from commit 834110b)

Author: markgoddard

.github/workflows/stackhpc-all-in-one.yml

@@ -156,6 +156,7 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
       - name: Terraform Apply
+        id: tf_apply
         run: |
           for attempt in $(seq 5); do
               if terraform apply -auto-approve; then
@@ -290,6 +291,7 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
       - name: Tempest tests
+        id: tempest
         run: |
           mkdir -p tempest-artifacts
           docker run -t --rm \
@@ -301,11 +303,28 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Collect diagnostic information
+        id: diagnostics
+        run: |
+          mkdir -p diagnostics
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -v $(pwd)/diagnostics:/stack/diagnostics \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }}
+
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
-          path: tempest-artifacts/*
+          name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
+          path: |
+            diagnostics/
+            tempest-artifacts/
+        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
       - name: Fail if any Tempest tests failed
         run: |

etc/kayobe/ansible/diagnostics.yml

@@ -32,17 +32,30 @@
             LOG_DIR: "{{ diagnostics_tmpdir.path }}"
             CONFIG_DIR: "{{ kayobe_config_path }}/../.."
 
+        # FIXME: The synchronize module fails on Yoga, due to not templating
+        # the SSH user.
+        # - name: Download diagnostic logs to localhost
+        #   ansible.posix.synchronize:
+        #     src: "{{ diagnostics_tmpdir.path }}/"
+        #     dest: "{{ diagnostics_path_local }}/{{ inventory_hostname }}"
+        #     mode: pull
+        #     archive: no
+        #     recursive: true
+        #     copy_links: true
+        #     verify_host: true
+        #     # For jump host
+        #     #use_ssh_args: true
+        #   failed_when: false
+
         - name: Download diagnostic logs to localhost
-          ansible.posix.synchronize:
-            src: "{{ diagnostics_tmpdir.path }}/"
-            dest: "{{ diagnostics_path_local }}/{{ inventory_hostname }}"
-            mode: pull
-            archive: no
-            recursive: true
-            copy_links: true
-            verify_host: true
-            # For jump host
-            use_ssh_args: true
+          command:
+            cmd: >-
+              rsync
+              --delay-updates -F --compress --copy-links --recursive
+              --rsh='/usr/bin/ssh -S none'
+              '{{ ansible_user }}@{{ ansible_host }}:{{ diagnostics_tmpdir.path }}/'
+              '{{ diagnostics_path_local }}/{{ inventory_hostname }}'
+          changed_when: true
       always:
         - name: Clean up temporary directory
           ansible.builtin.file: