Merge zed antelope

.github/workflows/overcloud-host-image-build.yml

@@ -38,7 +38,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       # Generate a tag to apply to all built overcloud host images.
       - name: Generate overcloud host image tag

.github/workflows/overcloud-host-image-promote.yml

@@ -31,7 +31,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
         working-directory: src/kayobe-config
 
       - name: Clone StackHPC Kayobe repository

.github/workflows/overcloud-host-image-upload.yml

@@ -47,7 +47,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4

.github/workflows/stackhpc-ci-cleanup.yml

@@ -30,7 +30,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       - name: Install OpenStack client
         run: |

.github/workflows/stackhpc-container-image-build.yml

@@ -33,6 +33,12 @@ on:
         type: boolean
         required: false
         default: true
+      push-dirty:
+        description: Push scanned images that have vulnerabilities?
+        type: boolean
+        required: false
+        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
+        default: true
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -54,7 +60,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       # Generate a tag to apply to all built container images.
       # Without this, each kayobe * container image build command would use a different tag.
@@ -100,7 +106,15 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt update
-          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -118,6 +132,10 @@ jobs:
         run: |
           docker ps
 
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -132,6 +150,10 @@ jobs:
       - name: Install Docker Python SDK
         run: |
           sudo pip install docker
+      
+      - name: Get Kolla tag
+        id: write-kolla-tag
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
 
       - name: Configure localhost as a seed
         run: |
@@ -153,67 +175,124 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Build and push kolla overcloud images
+      - name: Create build logs output directory
+        run: mkdir image-build-logs 
+
+      - name: Build kolla overcloud images
+        id: build_overcloud_images
+        continue-on-error: true
         run: |
-          args="${{ github.event.inputs.regexes }}"
+          args="${{ inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-          KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"
-        if: github.event.inputs.overcloud == 'true'
+        if: inputs.overcloud
+
+      - name: Copy overcloud container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
+        if: inputs.overcloud
 
-      - name: Build and push kolla seed images
+      - name: Build kolla seed images
+        id: build_seed_images
+        continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-          KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"
-        if: github.event.inputs.seed == 'true'
+        if: inputs.seed
+
+      - name: Copy seed container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
+        if: inputs.seed
 
       - name: Get built container images
-        run: |
-          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
-      - name: Upload container images artifact
+      - name: Scan built container images
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
+
+      - name: Move image scan logs to output artifact
+        run: mv image-scan-output image-build-logs/image-scan-output
+
+      - name: Fail if no images have passed scanning
+        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        if: ${{ !inputs.push-dirty }}
+
+      - name: Copy clean images to push-attempt-images list
+        run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
+        if: inputs.push
+
+      - name: Append dirty images to push list
+        run: |
+          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push && inputs.push-dirty }}
+
+      - name: Push images
+        run: |
+          touch image-build-logs/push-failed-images.txt
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
+
+          while read -r image; do
+            # Retries!
+            for i in {1..5}; do 
+              if docker push $image; then
+                echo "Pushed $image"
+                break
+              elif $i == 5; then
+                echo "Failed to push $image"
+                echo $image >> image-build-logs/push-failed-images.txt
+              else
+                echo "Failed on retry $i"
+                sleep 5
+              fi;
+            done
+          done < image-build-logs/push-attempt-images.txt
+        shell: bash
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.push
+
+      - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }} container images
-          path: ${{ matrix.distro }}-container-images
+          name: ${{ matrix.distro }}-logs
+          path: image-build-logs
           retention-days: 7
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed to build
+        run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
+        if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
+
+      - name: Fail when images failed to push
+        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed scanning
+        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+        if: ${{ !inputs.push-dirty && !cancelled() }}
 
-  sync-container-repositories:
-    name: Trigger container image repository sync
-    needs:
-      - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
-    runs-on: ubuntu-latest
-    permissions: {}
-    steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync
         run: |
           filter='${{ inputs.regexes }}'
-          if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
+          if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
             filter="$filter bifrost"
           fi
           gh workflow run \
@@ -224,7 +303,9 @@ jobs:
           -f sync-old-images=false
         env:
           GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
 
       - name: Display link to container image repository sync workflows
         run: |
           echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}

doc/source/configuration/vault.rst

@@ -296,7 +296,9 @@ Configure Barbican
       [vault_plugin]
       vault_url = https://{{ kolla_internal_vip_address }}:8200
       use_ssl = True
-      ssl_ca_crt_file = {% raw %}{{ openstack_cacert }}{% endraw %}
+      {% raw %}
+      ssl_ca_crt_file = {{ openstack_cacert }}
+      {% endraw %}
       approle_role_id = {{ secrets_barbican_approle_role_id }}
       approle_secret_id = {{ secrets_barbican_approle_secret_id }}
       kv_mountpoint = barbican

doc/source/contributor/environments/aufn-ceph.rst

@@ -14,7 +14,7 @@ This environment creates a Universe-from-nothing_-style deployment of Kayobe con
 Prerequisites
 =============
 
-* a baremetal node with at least 64GB of RAM running CentOS Stream 8 (or Ubuntu)
+* a baremetal node with at least 64GB of RAM running Rocky Linux 9 or Ubuntu Jammy.
 
 * access to the test pulp server on SMS lab
 

doc/source/contributor/environments/ci-builder.rst

@@ -25,7 +25,7 @@ Access the host via SSH.
 
 Install package dependencies.
 
-On CentOS:
+On Rocky Linux:
 
 .. parsed-literal::
 

doc/source/operations/tempest.rst

@@ -65,7 +65,7 @@ To install Docker on Ubuntu:
     sudo apt-get update
     sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 
-Installing Docker on CentOS/Rocky:
+Installing Docker on Rocky:
 
 .. code-block:: bash
 
@@ -99,9 +99,9 @@ Build a Kayobe automation image:
 
     git submodule init
     git submodule update
-    # If running on Ubuntu, the fact cache can confuse Kayobe in the CentOS-based container
+    # If running on Ubuntu, the fact cache can confuse Kayobe in the Rocky-based container
     mv etc/kayobe/facts{,-old}
-    sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest .
+    sudo DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=rockylinux:9 --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest .
 
 Configuration
 =============
@@ -277,7 +277,10 @@ command from the base of the ``kayobe-config`` directory:
 
 .. code-block:: bash
 
-    sudo -E docker run --detach -it --rm --network host -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
+    sudo -E docker run --name kayobe-automation --detach -it --rm --network host \
+    -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
+    -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \
+    /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
 
 By default, ``no_log`` is set to stop credentials from leaking. This can be
 disabled by adding ``-e rally_no_sensitive_log=false`` to the end.

doc/source/operations/upgrading.rst

@@ -199,14 +199,14 @@ Known issues
 
 * Docker log-opts are currently not configured in Antelope. You will see these
   being removed when running a host configure in check+diff mode. See bug for
-  details (fix in progress):
+  details (fix released):
   https://bugs.launchpad.net/ansible-collection-kolla/+bug/2040105
 
 * /etc/hosts are not templated correctly when running a host configure with
   ``--limit``. To work around this, run your host configures with
   ``--skip-tags etc-hosts``. If you do need to change ``/etc/hosts``, for
   example with any newly-added hosts, run a full host configure afterward with
-  ``--tags etc-hosts``. See bug for details (fix in progress):
+  ``--tags etc-hosts``. See bug for details (fix released):
   https://bugs.launchpad.net/kayobe/+bug/2051714
 
 Security baseline

etc/kayobe/ansible/docker-registry-login.yml

@@ -0,0 +1,11 @@
+---
+- name: Login to docker registry
+  gather_facts: false
+  hosts: container-image-builders
+  tasks:
+    - name: Login to docker registry
+      docker_login:
+        registry_url: "{{ kolla_docker_registry or omit }}"
+        username: "{{ kolla_docker_registry_username }}"
+        password: "{{ kolla_docker_registry_password }}"
+        reauthorize: yes

etc/kayobe/ansible/ovn-fix-chassis-priorities.yml

@@ -21,22 +21,25 @@
 - name: Find OVN DB DB Leader
   hosts: "{{ ovn_nb_db_group | default('controllers') }}"
   tasks:
-    - name: Find the OVN NB DB leader
-      ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection
-      changed_when: false
-      failed_when: false
-      register: ovn_check_result
-      check_mode: false
+    - name: Find OVN DB Leader
+      when: kolla_enable_ovn | bool
+      block:
+        - name: Find the OVN NB DB leader
+          ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection
+          changed_when: false
+          failed_when: false
+          register: ovn_check_result
+          check_mode: false
 
-    - name: Group hosts by leader/follower role
-      ansible.builtin.group_by:
-        key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}"
-      changed_when: false
+        - name: Group hosts by leader/follower role
+          ansible.builtin.group_by:
+            key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}"
+          changed_when: false
 
-    - name: Assert one leader exists
-      ansible.builtin.assert:
-        that:
-          - groups['ovn_nb_leader'] | default([]) | length == 1
+        - name: Assert one leader exists
+          ansible.builtin.assert:
+            that:
+              - groups['ovn_nb_leader'] | default([]) | length == 1
 
 - name: Fix OVN chassis priorities
   hosts: ovn_nb_leader

etc/kayobe/ansible/ubuntu-upgrade.yml

@@ -104,7 +104,3 @@
         that:
           - ansible_facts.distribution_major_version == '22'
           - ansible_facts.distribution_release == 'jammy'
-
-- name: Run the OVN chassis priority fix playbook
-  import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml"
-  when: kolla_enable_ovn