CI: overcloud host image matrix build

This change refactors the overcloud host image build workflow to make
the image tag consistent across different images built from the same
workflow run.

Author: Alex-Welsh

.github/workflows/overcloud-host-image-build.yml

@@ -27,26 +27,124 @@ on:
 env:
   ANSIBLE_FORCE_COLOR: True
 jobs:
-  overcloud-host-image-build:
-    name: Build overcloud host images
+  generate-tag:
+    name: Generate overcloud host image tag
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    permissions: {}
+    outputs:
+      host_image_tag: ${{ steps.host_image_tag.outputs.host_image_tag }}
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          path: src/kayobe-config
+      - name: Checkout
+        uses: actions/checkout@v3
 
       - name: Determine OpenStack release
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
           echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
 
+      # Generate a tag to apply to all built overcloud host images.
+      - name: Generate overcloud host image tag
+        id: host_image_tag
+        run: |
+          echo "host_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
+      # Dynamically define job matrix.
+      # We need a separate matrix entry for each distribution, when the relevant input is true.
+      # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
+      # The resultant matrix will have the following structure, filtered by inputs:
+      # matrix:
+      #   distro: [centos, rocky8, rocky9, ubuntu-focal, ubuntu-jammy]
+      #   include:
+      #     - distro: centos
+      #       os_distribution: centos
+      #       os_release: 8-stream
+      #     - distro: rocky8
+      #       os_distribution: rocky
+      #       os_release: 8
+      #     - distro: rocky9
+      #       os_distribution: rocky
+      #       os_release: 9
+      #     - distro: ubuntu-focal
+      #       os_distribution: ubuntu
+      #       os_release: focal
+      #     - distro: ubuntu-jammy
+      #       os_distribution: ubuntu
+      #       os_release: jammy
+      - name: Generate build matrix
+        id: set-matrix
+        run: |
+          comma=""
+          echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          if [[ ${{ inputs.centos }} == 'true' ]]; then
+            echo -n "$comma\"centos\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.rocky8 }} == 'true' ]]; then
+            echo -n "$comma\"rocky8\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.rocky9 }} == 'true' ]]; then
+            echo -n "$comma\"rocky9\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.ubuntu-focal }} == 'true' ]]; then
+            echo -n "$comma\"ubuntu-focal\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then
+            echo -n "$comma\"ubuntu-jammy\"" >> $GITHUB_OUTPUT
+          fi
+          echo -n "]" >> $GITHUB_OUTPUT
+          echo -n ", \"include\": [" >> $GITHUB_OUTPUT
+          comma=""
+          if [[ ${{ inputs.centos }} == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"centos\", \"os_distribution\": \"centos\", \"os_release\": \"8-stream\"}" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.rocky8 }} == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"rocky8\", \"os_distribution\": \"rocky\", \"os_release\": \"8\"}" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.rocky9 }} == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"rocky9\", \"os_distribution\": \"rocky\", \"os_release\": \"9\"}" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.ubuntu-focal }} == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"ubuntu-focal\", \"os_distribution\": \"ubuntu\", \"os_release\": \"focal\"}" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"ubuntu-jammy\", \"os_distribution\": \"ubuntu\", \"os_release\": \"jammy\"}" >> $GITHUB_OUTPUT
+          fi
+          echo "]}" >> $GITHUB_OUTPUT
+
+      - name: Display container image tag
+        run: |
+          echo "${{ steps.host_image_tag.outputs.host_image_tag }}"
+
+  overcloud-host-image-build:
+    name: Build overcloud host images
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    strategy:
+      matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
+      fail-fast: false
+    needs:
+      - generate-tag
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: src/kayobe-config
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v3
         with:
           repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
           path: src/kayobe
 
       # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
@@ -69,6 +167,14 @@ jobs:
           sudo ip l set dummy1 up
           sudo ip l set dummy1 master breth1
 
+      # FIXME: Without this workaround we see the following issue after the runner is power cycled:
+      # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] ***
+      #   Unable to start service network: Job for network.service failed because the control process exited with error code.
+      #   See \"systemctl status network.service\" and \"journalctl -xe\" for details.
+      - name: Kill dhclient (workaround)
+        run: |
+          (sudo killall dhclient || true) && sudo systemctl restart network
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -98,148 +204,26 @@ jobs:
             sudo docker volume create bifrost_httpboot
           fi
 
-      - name: Clean any previous build artifact
-        run: |
-          rm -f /tmp/updated_images.txt
-
-      - name: Build a CentOS Stream 8 overcloud host image
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild \
-          -e os_distribution=centos \
-          -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.centos
-
-      - name: Upload CentOS Stream 8 overcloud host image artifact
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
-          -e os_distribution='centos'
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.centos
-
-      - name: Build a Rocky Linux 8 overcloud host image
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild \
-          -e os_distribution=rocky \
-          -e os_release='8' \
-          -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky8
-
-      - name: Upload Rocky Linux 8 overcloud host image artifact
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
-          -e os_distribution='rocky'
-          -e os_release='8' \
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky8
-
-      - name: Build a Rocky Linux 9 overcloud host image
+      - name: Build an overcloud host image
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud host image build --force-rebuild \
-          -e os_distribution=rocky \
-          -e os_release='9' \
-          -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-9
+          -e os_distribution=${{ matrix.os_distribution }} \
+          -e os_release="${{ matrix.os_release }}" \
+          -e stackhpc_overcloud_dib_name=overcloud-${{ matrix.os_distribution }}-"${{ matrix.os_release }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky9
 
-      - name: Upload Rocky Linux 9 overcloud host image artifact
+      - name: Upload overcloud host image artifact
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
           src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path='/opt/kayobe/images/overcloud-rocky-linux-9' \
-          -e os_distribution='rocky' \
-          -e os_release='9'
+          -e image_path=/opt/kayobe/images/overcloud-${{ matrix.os_distribution }}-"${{ matrix.os_release }}" \
+          -e host_image_tag=${{ needs.generate-tag.outputs.host_image_tag }} \
+          -e os_distribution=${{ matrix.os_distribution }} \
+          -e os_release="${{ matrix.os_release }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky9
-
-      # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
-      # Do this via config?
-      - name: Enable EPEL repository
-        run: |
-          sudo dnf config-manager --set-enabled epel
-
-      - name: Build a Ubuntu Focal 20.04 overcloud host image
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild \
-          -e os_distribution=ubuntu \
-          -e os_release=focal \
-          -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-focal
-
-      - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
-          -e os_distribution='ubuntu' \
-          -e os_release=focal
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-focal
-
-      - name: Build a Ubuntu Jammy 22.04 overcloud host image
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild \
-          -e os_distribution=ubuntu \
-          -e os_release=jammy \
-          -e stackhpc_overcloud_dib_name=overcloud-ubuntu-jammy
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy
-
-      - name: Upload Ubuntu Jammy 22.04 overcloud host image artifact
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path='/opt/kayobe/images/overcloud-ubuntu-jammy' \
-          -e os_distribution='ubuntu' \
-          -e os_release=jammy
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy
-
-      - name: Upload updated images artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: Updated images list
-          path: /tmp/updated_images.txt
-          retention-days: 7
-        if: always()
-
-      - name: Clean up old images
-        run: |
-          sudo rm -rf /opt/kayobe/images/
-        if: always()

etc/kayobe/ansible/pulp-host-image-upload.yml

@@ -8,6 +8,10 @@
     repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
     base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
+    - name: Print image tag
+      debug:
+        msg: "Image tag: {{ host_image_tag }}"
+
     - name: Get filename
       find:
         paths: "{{ image_path }}"
@@ -81,29 +85,26 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ repository_name }}_{{ ansible_facts.date_time.iso8601_basic_short }}"
-        base_path: "{{ base_path }}/{{ ansible_facts.date_time.iso8601_basic_short }}"
+        name: "{{ repository_name }}_{{ host_image_tag }}"
+        base_path: "{{ base_path }}/{{ host_image_tag }}"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
       when: latest_distribution_details.changed
 
-    - name: Update new images file with versioned path
-      lineinfile:
-        path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
-        {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}"
-        create: true
+    - name: Print versioned path
+      debug:
+        msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
-    - name: Update new images file with latest path
-      lineinfile:
-        path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+    - name: Print latest path
+      debug:
+        msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print version tag
       debug:
-        msg: "New tag: {{ ansible_facts.date_time.iso8601_basic_short }}"
+        msg: "New tag: {{ host_image_tag }}"
       when: latest_distribution_details.changed

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -103,3 +103,7 @@ stackhpc_docker_registry_password: !vault |
   34386133383366326639353432386235336132663839333337323739633434613934346462363031
   3265323831663964360a643962346231386462323236373963633066393736323234303833363535
   3664
+
+# Username and password of the overcloud host image repository.
+stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"

releasenotes/notes/host-image-matrix-build-5c8a7b01186ccdcd.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    Host images are now built in a matrix. Host images built in the same
+    workflow will have the same tag.