2023.1: Update to Rocky 9.3 based kolla images

.github/workflows/stackhpc-all-in-one.yml

@@ -55,6 +55,10 @@ on:
         description: Whether to run the workflow (workaround for required status checks issue)
         type: boolean
         default: true
+      upgrade:
+        description: Whether to perform an upgrade
+        type: boolean
+        default: false
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -76,10 +80,27 @@ jobs:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
+      # NOTE(upgrade): Reference the PREVIOUS release here.
+      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-zed
+      # NOTE(upgrade): Reference the PREVIOUS release branch here.
+      PREVIOUS_BRANCH: stackhpc/zed
     steps:
-      - uses: actions/checkout@v3
+      # If testing upgrade, checkout previous release, otherwise checkout current branch
+      - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
+        uses: actions/checkout@v4
         with:
-           submodules: true
+          ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }}
+          submodules: true
+
+      - name: Output Kayobe image
+        id: kayobe_image
+        run: |
+          if ${{ inputs.upgrade }}; then
+            kayobe_image=$PREVIOUS_KAYOBE_IMAGE
+          else
+            kayobe_image=$KAYOBE_IMAGE
+          fi
+          echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT
 
       - name: Output image tag
         id: image_tag
@@ -125,6 +146,7 @@ jobs:
           aio_vm_flavor = "${{ env.VM_FLAVOR }}"
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -135,6 +157,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
 
       - name: Terraform Plan
         run: terraform plan
@@ -206,8 +229,14 @@ jobs:
           cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
+      # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
+      - name: Pull previous Kayobe image
+        run: |
+          sudo docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
+        if: inputs.upgrade
+
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
-      - name: Pull latest Kayobe image
+      - name: Pull current Kayobe image
         run: |
           sudo docker image pull $KAYOBE_IMAGE
 
@@ -216,7 +245,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -226,7 +255,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -236,7 +265,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -246,11 +275,62 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      # If testing upgrade, checkout the current release branch
+      # Stash changes to tracked files, and set clean=false to avoid removing untracked files.
+      - name: Stash config changes
+        run: git stash
+        if: inputs.upgrade
+
+      - name: Checkout current release config
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          clean: false
+        if: inputs.upgrade
+
+      - name: Pop stashed config changes
+        run: git stash pop
+        if: inputs.upgrade
+
+      # Now begin upgrade
+      - name: Host upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
+      - name: Host configure
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
+      - name: Service upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
@@ -266,7 +346,7 @@ jobs:
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
+          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' }}
           path: tempest-artifacts/*
 
       - name: Fail if any Tempest tests failed

.github/workflows/stackhpc-check-tags.yml

@@ -0,0 +1,48 @@
+---
+# This workflow queries the Test Pulp server to check that all image tags
+# specified in kolla_image_tags are present.
+
+name: Check container image tags
+on:
+  workflow_call:
+    inputs:
+      kayobe_image:
+        description: Kayobe container image
+        type: string
+        required: true
+    secrets:
+      KAYOBE_VAULT_PASSWORD:
+        required: true
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+jobs:
+  check-tags:
+    name: Check container image tags
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    permissions: {}
+    env:
+      KAYOBE_ENVIRONMENT: ci-aio
+      KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+      KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
+      - name: Pull latest Kayobe image
+        run: |
+          sudo docker image pull $KAYOBE_IMAGE
+
+      - name: Check container image tags
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
+            '$KAYOBE_CONFIG_PATH/ansible/check-tags.yml'
+        #env:
+          #KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

.github/workflows/stackhpc-promote.yml

@@ -1,13 +1,13 @@
 ---
-name: Promote package repositories
+name: Promote Pulp repositories
 on:
   push:
     branches:
       # NOTE(upgrade): Reference only the current release branch here.
       - stackhpc/2023.1
 jobs:
   promote:
-    name: Trigger package repository promotion
+    name: Trigger Pulp promotion workflows
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-latest
     permissions: {}
@@ -27,3 +27,19 @@ jobs:
       - name: Display link to package repository promotion workflows
         run: |
           echo "::notice Package repository promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-promote.yml"
+
+      # NOTE(mgoddard): Trigger another CI workflow in the
+      # stackhpc-release-train repository.
+      - name: Trigger container image promotion
+        run: |
+          gh workflow run \
+          container-promote.yml \
+          --repo stackhpc/stackhpc-release-train \
+          --ref main \
+          -f kayobe_config_branch=${{ github.ref_name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+
+      - name: Display link to container image promotion workflows
+        run: |
+          echo "::notice Container image promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-promote.yml"

.github/workflows/stackhpc-pull-request.yml

@@ -76,6 +76,16 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
+  check-tags:
+    name: Check container image tags
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-check-tags.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
   all-in-one-ubuntu-jammy-ovs:
     name: aio (Ubuntu Jammy OVS)
     needs:
@@ -143,3 +153,41 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
+
+  all-in-one-upgrade-ubuntu-jammy-ovs:
+    name: aio upgrade (Ubuntu Jammy OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovs
+      OS_CLOUD: sms-lab-release
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-upgrade-rocky-9-ovn:
+    name: aio upgrade (Rocky 9 OVN)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovn
+      OS_CLOUD: sms-lab-release
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

etc/kayobe/ansible/check-tags.yml

@@ -0,0 +1,39 @@
+---
+# This playbook queries the Pulp server to check that all image tags specified
+# in kolla_image_tags are present.
+
+- name: Check whether tags exist in Pulp container registry
+  hosts: localhost
+  tasks:
+    - name: Query images and tags
+      command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py list-tags
+      register: kolla_images_result
+      changed_when: false
+
+    - name: Set a fact about images and tags
+      set_fact:
+        kolla_images: "{{ kolla_images_result.stdout | from_yaml }}"
+
+    - name: Set a fact about the Pulp URL
+      set_fact:
+        pulp_url: "{{ stackhpc_repo_mirror_url }}"
+
+    # Use state=read and allow_missing=false to check for missing tags in test pulp.
+    - import_role:
+        name: stackhpc.pulp.pulp_container_content
+      vars:
+        pulp_container_content: >-
+          {%- set contents = [] -%}
+          {%- for image, tags in kolla_images.items() -%}
+          {%- set repository = kolla_docker_namespace ~ "/" ~ image -%}
+          {%- set content = {
+            "allow_missing": False,
+            "repository": repository,
+            "state": "read",
+            "tags": tags,
+          } -%}
+          {%- set _ = contents.append(content) -%}
+          {%- endfor -%}
+          {{ contents }}

etc/kayobe/ansible/requirements.yml

@@ -7,7 +7,7 @@ collections:
   - name: pulp.squeezer
     version: 0.0.13
   - name: stackhpc.pulp
-    version: 0.5.2
+    version: 0.5.4
   - name: stackhpc.hashicorp
     version: 2.4.0
   - name: stackhpc.kayobe_workflows

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -62,3 +62,9 @@ stackhpc_docker_registry_password: !vault |
   38333133393730633666613965653364316162353337313330346164303631313731646461363461
   3963323635373866630a633533376339363734626664333765313665623662613764363038383735
   38646138376438643533376161376634653439386230353365316239613430363338
+
+# Override Pulp credentials to allow querying container image tags in the
+# check-tags.yml custom playbook.
+pulp_url: "{{ stackhpc_repo_mirror_url }}"
+pulp_username: "{{ stackhpc_docker_registry_username }}"
+pulp_password: "{{ stackhpc_docker_registry_password }}"