WIP: CI: Add upgrade tests

markgoddard · markgoddard · commit 339824a7c658 · 2024-01-16T12:51:34.000Z
diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
@@ -76,10 +76,15 @@ jobs:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
+      # NOTE(upgrade): Reference the PREVIOUS release here.
+      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-yoga
     steps:
+      # Checkout previous release
       - uses: actions/checkout@v3
         with:
-           submodules: true
+        # NOTE(upgrade): Reference the PREVIOUS release branch here.
+          ref: stackhpc/yoga
+          submodules: true
 
       - name: Output image tag
         id: image_tag
@@ -125,6 +130,7 @@ jobs:
           aio_vm_flavor = "${{ env.VM_FLAVOR }}"
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -135,6 +141,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
+          VM_VOLUME_SIZE: 45
 
       - name: Terraform Plan
         run: terraform plan
@@ -206,8 +213,13 @@ jobs:
           cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
+      # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
+      - name: Pull previous Kayobe image
+        run: |
+          sudo docker image pull $PREVIOUS_KAYOBE_IMAGE
+
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
-      - name: Pull latest Kayobe image
+      - name: Pull current Kayobe image
         run: |
           sudo docker image pull $KAYOBE_IMAGE
 
@@ -216,7 +228,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            $PREVIOUS_KAYOBE_IMAGE \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -226,7 +238,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            $PREVIOUS_KAYOBE_IMAGE \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -236,7 +248,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            $PREVIOUS_KAYOBE_IMAGE \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -246,11 +258,53 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            $PREVIOUS_KAYOBE_IMAGE \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Stash config changes
+        run: git stash
+
+      # Now begin upgrade
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          clean: false
+
+      - name: Pop stashed config changes
+        run: git stash pop
+
+      - name: Host upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      - name: Host configure
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      - name: Service upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
@@ -76,56 +76,56 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
-  all-in-one-ubuntu-jammy-ovs:
-    name: aio (Ubuntu Jammy OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-ubuntu-jammy-ovn:
-    name: aio (Ubuntu Jammy OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-rocky-9-ovs:
-    name: aio (Rocky 9 OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+#  all-in-one-ubuntu-jammy-ovs:
+#    name: aio (Ubuntu Jammy OVS)
+#    needs:
+#      - check-changes
+#      - build-kayobe-image
+#    uses: ./.github/workflows/stackhpc-all-in-one.yml
+#    with:
+#      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+#      os_distribution: ubuntu
+#      os_release: jammy
+#      ssh_username: ubuntu
+#      neutron_plugin: ovs
+#      OS_CLOUD: sms-lab-release
+#      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+#    secrets: inherit
+#    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+#
+#  all-in-one-ubuntu-jammy-ovn:
+#    name: aio (Ubuntu Jammy OVN)
+#    needs:
+#      - check-changes
+#      - build-kayobe-image
+#    uses: ./.github/workflows/stackhpc-all-in-one.yml
+#    with:
+#      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+#      os_distribution: ubuntu
+#      os_release: jammy
+#      ssh_username: ubuntu
+#      neutron_plugin: ovn
+#      OS_CLOUD: sms-lab-release
+#      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+#    secrets: inherit
+#    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+#
+#  all-in-one-rocky-9-ovs:
+#    name: aio (Rocky 9 OVS)
+#    needs:
+#      - check-changes
+#      - build-kayobe-image
+#    uses: ./.github/workflows/stackhpc-all-in-one.yml
+#    with:
+#      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+#      os_distribution: rocky
+#      os_release: "9"
+#      ssh_username: cloud-user
+#      neutron_plugin: ovs
+#      OS_CLOUD: sms-lab-release
+#      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+#    secrets: inherit
+#    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-rocky-9-ovn:
     name: aio (Rocky 9 OVN)
