Merge pull request #1052 from stackhpc/2024.1-2023.1-merge

2024.1: 2023.1 merge

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -31,7 +31,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
         working-directory: src/kayobe-config
 
       - name: Clone StackHPC Kayobe repository

.github/workflows/overcloud-host-image-promote.yml

@@ -47,7 +47,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
@@ -80,7 +80,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
 
       - name: Output Rocky Linux 9 image tag
         id: rocky_9_image_tag

.github/workflows/overcloud-host-image-upload.yml

@@ -38,7 +38,7 @@ on:
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: en1.large
+        default: en1.medium
       vm_network:
         description: Network for the all-in-one VM
         type: string
@@ -73,7 +73,7 @@ jobs:
   # NOTE: Runner needs unzip and nodejs packages.
   all-in-one:
     name: All in one
-    if: inputs.if
+    if: ${{ inputs.if && !cancelled() }}
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:
@@ -156,6 +156,7 @@ jobs:
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
           aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
+          aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -167,6 +168,7 @@ jobs:
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
           VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
+          VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
         run: terraform plan
@@ -181,13 +183,15 @@ jobs:
           for attempt in $(seq 5); do
               if terraform apply -auto-approve; then
                   echo "Created infrastructure on attempt $attempt"
-                  break
+                  exit 0
               fi
               echo "Failed to create infrastructure on attempt $attempt"
               sleep 10
               terraform destroy -auto-approve
               sleep 60
           done
+          echo "Failed to create infrastructure after $attempt attempts"
+          exit 1
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
           OS_CLOUD: ${{ inputs.OS_CLOUD }}

.github/workflows/stackhpc-all-in-one.yml

@@ -23,7 +23,7 @@ env:
 jobs:
   check-tags:
     name: Check container image tags
-    if: inputs.if
+    if: ${{ inputs.if && ! cancelled() }}
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:

.github/workflows/stackhpc-check-tags.yml

@@ -0,0 +1,77 @@
+---
+name: Clean up stale CI resources
+on:
+  schedule:
+    # Every 2 hours at quarter past
+    - cron: '15 0/2 * * *'
+
+jobs:
+  ci-cleanup:
+    name: Clean up stale CI resources
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+
+      - name: Install OpenStack client
+        run: |
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+
+      - name: Clean up aio instances over 3 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -3hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Clean up host image builder instances over 5 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -5hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

.github/workflows/stackhpc-ci-cleanup.yml

@@ -33,6 +33,12 @@ on:
         type: boolean
         required: false
         default: true
+      push-dirty:
+        description: Push scanned images that have vulnerabilities?
+        type: boolean
+        required: false
+        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
+        default: true
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -54,7 +60,7 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       # Generate a tag to apply to all built container images.
       # Without this, each kayobe * container image build command would use a different tag.
@@ -100,7 +106,15 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt update
-          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -118,6 +132,10 @@ jobs:
         run: |
           docker ps
 
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -132,6 +150,10 @@ jobs:
       - name: Install Docker Python SDK
         run: |
           sudo pip install docker
+      
+      - name: Get Kolla tag
+        id: write-kolla-tag
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
 
       - name: Configure localhost as a seed
         run: |
@@ -153,67 +175,124 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Build and push kolla overcloud images
+      - name: Create build logs output directory
+        run: mkdir image-build-logs 
+
+      - name: Build kolla overcloud images
+        id: build_overcloud_images
+        continue-on-error: true
         run: |
-          args="${{ github.event.inputs.regexes }}"
+          args="${{ inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-          KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"
-        if: github.event.inputs.overcloud == 'true'
+        if: inputs.overcloud
+
+      - name: Copy overcloud container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
+        if: inputs.overcloud
 
-      - name: Build and push kolla seed images
+      - name: Build kolla seed images
+        id: build_seed_images
+        continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-          KOLLA_TAG: "${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"
-        if: github.event.inputs.seed == 'true'
+        if: inputs.seed
+
+      - name: Copy seed container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
+        if: inputs.seed
 
       - name: Get built container images
-        run: |
-          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
-      - name: Upload container images artifact
+      - name: Scan built container images
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
+
+      - name: Move image scan logs to output artifact
+        run: mv image-scan-output image-build-logs/image-scan-output
+
+      - name: Fail if no images have passed scanning
+        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        if: ${{ !inputs.push-dirty }}
+
+      - name: Copy clean images to push-attempt-images list
+        run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
+        if: inputs.push
+
+      - name: Append dirty images to push list
+        run: |
+          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push && inputs.push-dirty }}
+
+      - name: Push images
+        run: |
+          touch image-build-logs/push-failed-images.txt
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
+
+          while read -r image; do
+            # Retries!
+            for i in {1..5}; do 
+              if docker push $image; then
+                echo "Pushed $image"
+                break
+              elif $i == 5; then
+                echo "Failed to push $image"
+                echo $image >> image-build-logs/push-failed-images.txt
+              else
+                echo "Failed on retry $i"
+                sleep 5
+              fi;
+            done
+          done < image-build-logs/push-attempt-images.txt
+        shell: bash
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.push
+
+      - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }} container images
-          path: ${{ matrix.distro }}-container-images
+          name: ${{ matrix.distro }}-logs
+          path: image-build-logs
           retention-days: 7
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed to build
+        run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
+        if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
+
+      - name: Fail when images failed to push
+        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed scanning
+        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+        if: ${{ !inputs.push-dirty && !cancelled() }}
 
-  sync-container-repositories:
-    name: Trigger container image repository sync
-    needs:
-      - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
-    runs-on: ubuntu-latest
-    permissions: {}
-    steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync
         run: |
           filter='${{ inputs.regexes }}'
-          if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
+          if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
             filter="$filter bifrost"
           fi
           gh workflow run \
@@ -224,7 +303,9 @@ jobs:
           -f sync-old-images=false
         env:
           GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
 
       - name: Display link to container image repository sync workflows
         run: |
           echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}