CI: Add tags to aio VMs, periodically clean up stale aio instances (yoga)

.github/workflows/overcloud-host-image-build.yml

@@ -139,7 +139,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
 
       - name: Build a CentOS Stream 8 overcloud host image
         id: build_centos_stream_8

.github/workflows/overcloud-host-image-upload.yml

@@ -92,7 +92,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
 
       - name: Output CentOS Stream 8 image tag
         id: centos_8_stream_image_tag

.github/workflows/stackhpc-all-in-one.yml

@@ -134,6 +134,7 @@ jobs:
           aio_vm_flavor = "${{ env.VM_FLAVOR }}"
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -144,6 +145,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
+          VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
         run: terraform plan

.github/workflows/stackhpc-ci-cleanup.yml

@@ -0,0 +1,57 @@
+---
+name: Clean up stale CI resources
+on:
+  schedule:
+    # Every 2 hours at quarter past
+    - cron: '15 0/2 * * *'
+
+jobs:
+  ci-cleanup:
+    name: Clean up stale CI resources
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+
+      - name: Install OpenStack client
+        run: |
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+
+      - name: Clean up aio instances over 3 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -3hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

etc/kayobe/environments/ci-builder/inventory/hosts

@@ -1,3 +1,3 @@
 # A 'seed' host used for building images.
 [seed]
-builder
+localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3

terraform/aio/vm.tf

@@ -38,6 +38,11 @@ variable "aio_vm_volume_size" {
   default = 35
 }
 
+variable "aio_vm_tags" {
+  type = list(string)
+  default = []
+}
+
 locals {
   image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.aio_vm_image)) > 0
 }
@@ -69,6 +74,8 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
     destination_type      = "volume"
     delete_on_termination = true
   }
+
+  tags = var.aio_vm_tags
 }
 
 # Wait for the instance to be accessible via SSH before progressing.