Merge pull request #951 from stackhpc/2023.1-zed-merge

2023.1: zed merge

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -118,7 +118,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
 
       - name: Build a Rocky Linux 9 overcloud host image
         id: build_rocky_9

.github/workflows/overcloud-host-image-upload.yml

@@ -80,7 +80,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
 
       - name: Output Rocky Linux 9 image tag
         id: rocky_9_image_tag

.github/workflows/stackhpc-all-in-one.yml

@@ -38,7 +38,7 @@ on:
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: en1.large
+        default: en1.medium
       vm_network:
         description: Network for the all-in-one VM
         type: string
@@ -73,7 +73,7 @@ jobs:
   # NOTE: Runner needs unzip and nodejs packages.
   all-in-one:
     name: All in one
-    if: inputs.if
+    if: ${{ inputs.if && !cancelled() }}
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:
@@ -156,6 +156,7 @@ jobs:
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
           aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
+          aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -167,6 +168,7 @@ jobs:
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
           VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
+          VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
         run: terraform plan
@@ -181,13 +183,15 @@ jobs:
           for attempt in $(seq 5); do
               if terraform apply -auto-approve; then
                   echo "Created infrastructure on attempt $attempt"
-                  break
+                  exit 0
               fi
               echo "Failed to create infrastructure on attempt $attempt"
               sleep 10
               terraform destroy -auto-approve
               sleep 60
           done
+          echo "Failed to create infrastructure after $attempt attempts"
+          exit 1
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
           OS_CLOUD: ${{ inputs.OS_CLOUD }}

.github/workflows/stackhpc-check-tags.yml

@@ -23,7 +23,7 @@ env:
 jobs:
   check-tags:
     name: Check container image tags
-    if: inputs.if
+    if: ${{ inputs.if && ! cancelled() }}
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:

.github/workflows/stackhpc-ci-cleanup.yml

@@ -0,0 +1,57 @@
+---
+name: Clean up stale CI resources
+on:
+  schedule:
+    # Every 2 hours at quarter past
+    - cron: '15 0/2 * * *'
+
+jobs:
+  ci-cleanup:
+    name: Clean up stale CI resources
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+
+      - name: Install OpenStack client
+        run: |
+          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+
+      - name: Clean up aio instances over 3 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -3hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

.github/workflows/stackhpc-pull-request.yml

@@ -87,7 +87,7 @@ jobs:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       if: ${{ needs.check-changes.outputs.check-tags == 'true' }}
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-ubuntu-jammy-ovs:
     name: aio (Ubuntu Jammy OVS)
@@ -104,7 +104,7 @@ jobs:
       OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-ubuntu-jammy-ovn:
     name: aio (Ubuntu Jammy OVN)
@@ -121,7 +121,7 @@ jobs:
       OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-rocky-9-ovs:
     name: aio (Rocky 9 OVS)
@@ -138,7 +138,7 @@ jobs:
       OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-rocky-9-ovn:
     name: aio (Rocky 9 OVN)
@@ -155,7 +155,7 @@ jobs:
       OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
 
@@ -175,7 +175,7 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
       upgrade: true
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-upgrade-rocky-9-ovn:
     name: aio upgrade (Rocky 9 OVN)
@@ -193,4 +193,4 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
       upgrade: true
     secrets: inherit
-    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

etc/kayobe/ansible.cfg

@@ -8,6 +8,8 @@ bin_ansible_callbacks = True
 inject_facts_as_vars = False
 # Add timing information to output
 callbacks_enabled = ansible.posix.profile_tasks
+# Silence warning about invalid characters found in group names
+force_valid_group_names = ignore
 
 [ssh_connection]
 pipelining = True

etc/kayobe/ansible/ubuntu-upgrade.yml

@@ -0,0 +1,110 @@
+---
+# To prevent Ansible role dependency errors, this playbook requires that environment variable
+# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' on the Ansible control host.
+- name: Migrate hosts from Ubuntu Focal 20.04 to Jammy 22.04
+  hosts: overcloud:infra-vms:seed:seed-hypervisor
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+  tasks:
+    - name: Assert that hosts are running Ubuntu Focal
+      assert:
+        that:
+          - ansible_facts.distribution == 'Ubuntu'
+          - ansible_facts.distribution_major_version == '20'
+          - ansible_facts.distribution_release == 'focal'
+          - os_distribution == 'ubuntu'
+        fail_msg: >-
+          This playbook is only designed for Ubuntu Focal 20.04 hosts. Ensure
+          that you are limiting it to only run on Focal hosts and
+          os_distribution is set to ubuntu.
+
+    - name: Ensure apt packages are up to date
+      apt:
+        update_cache: true
+        upgrade: yes
+      become: true
+
+    - name: Ensure do-release-upgrade is installed
+      package:
+        name: ubuntu-release-upgrader-core
+        state: latest
+      become: true
+
+    - name: Check whether a reboot is required
+      stat:
+        path: /var/run/reboot-required
+      register: file_status
+
+    - name: Reboot to apply updates
+      reboot:
+        reboot_timeout: 1200
+        connect_timeout: 600
+      become: true
+      when: file_status.stat.exists
+
+    # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list
+    - name: Ensure Jammy repo definitions exist in sources.list
+      blockinfile:
+        path: /etc/apt/sources.list
+        block: |
+          deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main restricted universe multiverse
+          deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main restricted universe multiverse
+          deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main restricted universe multiverse
+          deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main restricted universe multiverse
+      become: true
+
+    - name: Do release upgrade
+      command: do-release-upgrade -f DistUpgradeViewNonInteractive
+      become: true
+
+    - name: Ensure old venvs do not exist
+      file:
+        path: "/opt/kayobe/venvs/{{ item }}"
+        state: absent
+      loop:
+        - kayobe
+        - kolla-ansible
+      become: true
+
+    - name: Update Python and current user facts before re-creating Kayobe venv
+      ansible.builtin.setup:
+        filter: "{{ kayobe_ansible_setup_filter }}"
+        gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+
+- name: Run the Kayobe kayobe-target-venv playbook to ensure kayobe venv exists on remote host
+  import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml"
+
+- name: Run the Kayobe network configuration playbook, to ensure definitions are not lost on reboot
+  import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml"
+
+- name: Reboot and confirm the host is upgraded to Jammy 22.04
+  hosts: overcloud:infra-vms:seed:seed-hypervisor
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+  tasks:
+    - name: Ensure Jammy repo definitions do not exist in sources.list
+      blockinfile:
+        path: /etc/apt/sources.list
+        state: absent
+      become: true
+
+    - name: Reboot and wait
+      reboot:
+        reboot_timeout: 1200
+        connect_timeout: 600
+      become: true
+
+    - name: Update distribution facts
+      ansible.builtin.setup:
+        filter: "{{ kayobe_ansible_setup_filter }}"
+        gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+
+    - name: Assert that hosts are now using Ubuntu 22
+      assert:
+        that:
+          - ansible_facts.distribution_major_version == '22'
+          - ansible_facts.distribution_release == 'jammy'
+
+- name: Run the OVN chassis priority fix playbook
+  import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml"
+  when: kolla_enable_ovn

etc/kayobe/environments/ci-aio/kolla/globals.yml

@@ -10,7 +10,7 @@ openstack_service_workers: "1"
 openstack_service_rpc_workers: "1"
 
 # OpenSearch memory tuning
-opensearch_heap_size: 1g
+opensearch_heap_size: 200m
 
 # Increase Grafana timeout
 grafana_start_first_node_retries: 20

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -5,6 +5,9 @@
 # Docker namespace to use for Kolla images. Default is 'kolla'.
 kolla_docker_namespace: stackhpc-dev
 
+# Disable some services to reduce memory footprint.
+kolla_enable_heat: false
+
 ###############################################################################
 # Network configuration.
 
@@ -19,8 +22,14 @@ resolv_is_managed: false
 # Build and deploy the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
 stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
-stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
-stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
+stackhpc_repo_mirror_username: "skc-ci-aio"
+stackhpc_repo_mirror_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  31386366383365666135336331663635396237623139306362633933636233613765663731666338
+  3633633736333936383439623066653663333964343234350a393137383537316164323837386437
+  36613139323161643766666565643739373037623363636234343965343436653261326238393566
+  3837336661653962340a316631366463623138623530373133336665376433633437306631383666
+  30333461333535363433363336663664316634343432633766346564323833346663
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
@@ -55,19 +64,11 @@ stackhpc_include_os_minor_version_in_repo_url: true
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
-
-# Username and password of container registry.
-stackhpc_docker_registry_username: "release-train-ci"
-stackhpc_docker_registry_password: !vault |
-  $ANSIBLE_VAULT;1.1;AES256
-  38356134376436656165303634626531653836366233383531343439646433376334396438373735
-  3135643664353934356237376134623235356137383263300a333165386562396134633534376532
-  34386133383366326639353432386235336132663839333337323739633434613934346462363031
-  3265323831663964360a643962346231386462323236373963633066393736323234303833363535
-  3664
+stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}"
+stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}"
 
 # Override Pulp credentials to allow querying container image tags in the
 # check-tags.yml custom playbook.
 pulp_url: "{{ stackhpc_repo_mirror_url }}"
-pulp_username: "{{ stackhpc_docker_registry_username }}"
-pulp_password: "{{ stackhpc_docker_registry_password }}"
+pulp_username: "{{ stackhpc_repo_mirror_username }}"
+pulp_password: "{{ stackhpc_repo_mirror_password }}"

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -41,8 +41,14 @@ resolv_is_managed: false
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
 stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
-stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
-stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
+stackhpc_repo_mirror_username: "skc-ci-aio"
+stackhpc_repo_mirror_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  31386366383365666135336331663635396237623139306362633933636233613765663731666338
+  3633633736333936383439623066653663333964343234350a393137383537316164323837386437
+  36613139323161643766666565643739373037623363636234343965343436653261326238393566
+  3837336661653962340a316631366463623138623530373133336665376433633437306631383666
+  30333461333535363433363336663664316634343432633766346564323833346663
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"