Merge branch 'main' into fix/hpctests-partitions-jinja

Author: sjpb

.github/workflows/stackhpc.yml

@@ -16,6 +16,10 @@ jobs:
       fail-fast: false # as want clouds to continue independently
     concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      TF_VAR_cluster_name: ci${{ github.run_id }}
     steps:
       - uses: actions/checkout@v2
 
@@ -26,7 +30,6 @@ jobs:
           echo "${${{ matrix.cloud }}_SSH_KEY}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
         env:
-          smslabs_SSH_KEY: ${{ secrets.SSH_KEY }}
           arcus_SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
 
       - name: Add bastion's ssh key to known_hosts
@@ -49,19 +52,14 @@ jobs:
           echo "${${{ matrix.cloud }}_CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml
         shell: bash
         env:
-          smslabs_CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
 
       - name: Provision ports, inventory and other infrastructure apart from nodes
-        id: provision_ports
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           TF_VAR_create_nodes=false terraform apply -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
       
       - name: Setup environment-specific inventory/terraform inputs
         run: |
@@ -71,7 +69,6 @@ jobs:
           echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           ansible-playbook ansible/adhoc/template-cloud-init.yml
         env:
-          ANSIBLE_FORCE_COLOR: True
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Provision servers
@@ -81,9 +78,6 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
 
       - name: Get server provisioning failure messages
         id: provision_failure
@@ -94,9 +88,6 @@ jobs:
           TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
           echo TF failure messages: $TF_FAIL_MSGS
           echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: always() && steps.provision_servers.outcome == 'failure'
 
       - name: Delete infrastructure if failed due to lack of hosts
@@ -105,29 +96,21 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
 
       - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
-          ansible-playbook -vv ansible/site.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
 
       - name: Run MPI-based tests
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
@@ -161,62 +144,62 @@ jobs:
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Build packer images
+        id: packer_build
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           cd packer/
           PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
-          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          ../dev/output_manifest.py packer-manifest.json # Sets NEW_{COMPUTE,CONTROL,LOGIN}_IMAGE_ID outputs
 
-      - name: Test reimage of nodes
+      - name: Test reimage of login nodes (via rebuild adhoc)
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          ansible all -m wait_for_connection
-          ansible-playbook -vv ansible/ci/test_reimage.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+          ansible-playbook -v --limit login ansible/adhoc/rebuild.yml -e rebuild_image=${{ steps.packer_build.outputs.NEW_LOGIN_IMAGE_ID }}
+          ansible login -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Test reimage of compute nodes (via slurm)
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible login -v -a "sudo scontrol reboot ASAP nextstate=RESUME reason='rebuild image:${{ steps.packer_build.outputs.NEW_COMPUTE_IMAGE_ID }}' ${TF_VAR_cluster_name}-compute-[0-3]"
+          ansible compute -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook -v ansible/ci/check_slurm.yml
+      
+      - name: Test reimage of control node (via rebuild adhoc)
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook -v --limit control ansible/adhoc/rebuild.yml -e rebuild_image=${{ steps.packer_build.outputs.NEW_CONTROL_IMAGE_ID }}
+          ansible control -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook ansible/slurm.yml --tags openhpc # configures partitions
+          ansible-playbook ansible/monitoring.yml --tags prometheus # configures scrapes
+          ansible-playbook -v ansible/ci/check_slurm.yml
       
       - name: Check sacct state survived reimage
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Check MPI-based tests are shown in Grafana
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/check_grafana.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ success() || cancelled() }}
 
       - name: Delete images
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/delete_images.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True

ansible/bootstrap.yml

@@ -117,8 +117,6 @@
         async: "{{ 30 * 60 }}" # wait for up to 30 minutes
         poll: 15 # check every 15 seconds
         register: updates
-      - debug:
-          var: updates
       - name: Ensure update log directory on localhost exists
         file:
           path: "{{ update_log_path | dirname }}"

ansible/ci/check_slurm.yml

@@ -0,0 +1,25 @@
+- hosts: login:!builder # won't have a slurm control daemon when in build
+  become: no
+  gather_facts: false
+  tasks:
+    - name: Run sinfo
+      shell: 'sinfo --noheader --format="%N %P %a %l %D %t" | sort' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
+      register: sinfo
+      changed_when: false
+      until: "'boot' not in sinfo.stdout_lines"
+      retries: 5
+      delay: 10
+    - name: Check nodes have expected slurm state
+      assert:
+        that: sinfo.stdout_lines == expected_sinfo
+        fail_msg: |
+          sinfo output not as expected:
+          actual:
+          {{ sinfo.stdout_lines }}
+          expected:
+          {{ expected_sinfo }}
+          <end>
+      vars:
+        expected_sinfo:
+          - "{{ openhpc_cluster_name }}-compute-[0-1] small* up 60-00:00:00 2 idle"
+          - "{{ openhpc_cluster_name }}-compute-[2-3] extra up 60-00:00:00 2 idle"

ansible/ci/test_reimage.yml

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# Set github workflow output parameters defining image IDs from a packer manifest.
+# Usage:
+#   ./packer/read_manifest.py packer/packer-manifest.json
+
+# E.g. assuming the default packer builds this will produce something like:
+#   ::set-output name=NEW_COMPUTE_IMAGE_ID::9aabd73d-e550-4116-a90c-700478b722ce
+#   ::set-output name=NEW_LOGIN_IMAGE_ID::87b41d58-d7e3-4c38-be05-453c3287ecab
+#   ::set-output name=NEW_CONTROL_IMAGE_ID::7f812168-73fe-4a60-b9e9-9109a405390d
+# which can be used in subsequent workflow steps: [1]
+#
+# [1]: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-setting-a-value
+
+import sys, json
+output = {}
+with open(sys.argv[1]) as f:
+    data = json.load(f)
+for build in data['builds']:
+    node_type = build['custom_data']['source']
+    image_id = build['artifact_id']
+    output[node_type] = image_id # NB: this deliberately gets the LAST build for a node type
+for node_type, image_id in output.items():
+    print('::set-output name=NEW_%s_IMAGE_ID::%s' % (node_type.upper(), image_id))