Merge branch 'main' into fix/elasticsearch

Author: sjpb

.github/workflows/stackhpc.yml

@@ -12,7 +12,6 @@ jobs:
     strategy:
       matrix:
         cloud:
-          - "smslabs" # SMS-Labs OpenStack in stackhpc-ci project
           - "arcus"   # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
       fail-fast: false # as want clouds to continue independently
     concurrency: ${{ matrix.cloud }}
@@ -70,7 +69,9 @@ jobs:
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          echo "::set-output name=messages::$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
+          TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
+          echo $TF_FAIL_MSGS
+          echo "::set-output name=messages::${TF_FAIL_MSGS}"
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
@@ -87,8 +88,7 @@ jobs:
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
-      - name: Directly configure cluster and build compute, login and control images
-        # see pre-hook for the image build
+      - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
@@ -131,9 +131,20 @@ jobs:
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-          
-      - name: Test reimage of login and compute nodes
-        # TODO: test control node reimage
+
+      - name: Build packer images
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
+          cd packer/
+          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Test reimage of nodes
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate

ansible/bootstrap.yml

@@ -23,6 +23,8 @@
       # Need to change working directory otherwise we try to switch back to non-existent directory.
       become_flags: '-i'
       become: true
+    - name: Reset ssh connection to allow user changes to affect ansible_user
+      meta: reset_connection
 
 - hosts: selinux
   gather_facts: false
@@ -94,6 +96,8 @@
   become: yes
   tags:
     - reboot
+    - selinux
+    - update
   tasks:
     - name: Check for pending reboot from package updates
       stat: 

ansible/roles/fail2ban/.travis.yml

@@ -0,0 +1,85 @@
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+from matplotlib import ticker
+import numpy as np
+import os
+
+def sizeof_fmt(num, suffix='B'):
+    """ TODO: """
+    # from https://stackoverflow.com/a/1094933/916373
+    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)
+
+def read_imb_out(path):
+    """ Read stdout from an IMB-MPI1 run.
+        
+        Returns a dict with:
+            key:= int, total number of processes involved
+            value:= pandas dataframe, i.e. one per results table. Columns as per table.
+        
+        If multiple results tables are present it is assumed that they are all the same benchmark,
+        and only differ in the number of processes.
+    """
+
+    data = {}
+
+    COLTYPES = { # all benchmark names here should be lowercase
+        'uniband': (int, int, float, int), # #bytes #repetitions Mbytes/sec Msg/sec
+        'biband': (int, int, float, int),
+        'pingpong':(int, int, float, float), # #bytes #repetitions t[usec] Mbytes/sec
+        'alltoall':(int, int, float, float, float) # #bytes #repetitions t_min[usec] t_max[usec] t_avg[usec]
+    }
+
+    with open(path) as f:
+        for line in f:
+            if line.startswith('# Benchmarking '):
+                benchmark = line.split()[-1].lower()
+                if benchmark not in COLTYPES:
+                    raise ValueError('Do not know how to read %r benchmark in %s' % (benchmark, path))
+                converters = COLTYPES[benchmark]
+                line = next(f)
+                if not line.startswith('# #processes = '):
+                    raise ValueError('expected %s, got %s' % (expect, nprocs_line))
+                n_procs = int(line.split('=')[-1].strip())
+                while line.startswith('#'):
+                    line = next(f) # may or may not include line "# .. additional processes waiting in MPI_Barrier", plus other # lines
+                rows = []
+                while True:
+                    line = next(f).strip()
+                    if line == '':
+                        break
+                    rows.append([f(v) for (f, v) in zip(converters, line.split())])
+                # turn data around:
+                cols = [[] for _ in range(len(converters))]
+                for r in rows:
+                    for ic, c in enumerate(cols):
+                        c.append(r[ic])
+                data[n_procs] = cols
+    return data
+
+if __name__ == '__main__':
+    import sys
+    d = read_imb_out(sys.argv[1])
+    if len(d) > 1:
+        raise ValueError('Found > 1 benchmark in', sys.argv[1])
+    outdir = os.path.dirname(sys.argv[1])
+    for n, df in d.items():
+        fig, ax1 = plt.subplots()
+        ax2 = ax1.twinx()
+        ax1.plot(df[0], df[2], label='latency', color='b')
+        ax2.plot(df[0], df[3], label='bandwidth', color='r')
+        ax1.set_xscale('log', base=2)
+        ax1.set_yscale('log', base=10)
+        ax1.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: sizeof_fmt(x)))
+        ax1.grid(True, which="both")
+        ax1.set_xlabel('#bytes')
+        ax1.set_ylabel('latency ($\mu$s)', color='b')
+        ax2.set_ylabel('bandwidth (Mbytes/sec)', color='r')
+        fig.legend(loc='upper left')
+        plt.tight_layout()
+        figpath = os.path.join(outdir, 'pingpong.png')
+        plt.savefig(figpath)
+        print(figpath)

ansible/roles/firewalld/.travis.yml

@@ -47,17 +47,16 @@ def run_module():
     if module.check_mode:
         module.exit_json(**result)
 
-    node_spec = ','.join(module.params['nodes'])
-    _, stdout,_ = module.run_command("sinfo --Format All --node %s" % node_spec, check_rc=True)
+    _, stdout,_ = module.run_command("sinfo --Format All --Node", check_rc=True) # `--nodes` doesn't filter enough, other partitions are still shown
     lines = stdout.splitlines()
-    # if len(lines) > 2:
-    #     raise ValueError('Info requested for nodes which are not homogenous: %s' % lines)
     info = {}
     params = [v.strip() for v in lines[0].split('|')]
     values = [line.split('|') for line in lines[1:]]
+    nodelist_ix = params.index('NODELIST')
     print(values)
     for ix, param in enumerate(params):
-        info[param] = [nodeinfo[ix].strip() for nodeinfo in values]
+        info[param] = [nodeinfo[ix].strip() for nodeinfo in values if nodeinfo[nodelist_ix].strip() in module.params['nodes']]
+        # info[param] = [nodeinfo[nodelist_ix] for nodeinfo in values]
     result['info'] = info
 
     module.exit_json(**result)

ansible/roles/hpctests/.travis.yml

@@ -16,7 +16,7 @@
 - name: Check nodes are homogenous
   assert:
     that: "{{ hpctests_nodeinfo.info[item] | unique | length == 1 }}"
-    fail_msg: "Selected nodes are not homogenous: {{ item }} = {{ hpctests_nodeinfo.info[item] }}"
+    fail_msg: "Selected nodes are not homogenous: {{ item }} ({{ hpctests_nodeinfo.info['NODELIST'] }}) = {{ hpctests_nodeinfo.info[item] }}"
   loop:
     - SOCKETS
     - CORES

ansible/roles/hpctests/files/plot_imb_pingpong.py

@@ -24,18 +24,41 @@
     chdir: "{{ hpctests_rootdir }}/pingpong"
   register: hpctests_pingpong_sbatch
 
-- name: Read pingpong
+- set_fact:
+    _pingpong_jobid: "{{ hpctests_pingpong_sbatch.stdout.split()[-1] }}"
+- set_fact:
+    _pingpong_local_output: "{{ hpctests_outdir }}/pingpong/{{_pingpong_jobid}}/pingpong.sh.out"
+
+- name: Retrieve results file
+  ansible.builtin.fetch:
+    src: "{{ hpctests_rootdir }}/pingpong/pingpong.sh.out"
+    dest: "{{ _pingpong_local_output }}"
+    flat: yes
+
+- name: Read pingpong results
   read_imb_pingpong:
-    path: "{{ hpctests_rootdir }}/pingpong/pingpong.sh.out"
+    path: "{{ _pingpong_local_output }}"
   register: hpctests_pingpong_out
+  delegate_to: localhost
 
 - name: Read nodes used
-  shell: "grep 'SLURM_JOB_NODELIST:' {{ hpctests_rootdir }}/pingpong/pingpong.sh.out"
+  shell: "grep 'SLURM_JOB_NODELIST:' {{ _pingpong_local_output }}"
   register: hpctests_pingpong_run_nodes
+  delegate_to: localhost
 
+- name: Plot image
+  shell:
+    cmd: "python {{lookup('env', 'APPLIANCES_REPO_ROOT') }}/ansible/roles/hpctests/files/plot_imb_pingpong.py {{ _pingpong_local_output }}"
+    creates: "{{ _pingpong_local_output | dirname }}/latency.png"
+  register: _pingpong_plot
+  delegate_to: localhost
+  
 - debug:
     msg: |
-      Summary for pingpong (2x scheduler-selected nodes) job {{ hpctests_pingpong_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}:
+      Summary for pingpong (2x scheduler-selected nodes) job {{ _pingpong_jobid }} (using interface {{ hpctests_ucx_net_devices }}):
       nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }}
       zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
       max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
+
+      See plot on localhost:
+      {{ _pingpong_plot.stdout }}

ansible/roles/hpctests/library/slurm_node_info.py

@@ -4,7 +4,7 @@
 #SBATCH --output=%x.%a.out
 #SBATCH --error=%x.%a.out
 #SBATCH --exclusive
-{%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_nodes }}{% endif %}
+{%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_computes.stdout_lines[0] }}{% endif %}
 
 echo HPL arch: {{ hpctests_hpl_arch }}
 

ansible/roles/hpctests/tasks/hpl-solo.yml

@@ -1,22 +1,3 @@
-- hosts: localhost
-  become: false
-  tags: build
-  tasks:
-    - name: Ensure secrets generated
-      include_role:
-        name: passwords
-    
-    - name: Build packer images
-      shell:
-        cmd: |
-          cd packer
-          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        chdir: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}"
-      when: "'builder' not in group_names" # avoid recursion!
-      register: packer_run
-      async: 2700 # 45 minutes
-      poll: 0
-
 - hosts: all
   become: true
   tags: etc_hosts

ansible/roles/hpctests/tasks/pingpong.yml

@@ -70,8 +70,9 @@ cluster
 # Hosts to install fail2ban on to protect SSH - uses firewalld
 # https://www.fail2ban.org/wiki/index.php/Main_Page
 
-[firewalld]
+[firewalld:children]
 # Hosts to install firewalld on - see ansible/roles/filewalld
+fail2ban
 
 [block_devices]
 # Superset of hosts to configure filesystems on - see ansible/roles/block_devices/README.md

ansible/roles/hpctests/templates/hpl-build.sh.j2

@@ -37,10 +37,6 @@ cluster
 # Hosts to install fail2ban on to protect SSH
 login
 
-[firewalld:children]
-# Hosts to install firewalld on
-fail2ban
-
 [block_devices:children]
 # Environment-specific so not defined here