support multiple partitions in hpctests

Author: sjpb

ansible/roles/hpctests/README.md

@@ -24,7 +24,8 @@ Role Variables
 --------------
 
 - `hpctests_rootdir`: Required. Path to root of test directory tree, which must be on a r/w filesystem shared to all cluster nodes under test. The last directory component will be created.
-- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the default partition are used. Note nodes selected **must** be in the default partition.
+- `hpctests_partition`: Optional. Name of partition to use, otherwise default partition is used.
+- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the selected partition are used.
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use).
 - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user).
 - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html).

ansible/roles/hpctests/defaults/main.yml

@@ -10,3 +10,4 @@ hpctests_hpl_NB: 192
 hpctests_hpl_mem_frac: 0.8
 hpctests_hpl_arch: linux64
 #hpctests_nodes:
+#hpctests_partition:

ansible/roles/hpctests/library/slurm_node_info.py

@@ -23,7 +23,7 @@
 options
     nodes:
         description:
-            - Slurm nodenames for which information is required. These must be homogenous.
+            - Slurm nodenames for which information is required.
         required: true
         type: list
 requirements:
@@ -56,7 +56,6 @@ def run_module():
     print(values)
     for ix, param in enumerate(params):
         info[param] = [nodeinfo[ix].strip() for nodeinfo in values if nodeinfo[nodelist_ix].strip() in module.params['nodes']]
-        # info[param] = [nodeinfo[nodelist_ix] for nodeinfo in values]
     result['info'] = info
 
     module.exit_json(**result)

ansible/roles/hpctests/tasks/hpl-solo.yml

@@ -42,8 +42,8 @@
 - debug:
     msg: "Using {{ hpctests_hplsolo_ntasks }} process per node with P={{ hpctests_hplsolo_pq.grid.P }}, Q={{ hpctests_hplsolo_pq.grid.Q }} targeting {{ (hpctests_hpl_mem_frac | float)  * 100 }}% of {{ hpctests_nodeinfo.info['MEMORY'][0] }} MB memory per node, block size (NB) = {{ hpctests_hpl_NB }}, problem size (N) = {{ hpctests_hplsolo_N }}"
 
-- name: Get all nodes
-  shell: "sinfo --Node --noheader --format %N" # TODO: assumes only one partition, although actually excluding nodes not in the default partition should be fine.
+- name: Get all nodes in partition
+  shell: "sinfo --Node --noheader --format %N --partition={{ hpctests_partition }}"
   register: all_nodes
   changed_when: false
 
@@ -74,6 +74,11 @@
   vars:
     hpctests_hplsolo_ntasks: 2 # TODO: FIXME
 
+- name: Remove previous outputs
+  # As depending on the number of nodes there will be different numbers of output files for different partitions so won't all get overwritten
+  shell:
+    cmd: "rm -f {{ hpctests_rootdir }}/hpl-solo/hpl-solo.sh.*.out"
+
 - name: Run hpl-solo
   shell: sbatch --wait hpl-solo.sh
   become: no

ansible/roles/hpctests/tasks/setup.yml

@@ -1,7 +1,17 @@
 ---
 
+- name: Get partition information
+  shell: "sinfo --format %P --noheader"
+  register: _sinfo_partitions
+  changed_when: false
+
+- name: Select default partition if hpctests_partition not given
+  set_fact:
+    hpctests_partition: "{{ _sinfo_partitions.stdout_lines  | select('contains', '*') | first | trim('*') }}"
+  when: hpctests_partition is not defined
+
 - name: Get info about compute nodes
-  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --format %N"
+  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --partition {{hpctests_partition}} --format %N"
   register: hpctests_computes
   changed_when: false
   failed_when: hpctests_computes.rc != 0

ansible/roles/hpctests/templates/hpl-build.sh.j2

@@ -4,6 +4,7 @@
 #SBATCH --output=%x.%a.out
 #SBATCH --error=%x.%a.out
 #SBATCH --exclusive
+#SBATCH --partition={{ hpctests_partition }}
 {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_computes.stdout_lines[0] }}{% endif %}
 
 echo HPL arch: {{ hpctests_hpl_arch }}

ansible/roles/hpctests/templates/hpl-solo.sh.j2

@@ -5,6 +5,7 @@
 #SBATCH --error=%x.%a.out
 #SBATCH --exclusive
 #SBATCH --array=0-{{ hpctests_computes.stdout_lines | length - 1 }}
+#SBATCH --partition={{ hpctests_partition }}
 {% if hpctests_hplsolo_excluded_nodes | length > 0 %}
 #SBATCH --exclude={{ hpctests_hplsolo_excluded_nodes | join(',') }}
 {% endif %}

ansible/roles/hpctests/templates/pingmatrix.sh.j2

@@ -5,6 +5,7 @@
 #SBATCH --output=%x.out
 #SBATCH --error=%x.out
 #SBATCH --exclusive
+#SBATCH --partition={{ hpctests_partition }}
 {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_nodes }}{% endif %}
 
 export UCX_NET_DEVICES={{ hpctests_ucx_net_devices }}

ansible/roles/hpctests/templates/pingpong.sh.j2

@@ -5,6 +5,7 @@
 #SBATCH --output=%x.out
 #SBATCH --error=%x.out
 #SBATCH --exclusive
+#SBATCH --partition={{ hpctests_partition }}
 {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_nodes }}{% endif %}
 
 export UCX_NET_DEVICES={{ hpctests_ucx_net_devices }}