Merge branch 'main' into fix/templating2

Author: sjpb

.github/workflows/stackhpc.yml

@@ -14,7 +14,7 @@ jobs:
         cloud:
           - "arcus"   # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
       fail-fast: false # as want clouds to continue independently
-    concurrency: ${{ matrix.cloud }}
+    concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -52,30 +52,52 @@ jobs:
           smslabs_CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
 
-      - name: Provision infrastructure
-        id: provision
+      - name: Provision ports, inventory and other infrastructure apart from nodes
+        id: provision_ports
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform apply -auto-approve
+          TF_VAR_create_nodes=false terraform apply -auto-approve
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
 
+      - name: Setup environment-specific inventory/terraform inputs
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
+          ansible-playbook ansible/adhoc/template-cloud-init.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Provision servers
+        id: provision_servers
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform apply -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+
       - name: Get server provisioning failure messages
         id: provision_failure
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo $TF_FAIL_MSGS
+          echo TF failure messages: $TF_FAIL_MSGS
           echo "::set-output name=messages::${TF_FAIL_MSGS}"
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: always() && steps.provision.outcome == 'failure'
+        if: always() && steps.provision_servers.outcome == 'failure'
 
       - name: Delete infrastructure if failed due to lack of hosts
         run: |
@@ -86,20 +108,17 @@ jobs:
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+        if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
 
       - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -vv ansible/site.yml
         env:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Run MPI-based tests
         run: |
@@ -135,23 +154,24 @@ jobs:
             --server-response \
             --no-check-certificate \
             --http-user=testuser \
-            --http-password=${TEST_USER_PASSWORD} https://${openondemand_servername} \
+            --http-password=${TESTUSER_PASSWORD} https://${openondemand_servername} \
             2>&1)
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Build packer images
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           cd packer/
           PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
         env:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Test reimage of nodes
         run: |

ansible/.gitignore

@@ -26,6 +26,10 @@ roles/*
 !roles/slurm_exporter/**
 !roles/firewalld/
 !roles/firewalld/**
+!roles/etc_hosts/
+!roles/etc_hosts/**
+!roles/cloud_init/
+!roles/cloud_init/**
 !roles/mysql/
 !roles/mysql/**
 !roles/systemd/

ansible/adhoc/hpctests.yml

@@ -4,7 +4,7 @@
 
 ---
 
-- hosts: hpctests[0] # TODO: might want to make which node is used selectable?
+- hosts: login[0] # TODO: might want to make which node is used selectable?
   become: false
   gather_facts: false
   tasks:

ansible/adhoc/template-cloud-init.yml

@@ -0,0 +1,9 @@
+- hosts: cloud_init
+  become: no
+  gather_facts: no
+  tasks:
+    - name: Template out cloud-init userdata
+      import_role:
+        name: cloud_init
+        tasks_from: template.yml
+      delegate_to: localhost

ansible/bootstrap.yml

@@ -13,6 +13,19 @@
           to update these variable names. ** NB: The actual secrets will not be changed.**
       when: "'secrets_openhpc_' in (hostvars[inventory_hostname] | join)"
 
+- hosts: etc_hosts
+  gather_facts: false
+  tags: etc_hosts
+  become: yes
+  tasks:
+    - name: Template /etc/hosts
+      copy:
+        content: "{{ etc_hosts_template }}"
+        dest: /etc/hosts
+        owner: root
+        group: root
+        mode: u=rw,og=r
+
 - hosts: cluster
   gather_facts: false
   tasks:
@@ -74,6 +87,20 @@
     - import_role:
         name: fail2ban
 
+- name: Setup podman
+  hosts: podman
+  tags: podman
+  tasks:
+    - import_role:
+        name: podman
+        tasks_from: prereqs.yml
+      tags: prereqs
+
+    - import_role:
+        name: podman
+        tasks_from: config.yml
+      tags: config
+
 - hosts: update
   gather_facts: false
   become: yes

ansible/ci/test_reimage.yml

@@ -45,10 +45,10 @@
   gather_facts: no
   tags: reimage_compute
   tasks:
-    # TODO: This is specific to smslabs/arcus environment config - could generalise to all compute nodes
+    # TODO: This is specific to arcus environment config - could generalise to all compute nodes
     - name: Request compute node rebuild via Slurm
       shell:
-        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]
+        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-3]
       become: yes
 
     - name: Check compute node rebuild completed

ansible/monitoring.yml

@@ -1,20 +1,6 @@
 # ---
 # # NOTE: Requires slurmdbd
 
-- name: Setup podman
-  hosts: podman
-  tags: podman
-  tasks:
-    - import_role:
-        name: podman
-        tasks_from: prereqs.yml
-      tags: prereqs
-
-    - import_role:
-        name: podman
-        tasks_from: config.yml
-      tags: config
-
 - name: Setup elasticsearch
   hosts: opendistro
   tags: opendistro

ansible/roles/cloud_init/README.md

@@ -0,0 +1,31 @@
+# cloud_init
+
+Create cloud init userdata for instance groups.
+
+# Requirements
+Image and cloud environment supporting cloud-init.
+
+# Role Variables
+
+- `cloud_init_output_path`: Required. Path to output userdata files to.
+- `cloud_init_userdata_templates`: Optional list. Each element is a dict with keys/values as follows:
+    - `module`: Required str. Name of cloud_init [module](https://cloudinit.readthedocs.io/en/latest/topics/modules.html)
+    - `group`: Optional str. Name of inventory group to which this config applies - if no group is specified then it applies to all groups. This allows defining `cloud_init_userdata_templates` for group `all`.
+    - `template`: Jinja template for cloud_init module [configuration](https://cloudinit.readthedocs.io/en/latest/topics/modules.html).
+
+  Elements may repeat `module`; the resulting userdata cloud-config file will will contain configuration from all applicable (by group) elements for that module.
+  
+  Note that the appliance [constructs](../../../environments/common/inventory/group_vars/all/cloud_init.yml) `cloud_init_userdata_templates` from `cloud_init_userdata_templates_default` and `cloud_init_userdata_templates_extra` to 
+  allow easier customisation in specific environments.
+
+# Dependencies
+None.
+
+# Example Playbook
+See `ansible/adhoc/rebuild.yml`.
+
+# License
+Apache 2.0
+
+# Author Information
+[email protected]

ansible/roles/cloud_init/defaults/main.yml

@@ -0,0 +1,2 @@
+#cloud_init_output_path: 
+cloud_init_userdata_templates: []

ansible/roles/cloud_init/tasks/template.yml

@@ -0,0 +1,5 @@
+
+- name: Template out cloud-init userdata
+  ansible.builtin.template:
+    src: userdata.yml.j2
+    dest: "{{ cloud_init_output_path }}/{{ inventory_hostname }}.userdata.yml"

ansible/roles/cloud_init/templates/userdata.yml.j2

@@ -0,0 +1,13 @@
+#cloud-config
+disable_ec2_metadata: true
+
+{% for module, tmpls in cloud_init_userdata_templates | groupby(attribute='module') %}
+{% for tmpl in tmpls %}
+{% if not 'group' in tmpl or tmpl.group in group_names %}
+{% if loop.first %}
+{{ module }}:
+{% endif %}
+{{ tmpl.template }}
+{% endif %}
+{% endfor %}
+{% endfor %}

ansible/roles/etc_hosts/README.md

@@ -0,0 +1,5 @@
+# etc_hosts
+
+This role provides documentation only.
+
+Hosts in the `etc_hosts` groups get `/etc/hosts` created via `cloud-init`. The generated file defines all hosts in this group using `ansible_host` as the IP address and `inventory_hostname` as the canonical hostname. This may need overriding for multi-homed hosts. See `environments/common/inventory/group_vars/all/cloud_init.yml` for configuration.

ansible/roles/hpctests/README.md

@@ -24,7 +24,8 @@ Role Variables
 --------------
 
 - `hpctests_rootdir`: Required. Path to root of test directory tree, which must be on a r/w filesystem shared to all cluster nodes under test. The last directory component will be created.
-- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the default partition are used. Note nodes selected **must** be in the default partition.
+- `hpctests_partition`: Optional. Name of partition to use, otherwise default partition is used.
+- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the selected partition are used.
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use).
 - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user).
 - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html).

ansible/roles/hpctests/defaults/main.yml

@@ -10,3 +10,4 @@ hpctests_hpl_NB: 192
 hpctests_hpl_mem_frac: 0.8
 hpctests_hpl_arch: linux64
 #hpctests_nodes:
+#hpctests_partition:

ansible/roles/hpctests/library/slurm_node_info.py

@@ -23,7 +23,7 @@
 options
     nodes:
         description:
-            - Slurm nodenames for which information is required. These must be homogenous.
+            - Slurm nodenames for which information is required.
         required: true
         type: list
 requirements:
@@ -56,7 +56,6 @@ def run_module():
     print(values)
     for ix, param in enumerate(params):
         info[param] = [nodeinfo[ix].strip() for nodeinfo in values if nodeinfo[nodelist_ix].strip() in module.params['nodes']]
-        # info[param] = [nodeinfo[nodelist_ix] for nodeinfo in values]
     result['info'] = info
 
     module.exit_json(**result)

ansible/roles/hpctests/tasks/hpl-solo.yml

@@ -42,8 +42,8 @@
 - debug:
     msg: "Using {{ hpctests_hplsolo_ntasks }} process per node with P={{ hpctests_hplsolo_pq.grid.P }}, Q={{ hpctests_hplsolo_pq.grid.Q }} targeting {{ (hpctests_hpl_mem_frac | float)  * 100 }}% of {{ hpctests_nodeinfo.info['MEMORY'][0] }} MB memory per node, block size (NB) = {{ hpctests_hpl_NB }}, problem size (N) = {{ hpctests_hplsolo_N }}"
 
-- name: Get all nodes
-  shell: "sinfo --Node --noheader --format %N" # TODO: assumes only one partition, although actually excluding nodes not in the default partition should be fine.
+- name: Get all nodes in partition
+  shell: "sinfo --Node --noheader --format %N --partition={{ hpctests_partition }}"
   register: all_nodes
   changed_when: false
 
@@ -74,6 +74,11 @@
   vars:
     hpctests_hplsolo_ntasks: 2 # TODO: FIXME
 
+- name: Remove previous outputs
+  # As depending on the number of nodes there will be different numbers of output files for different partitions so won't all get overwritten
+  shell:
+    cmd: "rm -f {{ hpctests_rootdir }}/hpl-solo/hpl-solo.sh.*.out"
+
 - name: Run hpl-solo
   shell: sbatch --wait hpl-solo.sh
   become: no
@@ -111,10 +116,11 @@
   tags: postpro
   debug:
     msg: |
-      Summary for hpl-solo ({{ hpctests_computes.stdout_lines | length }} nodes) job {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}:
+      Summary for hpl-solo on {{ hpctests_computes.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}':
+
         Max:  {{ perf.stdout_lines | map('float') | max }} gflops
         Min:  {{ perf.stdout_lines | map('float') | min }} gflops
-        Mean: {{ (perf.stdout_lines | map('float') | sum) /  (hpctests_computes.stdout_lines | length) }} gflops
+        Mean: {{ (perf.stdout_lines | map('float') | sum) / (hpctests_computes.stdout_lines | length) }} gflops
 
       Individual node results (gflops):
       {{ dict(hpctests_computes.stdout_lines | zip(perf.stdout_lines | map('float') )) | to_nice_yaml }}

ansible/roles/hpctests/tasks/pingmatrix.yml

@@ -61,6 +61,8 @@
 - name: Summarise results
   debug:
     msg: |
-      Summary for pingmatrix (pairwise on {{ slurm_names.stdout_lines | length }} nodes) job {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}:
+      Summary for pingmatrix pairwise over {{ slurm_names.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}':
+      
       {{ nxnlatbw['stats'] | to_nice_yaml }}
+      
       Tabular output on ansible control host at {{ hpctests_outdir }}/pingmatrix.html

ansible/roles/hpctests/tasks/pingpong.yml

@@ -55,10 +55,11 @@
 
 - debug:
     msg: |
-      Summary for pingpong (2x scheduler-selected nodes) job {{ _pingpong_jobid }} (using interface {{ hpctests_ucx_net_devices }}):
-      nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }}
-      zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
-      max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
+      Summary for pingpong using 2x scheduler-selected nodes in '{{ hpctests_partition }}' partition, job ID {{ _pingpong_jobid }}, device '{{ hpctests_ucx_net_devices }}':
+
+      Nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }}
+      Zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
+      Max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
 
       See plot on localhost:
       {{ _pingpong_plot.stdout }}

ansible/roles/hpctests/tasks/setup.yml

@@ -1,7 +1,17 @@
 ---
 
+- name: Get partition information
+  shell: "sinfo --format %P --noheader"
+  register: _sinfo_partitions
+  changed_when: false
+
+- name: Select default partition if hpctests_partition not given
+  set_fact:
+    hpctests_partition: "{{ _sinfo_partitions.stdout_lines  | select('contains', '*') | first | trim('*') }}"
+  when: hpctests_partition is not defined
+
 - name: Get info about compute nodes
-  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --format %N"
+  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --partition {{hpctests_partition}} --format %N"
   register: hpctests_computes
   changed_when: false
   failed_when: hpctests_computes.rc != 0