Merge branch 'main' into feature/control-images2

Author: sjpb

.github/workflows/smslabs.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 concurrency: stackhpc-ci # openstack project
 jobs:
-  openstack-example:
+  smslabs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -77,63 +77,56 @@ jobs:
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
-      - name: Configure infrastructure
+      - name: Directly configure cluster and build compute, login and control images
+        # see pre-hook for the image build
         run: |
           . venv/bin/activate
           . environments/smslabs/activate
           ansible all -m wait_for_connection
           ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -vv ansible/site.yml
         env:
+          OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Run MPI-based tests
-        run: |
-          . venv/bin/activate
-          . environments/smslabs/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-      
-      - name: Build control, login and compute images
+      - name: Test reimage of login and compute nodes
+        # TODO: test control node reimage
         run: |
           . venv/bin/activate
           . environments/smslabs/activate
-          cd packer
-          PACKER_LOG=1 PACKER_LOG_PATH=build.log packer build -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+          ansible all -m wait_for_connection
+          ansible-playbook -vv ansible/ci/test_reimage.yml
         env:
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          ANSIBLE_FORCE_COLOR: True
 
-      - name: Reimage compute nodes via slurm and check cluster still up
+      - name: Run MPI-based tests
         run: |
           . venv/bin/activate
           . environments/smslabs/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-compute.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
         env:
+          ANSIBLE_FORCE_COLOR: True
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Reimage login nodes via openstack and check cluster still up
+      - name: Delete infrastructure
         run: |
           . venv/bin/activate
           . environments/smslabs/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-login.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform destroy -auto-approve
         env:
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: ${{ success() || cancelled() }}
 
-      - name: Delete infrastructure
+      - name: Delete images
         run: |
           . venv/bin/activate
           . environments/smslabs/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform destroy -auto-approve
+          ansible-playbook -vv ansible/ci/delete_images.yml
         env:
           OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: ${{ success() || cancelled() }}
+          ANSIBLE_FORCE_COLOR: True

environments/smslabs/ci/reimage-login.yml renamed to ansible/ci/delete_images.yml

@@ -1,23 +1,23 @@
-# Reimage login nodes via OpenStack
-
-- hosts: login
+- hosts: login:!builder
   become: no
+  gather_facts: no
   tasks:
     - name: Read packer build manifest
       set_fact:
         manifest: "{{ lookup('file', manifest_path) | from_json }}"
       vars:
         manifest_path: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}/packer/packer-manifest.json"
       delegate_to: localhost
-      
-    - name: Get latest login image build
+    
+    - name: Get latest image builds
       set_fact:
         login_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'login'}) | last }}"
+        compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
 
-    - name: Reimage node via openstack
+    - name: Delete images
       shell:
-        cmd: "openstack server rebuild {{ instance_id | default(inventory_hostname) }} --image {{ login_build.artifact_id }}"
+        cmd: |
+          openstack image delete {{ login_build.artifact_id }}
+          openstack image delete {{ compute_build.artifact_id }}
       delegate_to: localhost
-    
-    - name: Wait for connection
-      wait_for_connection:
+    

ansible/ci/test_reimage.yml

@@ -0,0 +1,65 @@
+- hosts: login:!builder
+  become: no
+  tasks:
+    - name: Read packer build manifest
+      set_fact:
+        manifest: "{{ lookup('file', manifest_path) | from_json }}"
+      vars:
+        manifest_path: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}/packer/packer-manifest.json"
+      delegate_to: localhost
+    
+    - name: Get latest image builds
+      set_fact:
+        login_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'login'}) | last }}"
+        compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
+    
+    - name: Reimage login node via openstack
+      shell:
+        cmd: "openstack server rebuild {{ instance_id | default(inventory_hostname) }} --image {{ login_build.artifact_id }}"
+      delegate_to: localhost
+    
+    - name: Check login node rebuild completed
+      shell:
+        cmd: openstack server show {{ inventory_hostname }} --format value -c image
+      register: openstack_login
+      delegate_to: localhost
+      retries: 5
+      delay: 30
+      until: login_build.artifact_id in openstack_login.stdout
+      changed_when: false
+    
+    - name: Wait for login connection
+      wait_for_connection:
+        timeout: 800
+    
+    - name: Check slurm up after reimaging login node
+      import_tasks: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/hooks/check_slurm.yml"
+    
+    # TODO: This is specific to smslabs/arcus environment config - could generalise to all compute nodes
+    - name: Request compute node rebuild via Slurm
+      shell:
+        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]
+      become: yes
+        
+    - name: Check compute node rebuild completed
+      shell:
+        cmd: openstack server show {{ item }} --format value -c image
+      register: openstack_compute
+      delegate_to: localhost
+      loop: "{{ groups['compute'] }}"
+      retries: 5
+      delay: 30
+      until: compute_build.artifact_id in openstack_compute.stdout
+      changed_when: false
+
+- hosts: compute:!builder
+  become: no
+  gather_facts: no
+  tasks:
+    - name: Wait for compute connection
+      wait_for_connection:
+        timeout: 800
+    
+    - name: Check slurm up after reimaging login node
+      import_tasks: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/hooks/check_slurm.yml"
+      run_once: true

ansible/roles/block_devices/README.md

@@ -11,6 +11,8 @@ This is a convenience wrapper around the ansible modules:
 
 It includes logic to handle OpenStack-provided volumes appropriately both for appliance instances and the Packer build VM.
 
+To avoid issues with device names changing after e.g. reboots, devices are identified by serial number and mounted by filesystem UUID. 
+
 Requirements
 ------------
 
@@ -20,7 +22,7 @@ Role Variables
 --------------
 
 - `block_devices_partition_state`: Optional. Partition state, 'present' or 'absent' (as for parted) or 'skip'. Defaults to 'present'.
-- `block_devices_device`: Required. Path to block device, e.g. '/dev/sda'. See `community.general.parted:device` and `community.general.filesystem:dev`.
+- `block_devices_serial`: Required. Serial number of block device. For an OpenStack volume this is the volume ID.
 - `block_devices_number`: Required. Partition number, e.g 1 for "/dev/sda1". See `community.general.parted:number`.
 - `block_devices_fstype`: Required. Filesystem type, e.g.'ext4'. See `community.general.filesystem:fstype`
 - `block_devices_resizefs`: Optional. Grow filesystem into block device space, 'yes' or 'no' (default). See `community.general.filesystem:resizefs` for applicable fileysystem types.
@@ -51,7 +53,7 @@ Example Playbook
 The example variables below create an `ext4` partition on `/dev/sdb1` and mount it as `/mnt/files` with the default owner/group:
 
 ```yaml
-block_devices_device: /dev/sdb
+block_devices_serial: a1076455-da55-4e0c-bac8-ccc4698cff97
 block_devices_number: 1
 block_devices_fstype: ext4
 block_devices_path: /mnt/files
@@ -61,25 +63,12 @@ This does the same:
 
 ```yaml
 block_devices_configurations:
-  - device: /dev/sdb
+  - serial: a1076455-da55-4e0c-bac8-ccc4698cff97
     number: 1
     fstype: ext4
     path: /mnt/files
 ```
 
-This creates 'ext4' partitions on `/dev/sdb1` on `server` and `/dev/sdc1` on `server2`, both mounted at `/mnt/files`:
-
-```yaml
-block_devices_fstype: ext4
-block_devices_path: /mnt/files
-block_devices_number: 1
-block_devices_configurations:
-  - device: /dev/sdb
-    hostnames: server1
-  - device: /dev/sdc
-    hostnames: server2
-```
-
 License
 -------
 

ansible/roles/block_devices/library/block_devices.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2021, StackHPC
+# Apache 2 License
+
+DOCUMENTATION = r'''
+---
+module: block_devices
+
+short_description: Return block device paths by serial number.
+
+options: (none)
+
+author:
+    - Steve Brasier (@sjpb)
+'''
+
+RETURN = r'''
+devices:
+    description: dict with device serial numbers as keys and full paths (e.g. /dev/sdb) as values
+    type: dict
+    return: always
+'''
+
+import json
+
+from ansible.module_utils.basic import AnsibleModule
+
+def run_module():
+    module_args = dict()
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+    result = {"changed": False}
+    _, stdout, _ = module.run_command("lsblk --paths --json -O", check_rc=True)
+    
+    device_info = json.loads(stdout)['blockdevices']
+    result['devices'] = dict((item['serial'], item['name']) for item in device_info)
+    module.exit_json(**result)
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

ansible/roles/block_devices/tasks/main.yml

@@ -1,33 +1,55 @@
+- name: Enumerate block device paths by serial number
+  block_devices:
+  register: _block_devices
+
 - name:  Create partitions
   parted:
-    device: "{{ item.get('device', block_devices_device) }}"
+    device: "{{ _device }}"
     number: "{{ item.get('number', block_devices_number) }}"
     state: "{{ item.get('partition_state', block_devices_partition_state) }}"
   when: "item.get('partition_state', block_devices_partition_state) != 'skip'"
   loop: "{{ block_devices_configurations }}"
+  vars:
+    _device: "{{ _block_devices.devices[ item.get('serial', block_devices_serial) ] }}"
 
 - name: Create filesystems
   filesystem:
     fstype: "{{ item.get('fstype', block_devices_fstype) }}"
-    dev: "{{ item.get('device', block_devices_device) }}{{ item.get('number', block_devices_number) }}"
+    dev: "{{ _device }}{{ item.get('number', block_devices_number) }}"
     resizefs: "{{ item.get('resizefs', block_devices_resizefs) }}"
     state: "{{ item.get('filesystem_state', block_devices_filesystem_state) }}"
   when: "item.get('filesystem_state', block_devices_filesystem_state) != 'skip'"
   loop: "{{ block_devices_configurations }}"
+  vars:
+    _device: "{{ _block_devices.devices[ item.get('serial', block_devices_serial) ] }}"
+
+- name: Get filesystem UUIDs
+  command:
+    cmd: "lsblk {{ _device }}{{ item.get('number', block_devices_number) }} --noheadings --output UUID"
+  loop: "{{ block_devices_configurations }}"
+  vars:
+    _device: "{{ _block_devices.devices[ item.get('serial', block_devices_serial) ] }}"
+  register: block_devices_uuids
+  changed_when: false
+  check_mode: no
 
 - name: Ensure mount point exists
   file:
     path: "{{ item.get('path', block_devices_path) }}"
     state: directory
   loop: "{{ block_devices_configurations }}"
 
-- name: Mount filesystems
+- name: Mount filesystems by UUID
   mount:
     path: "{{ item.get('path', block_devices_path) }}"
-    src: "{{ item.get('device', block_devices_device) }}{{ item.get('number', block_devices_number) }}"
+    src: "UUID={{ _uuid }}"
     fstype: "{{ item.get('fstype', block_devices_fstype) }}"
     state: "{{ item.get('mount_state', block_devices_mount_state) }}"
+  vars:
+    _uuid: "{{ block_devices_uuids.results[block_devices_idx].stdout }}"
   loop: "{{ block_devices_configurations }}"
+  loop_control:
+    index_var: block_devices_idx
 
 - name: Set owner/group for mounted directory
   file:

ansible/roles/openondemand/tasks/pam_auth.yml

@@ -21,4 +21,11 @@
     mode: 0640
     group: apache
 
+- name: Allow httpd access to PAM in SELinux
+  ansible.posix.seboolean:
+    name: httpd_mod_auth_pam
+    state: yes
+    persistent: yes
+  when: ansible_facts.selinux.status == 'enabled'
+
 # TODO: do we need to restart OOD here??