Merge branch 'main' into block_device_ids

Author: sjpb

.github/workflows/smslabs.yml

@@ -0,0 +1,139 @@
+
+name: Test on SMS-Labs OpenStack in stackhpc-ci
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+concurrency: stackhpc-ci # openstack project
+jobs:
+  openstack-example:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "$SSH_KEY" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        env:
+          SSH_KEY: ${{ secrets.SSH_KEY }}
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat environments/smslabs/bastion_fingerprint >> ~/.ssh/known_hosts
+        shell: bash
+      
+      - name: Install ansible etc
+        run: dev/setup-env.sh
+
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v1
+      
+      - name: Initialise terraform
+        run: terraform init
+        working-directory: ${{ github.workspace }}/environments/smslabs/terraform
+        
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "$CLOUDS_YAML" > ~/.config/openstack/clouds.yaml
+        shell: bash
+        env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
+      
+      - name: Provision infrastructure
+        id: provision
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform apply -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+      
+      - name: Get server provisioning failure messages
+        id: provision_failure
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          echo "::set-output name=messages::$(./getfaults.py)"
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: always() && steps.provision.outcome == 'failure'
+        
+      - name: Delete infrastructure if failed due to lack of hosts
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform destroy -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+
+      - name: Configure infrastructure
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          ansible all -m wait_for_connection
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          ansible-playbook -vv ansible/site.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+      
+      - name: Run MPI-based tests
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+      
+      - name: Build control and compute images
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          cd packer
+          PACKER_LOG=1 PACKER_LOG_PATH=build.log packer build -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+        env:
+          OS_CLOUD: openstack
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+      
+      - name: Reimage compute nodes via slurm and check cluster still up
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-compute.yml
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+        env:
+          OS_CLOUD: openstack
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Reimage login nodes via openstack and check cluster still up
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-login.yml
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+        env:
+          OS_CLOUD: openstack
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Delete infrastructure
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform destroy -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: ${{ success() || cancelled() }}

.github/workflows/vagrant.yml

@@ -1,7 +1,9 @@
+[![Test on OpenStack via smslabs](https://github.com/stackhpc/ansible-slurm-appliance/actions/workflows/smslabs.yml/badge.svg)](https://github.com/stackhpc/ansible-slurm-appliance/actions/workflows/smslabs.yml)
+
 # StackHPC Slurm Appliance
 
 This repository contains playbooks and configuration to define a Slurm-based HPC environment including:
-- A Centos 8 and OpenHPC v2-based Slurm cluster.
+- A Rocky Linux 8 and OpenHPC v2-based Slurm cluster.
 - Shared fileystem(s) using NFS (with servers within or external to the cluster).
 - Slurm accounting using a MySQL backend.
 - A monitoring backend using Prometheus and ElasticSearch.
@@ -16,15 +18,15 @@ While it is tested on OpenStack it should work on any cloud, except for node reb
 ## Prerequisites
 It is recommended to check the following before starting:
 - You have root access on the "ansible deploy host" which will be used to deploy the appliance.
-- You can create instances using a CentOS 8 GenericCloud image (or an image based on that).
+- You can create instances using a Rocky 8 GenericCloud image (or an image based on that).
 - SSH keys get correctly injected into instances.
 - Instances have access to internet (note proxies can be setup through the appliance if necessary).
 - DNS works (if not this can be partially worked around but additional configuration will be required).
 - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance).
 
 ## Installation on deployment host
 
-These instructions assume the deployment host is running Centos 8:
+These instructions assume the deployment host is running Centos/Rocky 8:
 
     sudo yum install -y git python3
     git clone https://github.com/stackhpc/ansible-slurm-appliance

README.md

@@ -8,8 +8,6 @@ roles/*
 !roles/opendistro/**
 !roles/podman/
 !roles/podman/**
-!roles/kibana/
-!roles/kibana/**
 !roles/grafana-dashboards/
 !roles/grafana-dashboards/**
 !roles/grafana-datasources/
@@ -22,3 +20,5 @@ roles/*
 !roles/block_devices/**
 !roles/basic_users/
 !roles/basic_users/**
+!roles/openondemand/
+!roles/openondemand/**

ansible/.gitignore

@@ -49,6 +49,8 @@
           state: "{{ update_state }}"
           exclude: "{{ update_exclude }}"
           disablerepo: "{{ update_disablerepo }}"
+        async: "{{ 30 * 60 }}" # wait for up to 30 minutes
+        poll: 15 # check every 15 seconds
         register: updates
       - debug:
           var: updates

ansible/bootstrap.yml

@@ -9,22 +9,10 @@
 from ansible.module_utils.six import string_types
 import os.path
 
-def _get_hostvar(context, var_name, inventory_hostname=None):
-    if inventory_hostname is None:
-        namespace = context
-    else:
-        if inventory_hostname not in context['hostvars']:
-            raise AnsibleFilterError(
-                "Inventory hostname '%s' not in hostvars" % inventory_hostname)
-        namespace = context["hostvars"][inventory_hostname]
-    return namespace.get(var_name)
-
-@jinja2.contextfilter
-def prometheus_node_exporter_targets(context, hosts):
+def prometheus_node_exporter_targets(hosts, env):
     result = []
     per_env = defaultdict(list)
     for host in hosts:
-        env = _get_hostvar(context, "env", host) or "ungrouped"
         per_env[env].append(host)
     for env, hosts in per_env.items():
         target = {

ansible/filter_plugins/utils.py

@@ -1,5 +1,7 @@
 - hosts: basic_users
   become: yes
+  tags:
+    - basic_users
   gather_facts: yes
   tasks:
     - import_role:

ansible/iam.yml

@@ -29,25 +29,6 @@
         tasks_from: deploy.yml
       tags: deploy
 
-- name: Setup kibana
-  hosts: kibana
-  tags: kibana
-  tasks:
-    - import_role:
-        name: kibana
-        tasks_from: config.yml
-      tags: config
-
-    - import_role:
-        name: kibana
-        tasks_from: deploy.yml
-      tags: deploy
-
-    - import_role:
-        name: kibana
-        tasks_from: post.yml
-      tags: post
-
 - name: Setup slurm stats
   hosts: slurm_stats
   tags: slurm_stats
@@ -80,6 +61,17 @@
   tasks:
     - import_role: name=cloudalchemy.node_exporter
 
+- name: Deploy OpenOndemand exporter
+  hosts: openondemand
+  become: true
+  tags:
+    - openondemand
+    - openondemand_server
+  tasks:
+    - import_role:
+        name: openondemand
+        tasks_from: exporter.yml
+
 - name: Setup core monitoring software
   hosts: prometheus
   tags: prometheus

ansible/monitoring.yml

@@ -0,0 +1,32 @@
+- hosts: openondemand
+  tags:
+    - openondemand
+    - openondemand_server
+  become: yes
+  gather_facts: yes # TODO
+  tasks:
+    - import_role:
+        name: openondemand
+        tasks_from: main.yml
+
+- hosts: openondemand_desktop
+  tags:
+    - openondemand
+    - openondemand_desktop
+  become: yes
+  gather_facts: yes
+  tasks:
+    - import_role:
+        name: openondemand
+        tasks_from: vnc_compute.yml
+
+- hosts: openondemand_jupyter
+  tags:
+    - openondemand
+    - openondemand_jupyter
+  become: yes
+  gather_facts: yes
+  tasks:
+    - import_role:
+        name: openondemand
+        tasks_from: jupyter_compute.yml

ansible/portal.yml

@@ -16,10 +16,12 @@ Requirements
 Role Variables
 --------------
 
-`basic_users_users`: Required. A list of mappings defining information for each user. In general, mapping keys/values are the parameters to [ansible.builtin.user](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/user_module.html) and default values are as given there. However:
+`basic_users_users`: Required. A list of mappings defining information for each user. In general, mapping keys/values are passed through as parameters to [ansible.builtin.user](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/user_module.html) and default values are as given there. However:
 - `create_home`, `generate_ssh_key` and `ssh_key_comment` are set automatically and should not be overriden.
+- `uid` should be set, so that the UID/GID is consistent across the cluster (which Slurm requires).
 - `shell` may be set if required, but will be overriden with `/sbin/nologin` on `control` nodes to prevent user login.
 - An additional key `public_key` may optionally be specified to define a key to log into the cluster.
+- Any other keys may present for other purposes (i.e. not used by this role).
 
 Dependencies
 ------------

ansible/roles/basic_users/README.md

@@ -2,18 +2,23 @@
 
 import copy
 
+USER_MODULE_PARAMS = ('append authorization comment create_home createhome expires force generate_ssh_key group '
+                      'groups hidden home local login_class move_home name user non_unique password password_expire_min '
+                      'password_expire_max password_lock profile remove role seuser shell skeleton ssh_key_bits '
+                      'ssh_key_comment ssh_key_file ssh_key_passphrase ssh_key_type state system uid update_password').split()
 
 class FilterModule(object):
 
     def filters(self):
         return {
-            'filter_keys': self.filter_keys
+            'filter_user_params': self.filter_user_params
         }
 
-    def filter_keys(self, orig_dict, keys_to_remove):
-        ''' Return a copy of `orig_dict` without the keys in the list `keys_to_remove`'''
-        dict_to_return = copy.deepcopy(orig_dict)
-        for item in keys_to_remove:
-            if item in dict_to_return:
-                del dict_to_return[item]
-        return dict_to_return
+    def filter_user_params(self, d):
+        ''' Return a copy of dict `d` containing only keys which are parameters for the user module'''
+        
+        user_dict = copy.deepcopy(d)
+        remove_keys = set(user_dict).difference(USER_MODULE_PARAMS)
+        for key in remove_keys:
+            del user_dict[key]
+        return user_dict

ansible/roles/basic_users/filter_plugins/filter_keys.py

@@ -9,7 +9,7 @@
     - "item.state | default('present') == 'absent'"
 
 - name: Create users and generate public keys
-  user: "{{ basic_users_userdefaults | combine(item) | filter_keys(['public_key']) }}"
+  user: "{{ basic_users_userdefaults | combine(item) | filter_user_params() }}"
   loop: "{{ basic_users_users }}"
   loop_control:
     label: "{{ item.name }} [{{ item.state | default('present') }}]"

ansible/roles/basic_users/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
 
-# You must define this variable.
-#filebeat_config_path: undefined
-filebeat_podman_user: "{{ ansible_user }}"
+#filebeat_config_path: undefined # REQUIRED. Path to filebeat.yml configuration file template
+filebeat_podman_user: "{{ ansible_user }}" # User that runs the filebeat container