Add support for OFED

.github/workflows/fatimage.yml

@@ -61,18 +61,19 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
         env:
           PKR_VAR_os_version: ${{ matrix.os_version }}
 
-      - name: Get created image name from manifest
+      - name: Get created image names from manifest
         id: manifest
         run: |
           . venv/bin/activate
-          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
-          while ! openstack image show -f value -c name $IMAGE_ID; do
-            sleep 30
+          for IMAGE_ID in $(jq --raw-output '.builds[].artifact_id' packer/packer-manifest.json)
+          do
+            while ! openstack image show -f value -c name $IMAGE_ID; do
+              sleep 5
+            done
+            IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+            echo $IMAGE_NAME
           done
-          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
-          echo "IMAGE_ID=${IMAGE_ID}" >> "$GITHUB_OUTPUT"
-          echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"

ansible/.gitignore

@@ -52,4 +52,5 @@ roles/*
 !roles/image_build/**
 !roles/persist_hostkeys/
 !roles/persist_hostkeys/**
-!roles/requirements.yml
+!roles/ofed/
+!roles/ofed/**

ansible/bootstrap.yml

@@ -196,3 +196,11 @@
     - name: update facts
       setup:
       when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))
+
+- hosts: ofed
+  gather_facts: no
+  become: yes
+  tags: ofed
+  tasks:
+    - include_role:
+        name: ofed

ansible/roles/ofed/README.md

@@ -0,0 +1,21 @@
+# ofed
+
+This role installs Mellanox OFED:
+- It checks that the running kernel is the latest installed one, and errors if not.
+- Installation uses the `mlnxofedinstall` command, with support for the running kernel
+and (by default) without firmware updates.
+
+As OFED installation takes a long time generally this should only be used during image build,
+for example by setting:
+
+```
+environments/groups/<environment>/groups:
+[ofed:children]
+builder
+```
+
+# Role variables
+
+See `defaults/main.yml`
+
+Note ansible facts are required, unless setting `ofed_distro_version` and `ofed_arch` specifically.

ansible/roles/ofed/defaults/main.yml

@@ -0,0 +1,28 @@
+ofed_version: 24.01-0.3.3.1
+ofed_download_url: https://content.mellanox.com/ofed/MLNX_OFED-{{ ofed_version }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}.tgz
+ofed_distro: rhel # NB: not expected to work on other distros due to installation differences
+ofed_distro_version: "{{ ansible_distribution_version }}" # e.g. '8.9'
+ofed_arch: "{{ ansible_architecture }}"
+ofed_tmp_dir: /tmp
+ofed_update_firmware: false
+ofed_build_packages: # may require additional packages depending on ofed_package_selection
+  - autoconf
+  - automake
+  - gcc
+  - gcc-gfortran
+  - kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}
+  - kernel-rpm-macros
+  - libtool
+  - lsof
+  - patch
+  - pciutils
+  - perl
+  - rpm-build
+  - tcl
+  - tk
+ofed_build_rl8_packages:
+  - gdb-headless
+  - python36
+ofed_package_selection: # list of package selection flags for mlnxofedinstall script
+  - hpc
+  - with-nfsrdma

ansible/roles/ofed/tasks/install.yml

@@ -0,0 +1,73 @@
+- name: Get installed kernels
+  command: dnf list --installed kernel
+  register: _ofed_dnf_kernels
+  changed_when: false
+
+- name: Determine running kernel
+  command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
+  register: _ofed_loaded_kernel
+  changed_when: false
+
+- name: Check current kernel is newest installed
+  assert:
+    that: _ofed_loaded_kernel.stdout == _ofed_dnf_kernels_newest
+    fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
+  vars:
+    _ofed_dnf_kernels_newest: >-
+      {{ _ofed_dnf_kernels.stdout_lines[1:] | map('regex_replace', '^\w+\.(\w+)\s+(\S+)\s+\S+\s*$', '\2.\1') | community.general.version_sort | last }}
+    # dnf line format e.g. "kernel.x86_64  4.18.0-513.18.1.el8_9   @baseos  "
+
+- name: Enable epel
+  dnf:
+    name: epel-release
+
+- name: Check for existing OFED installation
+  command: ofed_info
+  changed_when: false
+  failed_when:
+    - _ofed_info.rc > 0
+    - "'No such file or directory' not in _ofed_info.msg"
+  register: _ofed_info
+
+- name: Install build prerequisites
+  dnf:
+    name: "{{ ofed_build_packages + (ofed_build_rl8_packages if ofed_distro_version == '8.9' else []) }}"
+  when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
+  # don't want to install a load of prereqs unnecessarily
+
+- name: Download and unpack Mellanox OFED tarball
+  ansible.builtin.unarchive:
+    src: "{{ ofed_download_url }}"
+    dest: "{{ ofed_tmp_dir }}"
+    remote_src: yes
+  become: no
+  when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
+
+# Below from https://docs.nvidia.com/networking/display/mlnxofedv24010331/user+manual
+- name: Run OFED install script
+  command:
+    cmd: >
+      ./mlnxofedinstall
+      --add-kernel-support
+      {% if not ofed_update_firmware %}--without-fw-update{% endif %}
+      --force
+      --skip-repo
+      {% for pkgsel in ofed_package_selection %}
+      --{{ pkgsel }}
+      {% endfor %}
+    chdir: "{{ ofed_tmp_dir }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}/"
+  register: _ofed_install
+  when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
+  async: "{{ 45 * 60 }}" # wait for up to 45 minutes
+  poll: 15 # check every 15 seconds
+
+- name: Update initramfs
+  command:
+    cmd: dracut -f
+  when: '"update your initramfs" in _ofed_install.stdout | default("")'
+  failed_when: false # always shows errors due to deleted modules for inbox RDMA drivers
+
+- name: Load the new driver
+  command:
+    cmd: /etc/init.d/openibd restart
+  when: '"To load the new driver" in _ofed_install.stdout | default("")'

ansible/roles/ofed/tasks/main.yml

@@ -0,0 +1 @@
+- include_tasks: install.yml

ansible/roles/openondemand/tasks/vnc_compute.yml

@@ -10,6 +10,22 @@
   yum:
     name: epel-release
 
+- name: Check /etc/init.d
+  ansible.builtin.stat:
+    path: /etc/init.d
+  register: init_d
+
+- name: Move OFED-installed init scripts
+  # turbovnc installs chkconfig which symlinks /etc/init.d from /etc/rc.d/init.d
+  # but OFED has already created that and installed files in it.
+  # See https://access.redhat.com/solutions/6969215
+  ansible.builtin.command:
+    cmd: mv /etc/init.d /etc/init.d.orig
+    creates: /etc/init.d.orig
+  when:
+    - init_d.stat.exists
+    - not init_d.stat.islnk
+
 - name: Install VNC-related packages
   tags: install
   dnf:
@@ -19,6 +35,15 @@
       - python3.9
       - dbus-x11
 
+- name: Replace OFED-installed init scripts
+  ansible.builtin.copy:
+    src: /etc/init.d.orig/ # trailing / to get contents
+    dest: /etc/init.d
+    remote_src: true
+  when:
+    - init_d.stat.exists
+    - not init_d.stat.islnk
+
 - name: Install Xfce desktop
   tags: install
   yum:

environments/.stackhpc/LEAFCLOUD.pkrvars.hcl

@@ -1,6 +1,7 @@
-flavor = "en1.xsmall"
+flavor = "ec1.medium"
 use_blockstorage_volume = true
 volume_size = 12 # GB. Compatible with SMS-lab's general.v1.tiny
+volume_size_ofed = 15 # GB
 volume_type = "unencrypted"
 image_disk_format = "qcow2"
 networks = ["909e49e8-6911-473a-bf88-0495ca63853c"] # slurmapp-ci

environments/.stackhpc/terraform/main.tf

@@ -28,9 +28,9 @@ variable "cluster_image" {
     description = "single image for all cluster nodes, keyed by os_version - a convenience for CI"
     type = map(string)
     default = {
-        # https://github.com/stackhpc/ansible-slurm-appliance/pull/382
-        RL8: "openhpc-RL8-240327-1050-4812f852"
-        RL9: "openhpc-RL9-240327-1026-4812f852"
+        # https://github.com/stackhpc/ansible-slurm-appliance/pull/353
+        RL8: "openhpc-RL8-240423-1002-4b09ba85"
+        RL9: "openhpc-ofed-RL9-240423-1059-4b09ba85"
     }
 }
 

packer/openstack.pkr.hcl

@@ -130,6 +130,11 @@ variable "volume_size" {
   default = null # When not specified use the size of the builder instance root disk
 }
 
+variable "volume_size_ofed" {
+  type = number
+  default = null # When not specified use the size of the builder instance root disk
+}
+
 variable "image_disk_format" {
   type = string
   default = null # When not specified use the image default
@@ -141,39 +146,55 @@ variable "metadata" {
 }
 
 source "openstack" "openhpc" {
-  flavor = "${var.flavor}"
-  volume_size = "${var.volume_size}"
-  use_blockstorage_volume = "${var.use_blockstorage_volume}"
+  # Build VM:
+  flavor = var.flavor
+  use_blockstorage_volume = var.use_blockstorage_volume
   volume_type = var.volume_type
-  image_disk_format = "${var.image_disk_format}"
-  metadata = "${var.metadata}"
-  networks = "${var.networks}"
-  ssh_username = "${var.ssh_username}"
+  metadata = var.metadata
+  networks = var.networks
+  floating_ip_network = var.floating_ip_network
+  security_groups = var.security_groups
+
+  # Input image:
+  source_image = "${var.fatimage_source_image[var.os_version]}"
+  source_image_name = "${var.fatimage_source_image_name[var.os_version]}" # NB: must already exist in OpenStack
+
+  # SSH:
+  ssh_username = var.ssh_username
   ssh_timeout = "20m"
-  ssh_private_key_file = "${var.ssh_private_key_file}" # TODO: doc same requirements as for qemu build?
-  ssh_keypair_name = "${var.ssh_keypair_name}" # TODO: doc this
-  ssh_bastion_host = "${var.ssh_bastion_host}"
-  ssh_bastion_username = "${var.ssh_bastion_username}"
-  ssh_bastion_private_key_file = "${var.ssh_bastion_private_key_file}"
-  security_groups = "${var.security_groups}"
-  image_visibility = "${var.image_visibility}"
-}
-
-# The "fat" image build with all binaries:
+  ssh_private_key_file = var.ssh_private_key_file
+  ssh_keypair_name = var.ssh_keypair_name # TODO: doc this
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
+
+  # Output image:
+  image_disk_format = var.image_disk_format
+  image_visibility = var.image_visibility
+  image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
+}
+
+# "fat" image builds:
 build {
+
+  # non-OFED:
+  source "source.openstack.openhpc" {
+    name = "openhpc"
+    volume_size = var.volume_size
+  }
+
+  # OFED:
   source "source.openstack.openhpc" {
-    floating_ip_network = "${var.floating_ip_network}"
-    source_image = "${var.fatimage_source_image[var.os_version]}"
-    source_image_name = "${var.fatimage_source_image_name[var.os_version]}" # NB: must already exist in OpenStack
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}" # similar to name from slurm_image_builder
+    name = "openhpc-ofed"
+    volume_size = var.volume_size_ofed
   }
 
   provisioner "ansible" {
     playbook_file = "${var.repo_root}/ansible/fatimage.yml"
-    groups = ["builder", "control", "compute", "login"]
+    groups = concat(["builder", "control", "compute", "login"], [for g in split("-", "${source.name}"): g if g != "openhpc"])
     keep_inventory_file = true # for debugging
     use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting
-    extra_arguments = ["--limit", "builder", "-i", "${var.repo_root}/packer/ansible-inventory.sh", "-vv", "-e", "@${var.repo_root}/packer/${source.name}_extravars.yml"]
+    extra_arguments = ["--limit", "builder", "-i", "${var.repo_root}/packer/ansible-inventory.sh", "-vv", "-e", "@${var.repo_root}/packer/openhpc_extravars.yml"]
   }
 
   post-processor "manifest" {