Merge branch 'main' into feat/compute-script-sb

Author: bertiethorpe

.github/workflows/doca.yml renamed to .github/workflows/extra.yml

@@ -1,4 +1,4 @@
-name: Test DOCA extra build
+name: Test extra build
 on:
   workflow_dispatch:
   push:
@@ -7,16 +7,18 @@ on:
     paths:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
-      - '.github/workflows/doca'
+      - 'ansible/roles/cuda/**'
+      - '.github/workflows/extra.yml'
   pull_request:
     paths:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
-      - '.github/workflows/doca'
+      - 'ansible/roles/cuda/**'
+      - '.github/workflows/extra.yml'
 
 jobs:
   doca:
-    name: doca-build
+    name: extra-build
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
       cancel-in-progress: true
@@ -25,12 +27,14 @@ jobs:
       fail-fast: false # allow other matrix jobs to continue even if one fails
       matrix: # build RL8, RL9
         build:
-          - image_name: openhpc-doca-RL8
+          - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
-            inventory_groups: doca
-          - image_name: openhpc-doca-RL9
+            inventory_groups: doca,cuda
+            volume_size: 30 # needed for cuda
+          - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
-            inventory_groups: doca
+            inventory_groups: doca,cuda
+            volume_size: 30 # needed for cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -95,6 +99,7 @@ jobs:
           -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
           -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
+          -var "volume_size=${{ matrix.build.volume_size }}" \
           openstack.pkr.hcl
         
       - name: Get created image names from manifest

.github/workflows/fatimage.yml

@@ -23,11 +23,11 @@ jobs:
       matrix: # build RL8, RL9
         build:
           - image_name: openhpc-RL8
-            source_image_name: rocky-latest-RL8
-            inventory_groups: control,compute,login
+            source_image_name: Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
           - image_name: openhpc-RL9
-            source_image_name: rocky-latest-RL9
-            inventory_groups: control,compute,login
+            source_image_name: Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack

ansible/cleanup.yml

@@ -66,5 +66,4 @@
       slurm-ohpc: "{{ ansible_facts.packages['slurm-ohpc'].0.version | default('-') }}"
 
 - name: Show image summary
-  debug:
-    var: image_info
+  command: cat /var/lib/image/image.json

ansible/extras.yml

@@ -24,8 +24,9 @@
   gather_facts: yes
   tags: cuda
   tasks:
-    - import_role:
+    - include_role:
         name: cuda
+        tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 
 - name: Persist hostkeys across rebuilds
   # Must be after filesystems.yml (for storage)
@@ -56,3 +57,13 @@
   tasks:
   - import_role:
       name: k9s
+
+- hosts: extra_packages
+  become: yes
+  tags:
+   - extra_packages
+  tasks:
+  - name: Install additional packages
+    dnf:
+      name: "{{ appliances_extra_packages }}"
+    when: appliances_mode != 'configure' or appliances_extra_packages_during_configure

ansible/fatimage.yml

@@ -29,6 +29,14 @@
 
 - import_playbook: bootstrap.yml
 
+- hosts: doca
+  become: yes
+  gather_facts: yes
+  tasks:
+    - name: Install NVIDIA DOCA
+      import_role:
+        name: doca
+
 - name: Run post-bootstrap.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
@@ -230,15 +238,15 @@
       import_role:
         name: doca
 
-- import_playbook: disable-repos.yml
-
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
     hook_path: "{{ appliances_environment_root }}/hooks/post.yml"
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- import_playbook: disable-repos.yml
+
 - hosts: builder
   become: yes
   gather_facts: yes

ansible/roles/cuda/README.md

@@ -1,15 +1,15 @@
 # cuda
 
-Install NVIDIA CUDA. The CUDA binaries are added to the PATH for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
+Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
 ## Prerequisites
 
 Requires OFED to be installed to provide required kernel-* packages.
 
 ## Role Variables
 
-- `cuda_distro`: Optional. Default `rhel8`.
-- `cuda_repo`: Optional. Default `https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo`
-- `cuda_driver_stream`: Optional. The default value `default` will, on first use of this role, enable the dkms-flavour `nvidia-driver` DNF module stream with the current highest version number. The `latest-dkms` stream is not enabled, and subsequent runs of the role will *not* change the enabled stream, even if a later version has become available. Changing this value once an `nvidia-driver` stream has been enabled raises an error. If an upgrade of the `nvidia-driver` module is required, the currently-enabled stream and all packages should be manually removed.
+- `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
+- `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
 - `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.
+- `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,7 +1,6 @@
-cuda_distro: "rhel{{ ansible_distribution_major_version }}"
-cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
-cuda_driver_stream: default
-cuda_package_version: 'latest'
+cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+cuda_nvidia_driver_stream: '560-open' # 565-open has problems with cuda packages
+cuda_package_version: '12.6.3-1'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds

ansible/roles/cuda/tasks/main.yml renamed to ansible/roles/cuda/tasks/install.yml

@@ -1,7 +1,7 @@
 
 # Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
 
-- name: Check for OFED
+- name: Check for OFED/DOCA
   command:
     cmd: dnf list --installed rdma-core
   register: _dnf_rdma_core
@@ -10,41 +10,53 @@
 - name: Assert OFED installed
   assert: 
     that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED installed?"
+    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
 
 - name: Install cuda repo
   get_url:
-    dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
-    url: "{{ cuda_repo }}"
+    dest: "/etc/yum.repos.d/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+    url: "{{ cuda_repo_url }}"
 
 - name: Check if nvidia driver module is enabled
-  shell:
-    cmd: dnf module list --enabled nvidia-driver
+  ansible.builtin.command: dnf module list --enabled nvidia-driver
   changed_when: false
   failed_when: false
   register: _cuda_driver_module_enabled
 
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:open-dkms"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ cuda_nvidia_driver_stream }}"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
+- name: Check if nvidia driver module is installed
+  ansible.builtin.command: dnf module list --installed nvidia-driver
+  changed_when: false
+  failed_when: false
+  register: _cuda_driver_module_installed
+
 - name: Install nvidia drivers
   ansible.builtin.command: dnf module install -y nvidia-driver
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
+- name: Check kernel has not been modified
+  assert:
+    that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
+    fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"
+  when: cuda_package_version != 'none'
   register: cuda_package_install
 
 - name: Add cuda binaries to path
   lineinfile:
     path: /etc/profile.d/sh.local
     line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'
+  when: cuda_package_version != 'none'
 
 - name: Enable NVIDIA Persistence Daemon
   systemd:
@@ -60,3 +72,4 @@
 - name: Wait for hosts to be reachable
   wait_for_connection:
     sleep: 15
+  when: cuda_package_install.changed

ansible/roles/cuda/tasks/runtime.yml

@@ -0,0 +1,5 @@
+- name: Ensure NVIDIA Persistence Daemon state
+  systemd:
+    name: nvidia-persistenced
+    enabled: true
+    state: "{{ cuda_persistenced_state }}"

docs/operations.md

@@ -63,17 +63,30 @@ This is a usually a two-step process:
 Deploying the additional nodes and applying these changes requires rerunning both Terraform and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster).
 
 # Adding Additional Packages
-Packages from any enabled DNF repositories (which always includes EPEL, PowerTools and OpenHPC) can be added to all nodes by defining a list `openhpc_packages_extra` (defaulted to the empty list in the common environment) in e.g. `environments/$SITE_ENV/inventory/group_vars/all/openhpc.yml`. For example:
-
-    # environments/foo-base/inventory/group_vars/all/openhpc.yml:
-    openhpc_packages_extra:
+By default, the following utility packages are installed during build:
+- htop
+- nano
+- screen
+- tmux
+- wget
+- bind-utils
+- net-tools
+- postfix
+- git
+- latest python version for system (3.6 for for Rocky 8.9 and 3.12 for Rocky 9.4)
+
+Additional packages from any DNF repositories which are enabled during build (which always includes EPEL, PowerTools and OpenHPC) can be added to the image by defining a list `appliances_extra_packages_other` (defaulted to the empty list in the common environment) in e.g. `environments/$SITE_ENV/inventory/group_vars/all/defaults.yml`. For example:
+
+```yaml
+    # environments/foo-base/inventory/group_vars/all/defaults.yml:
+    appliances_extra_packages_other:
     - somepackage
     - anotherpackage
 
 
 The packages available from the OpenHPC repos are described in Appendix E of the OpenHPC installation guide (linked from the [OpenHPC releases page](https://github.com/openhpc/ohpc/releases/)). Note "user-facing" OpenHPC packages such as compilers, mpi libraries etc. include corresponding `lmod` modules.
 
-To add these packages to the current cluster, run the same command as for [Reconfiguring Slurm](#Reconfiguring-Slurm). TODO: describe what's required to add these to site-specific images.
+If you wish to install packages during runtime, the `site.yml` playbook should be run with `appliances_packages_during_configure` overriden to `true` and `cluster` should be added as a child of the `dnf_repos` group in order to temporarily re-enable DNF repositories during runtime (WARNING: this should only be done if using an unauthenticated local Pulp server. If using StackHPC Ark directly, doing this WILL leak credentials to users).
 
 If additional repositories are required, these could be added/enabled as necessary in a play added to `environments/$SITE_ENV/hooks/{pre,post}.yml` as appropriate. Note such a plat should NOT exclude the builder group, so that the repositories are also added to built images. There are various Ansible modules which might be useful for this:
     - `ansible.builtin.yum_repository`: Add a repo from an URL providing a 'repodata' directory.

environments/.stackhpc/terraform/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-241216-1607-2357a730",
-        "RL9": "openhpc-RL9-241216-1607-2357a730"
+        "RL8": "openhpc-RL8-241219-1232-7f84fed4",
+        "RL9": "openhpc-RL9-241219-1145-7f84fed4"
     }
 }

environments/common/inventory/group_vars/all/defaults.yml

@@ -80,16 +80,37 @@ appliances_local_users_default:
 appliances_local_users_extra: [] # see format of appliances_local_users_default above
 appliances_local_users: "{{ appliances_local_users_default + appliances_local_users_extra }}"
 
-###########################################################################################
+################## bootstrap: extra package installs ######################################
+
+appliances_extra_packages_default:
+ - htop
+ - nano
+ - screen
+ - tmux
+ - wget
+ - bind-utils
+ - net-tools
+ - postfix
+ - git
+ - "{{ 'python36' if ansible_distribution_version == '8.9' else 'python312' }}"
+ 
+
+appliances_extra_packages_during_configure: false
+
+appliances_extra_packages_other: []
+  
+appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}"
+
+###################### ark repo timestamps ###################################################
 
 appliances_repo_timestamps:
   baseos:
-    '9.4': 20240816T002610
+    '9.4': 20241115T011711
   appstream:
-    '9.4': 20240816T002610
+    '9.4': 20241112T003151
   crb:
-    '9.4': 20240816T002610
+    '9.4': 20241115T003133
   extras:
-    '9.4': 20240816T002610
+    '9.4': 20241118T002802
   epel:
-    '9': 20240902T080424
+    '9': 20241213T010218

environments/common/inventory/groups

@@ -148,6 +148,9 @@ freeipa_client
 [lustre]
 # Hosts to run lustre client
 
+[extra_packages]
+# Hosts to install specified additional packages on
+
 [dnf_repos:children]
 # Hosts to replace system repos with Pulp repos
 # Warning: when using Ark directly rather than a local Pulp server, adding hosts other than `builder` will leak Ark creds to users

environments/common/layouts/everything

@@ -96,3 +96,7 @@ control
 
 [lustre]
 # Hosts to run lustre client
+
+[extra_packages:children]
+# Hosts to install specified additional packages on
+cluster