Merge pull request #315 from stackhpc/fix/nvidia-driver-install

Make nvidia-driver install idempotent

Author: sjpb

ansible/roles/cuda/README.md

@@ -10,5 +10,6 @@ Requires OFED to be installed to provide required kernel-* packages.
 
 - `cuda_distro`: Optional. Default `rhel8`.
 - `cuda_repo`: Optional. Default `https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo`
+- `cuda_driver_stream`: Optional. The default value `default` will, on first use of this role, enable the dkms-flavour `nvidia-driver` DNF module stream with the current highest version number. The `latest-dkms` stream is not enabled, and subsequent runs of the role will *not* change the enabled stream, even if a later version has become available. Changing this value once an `nvidia-driver` stream has been enabled raises an error. If an upgrade of the `nvidia-driver` module is required, the currently-enabled stream and all packages should be manually removed.
 - `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,10 +1,11 @@
 cuda_distro: rhel8
 cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
+cuda_driver_stream: default
 cuda_packages:
   - cuda
   - nvidia-gds
-# cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
-cuda_version_short: "{{ cuda_version_tuple[0] }}.{{ cuda_version_tuple[1] }}"
+# _cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
+cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ cuda_version_tuple[1] }}"
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/home/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/main.yml

@@ -17,22 +17,40 @@
     dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
     url: "{{ cuda_repo }}"
 
+- name: Check if nvidia driver module is enabled
+  shell:
+    cmd: dnf module list --enabled nvidia-driver
+  changed_when: false
+  failed_when: false
+  register: _cuda_driver_module_enabled
+
+- name: List nvidia driver dnf module stream versions
+  shell:
+    cmd: dnf module list nvidia-driver | grep -oP "\d+-dkms" | sort -V
+  # Output of interest from command is something like (some whitespace removed):
+  # "nvidia-driver 418-dkms   default [d], fm, ks    Nvidia driver for 418-dkms branch "
+  changed_when: false
+  register: _cuda_driver_module_streams
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+
 - name: Enable nvidia driver module
-  ansible.builtin.command: dnf module enable -y nvidia-driver:latest-dkms
-  register: nvidiadriver_enable
-  changed_when: "'Nothing to do' not in nvidiadriver_enable.stdout"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ _cuda_driver_module_streams.stdout_lines | last }}"
+  register: _cuda_driver_module_enable
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Install nvidia driver module
-  ansible.builtin.command: dnf module install -y nvidia-driver:latest-dkms
-  register: nvidiadriver_install
-  changed_when: "'Nothing to do' not in nvidiadriver_install.stdout"
+- name: Install nvidia drivers # TODO: make removal possible?
+  ansible.builtin.command: dnf module install -y nvidia-driver
+  register: _cuda_driver_install
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"
   register: cuda_package_install
 
-- name: Add latest cuda binaries to path
+- name: Add cuda binaries to path
   lineinfile:
     path: /etc/profile.d/sh.local
     line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'

ansible/roles/cuda/tasks/samples.yml

@@ -5,7 +5,7 @@
 
 - name: Set fact for discovered cuda version
   set_fact:
-    cuda_version_tuple: "{{ (_cuda_samples_version.content | b64decode | from_json).cuda.version | split('.') }}" # e.g. '12.1.0'
+    _cuda_version_tuple: "{{ (_cuda_samples_version.content | b64decode | from_json).cuda.version | split('.') }}" # e.g. '12.1.0'
 
 - name: Ensure cuda_samples_path exists
   file: