make dnf module install of nvidia-driver idempotent

Author: sjpb

ansible/roles/cuda/README.md

@@ -2,6 +2,8 @@
 
 Install NVIDIA CUDA. The CUDA binaries are added to the PATH for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
+To avoid unwanted package updates which break functionality, on first use this role enables the dkms-flavour `nvidia-driver` DNF module stream which has the current highest version number. The `latest-dkms` stream is not enabled, and subsequent runs of the role will *not* switch which stream is enabled even if later version streams become available. If an upgrade of the `nvidia-driver` module is required, the currently-enabled stream and all packages should be manually removed.
+
 ## Prerequisites
 
 Requires OFED to be installed to provide required kernel-* packages.

ansible/roles/cuda/tasks/main.yml

@@ -17,22 +17,40 @@
     dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
     url: "{{ cuda_repo }}"
 
+- name: Check if nvidia driver module is enabled
+  shell:
+    cmd: dnf module list --enabled nvidia-driver
+  changed_when: false
+  failed_when: false
+  register: _cuda_driver_module_enabled
+
+- name: List nvidia driver dnf module stream versions
+  shell:
+    cmd: dnf module list nvidia-driver | grep -oP "\d+-dkms" | sort -V
+  # Output of interest from command is something like (some whitespace removed):
+  # "nvidia-driver 418-dkms   default [d], fm, ks    Nvidia driver for 418-dkms branch "
+  changed_when: false
+  register: _cuda_driver_module_streams
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+
 - name: Enable nvidia driver module
-  ansible.builtin.command: dnf module enable -y nvidia-driver:latest-dkms
-  register: nvidiadriver_enable
-  changed_when: "'Nothing to do' not in nvidiadriver_enable.stdout"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ _cuda_driver_module_streams.stdout_lines | last }}"
+  register: _cuda_driver_module_enable
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Install nvidia driver module
-  ansible.builtin.command: dnf module install -y nvidia-driver:latest-dkms
-  register: nvidiadriver_install
-  changed_when: "'Nothing to do' not in nvidiadriver_install.stdout"
+- name: Install nvidia drivers # TODO: make removal possible?
+  ansible.builtin.command: dnf module install -y nvidia-driver
+  register: _cuda_driver_install
+  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"
   register: cuda_package_install
 
-- name: Add latest cuda binaries to path
+- name: Add cuda binaries to path
   lineinfile:
     path: /etc/profile.d/sh.local
     line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'