Skip to content

Commit 720852f

Browse files
sjpbsjpb
committed
use mlnxofedinstall as recommended for Rocky8/9 now
1 parent 9cffe0b commit 720852f

File tree

5 files changed

+118
-74
lines changed

5 files changed

+118
-74
lines changed

‎ansible/bootstrap.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,3 +193,12 @@
193193
- name: update facts
194194
setup:
195195
when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))
196+
197+
- hosts: ofed
198+
gather_facts: no
199+
become: yes
200+
tags: ofed
201+
tasks:
202+
- include_role:
203+
name: ofed
204+
tasks_from: main.yml

‎ansible/roles/ofed/README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# ofed
2+
3+
This role installs Mellanox OFED:
4+
- It checks that the running kernel is the latest installed one, and errors if not.
5+
- Installation uses the `mlnxofedinstall` command, with support for the running kernel
6+
and (by default) without firmware updates.
7+
8+
As OFED installation takes a long time generally this should only be used during image build,
9+
for example by setting:
10+
11+
```
12+
environments/groups/<environment>/groups:
13+
[ofed:children]
14+
builder
15+
```
16+
17+
# Role variables
18+
19+
See `defaults/main.yml`
20+
21+
Note ansible facts are required, unless setting `ofed_distro_version` and `ofed_arch` specifically.

‎ansible/roles/ofed/defaults/main.yml

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,26 @@
1-
ofed_install: true
2-
ofed_version: 5.8-2.0.3.0
3-
ofed_download_url: https://content.mellanox.com/ofed/MLNX_OFED-{{ ofed_version }}/MLNX_OFED_SRC-{{ ofed_version }}.tgz
1+
ofed_version: 24.01-0.3.3.1
2+
ofed_download_url: https://content.mellanox.com/ofed/MLNX_OFED-{{ ofed_version }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}.tgz
3+
ofed_distro: rhel # NB: not expected to work on other distros due to installation differences
4+
ofed_distro_version: "{{ ansible_distribution_version }}" # e.g. '8.9'
5+
ofed_arch: "{{ ansible_architecture }}"
46
ofed_tmp_dir: /tmp
5-
ofed_packages: all
7+
ofed_update_firmware: false
8+
ofed_build_packages:
9+
- perl
10+
- createrepo
11+
- kernel-rpm-macros
12+
- libtool
13+
- python36
14+
- autoconf
15+
- automake
16+
- gcc
17+
- rpm-build
18+
- lsof
19+
- gdb-headless
20+
- patch
21+
- kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}
22+
- pciutils
23+
- kernel-modules-extra
24+
- tk
25+
- gcc-gfortran
26+
- tcl

‎ansible/roles/ofed/tasks/install.yml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
- name: Get installed kernels
2+
command: dnf list --installed kernel
3+
register: _ofed_dnf_kernels
4+
changed_when: false
5+
6+
- name: Determine running kernel
7+
command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
8+
register: _ofed_loaded_kernel
9+
changed_when: false
10+
11+
- name: Check current kernel is newest installed
12+
assert:
13+
that: _ofed_loaded_kernel.stdout == _ofed_dnf_kernels_newest
14+
fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
15+
vars:
16+
_ofed_dnf_kernels_newest: >-
17+
{{ _ofed_dnf_kernels.stdout_lines[1:] | map('regex_replace', '^\w+\.(\w+)\s+(\S+)\s+\S+\s*$', '\2.\1') | community.general.version_sort | last }}
18+
# dnf line format e.g. "kernel.x86_64 4.18.0-513.18.1.el8_9 @baseos "
19+
20+
- name: Enable epel
21+
dnf:
22+
name: epel-release
23+
24+
- name: Check for existing OFED installation
25+
command: ofed_info
26+
changed_when: false
27+
failed_when:
28+
- _ofed_info.rc > 0
29+
- "'No such file or directory' not in ofed_info.msg"
30+
register: _ofed_info
31+
32+
- name: Install build prerequisites
33+
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
34+
# don't want to install a load of prereqs unnecessarily
35+
dnf:
36+
name: "{{ ofed_build_packages }}"
37+
38+
- name: Download and unpack Mellanox OFED tarball
39+
ansible.builtin.unarchive:
40+
src: "{{ ofed_download_url }}"
41+
dest: "{{ ofed_tmp_dir }}"
42+
remote_src: yes
43+
become: no
44+
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
45+
46+
# Below from https://docs.nvidia.com/networking/display/mlnxofedv24010331/user+manual
47+
- name: Run OFED install script
48+
shell:
49+
cmd: "{{ ofed_tmp_dir }}/MLNX_OFED_LINUX-{{ ofed_version }}/mlnxofedinstall --add-kernel-support {% if not ofed_update_firmware %}--without-fw-update{% endif %} --force"
50+
register: _ofed_install
51+
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
52+
53+
- name: Update initramfs
54+
command:
55+
cmd: dracut -f
56+
when: '"update your initramfs" in _ofed_install.stdout | default("")'
57+
failed_when: false # always shows errors due to deleted modules for inbox RDMA drivers
58+
59+
- name: Load the new driver
60+
command:
61+
cmd: /etc/init.d/openibd restart
62+
when: '"To load the new driver" in _ofed_install.stdout | default("")'

‎ansible/roles/ofed/tasks/main.yml

Lines changed: 1 addition & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1 @@
1-
# NB: This assumes the running kernel is the one we want to build against, i.e. any updates/reboots have been done
2-
# Implements https://docs.nvidia.com/networking/display/MLNXOFEDv561033/Installing+MLNX_OFED#InstallingMLNX_OFED-InstallingOFEDonCommunityOperatingSystems
3-
4-
- name: Download and unpack Mellanox OFED source tarball
5-
ansible.builtin.unarchive:
6-
src: "{{ ofed_download_url }}"
7-
dest: "{{ ofed_tmp_dir }}"
8-
remote_src: yes
9-
10-
- name: Determine running kernel
11-
# not using facts
12-
command:
13-
cmd: uname -r
14-
register: _ofed_running_kernel
15-
changed_when: false
16-
17-
- name: Install build prerequisites
18-
# mostly from just running the script and seeing what it says!
19-
dnf:
20-
name:
21-
- "kernel-devel-{{ _ofed_running_kernel.stdout}}"
22-
- perl
23-
- rpm-build
24-
- gcc-gfortran
25-
- libmnl-devel
26-
- numactl-devel
27-
- gcc
28-
- gcc-c++
29-
- kernel-modules-extra
30-
- zlib-devel
31-
- iptables-devel
32-
- elfutils-devel
33-
- perl-generators
34-
- openssl-devel
35-
- lsof
36-
- systemd-devel
37-
- libnl3-devel
38-
- pciutils-devel
39-
- binutils-devel
40-
- gdb-headless
41-
- glibc-devel
42-
- flex
43-
- libdb-devel
44-
- python36
45-
- fuse-devel
46-
- libusbx-devel
47-
- python3-docutils
48-
- bison
49-
- python3-Cython
50-
- tcsh
51-
- libselinux-devel
52-
- libstdc++-devel
53-
- glib2-devel
54-
- cmake
55-
- pkgconf-pkg-config
56-
- libtool
57-
- pciutils
58-
- patch
59-
- valgrind-devel
60-
- kernel-rpm-macros
61-
- python36-devel
62-
enablerepo:
63-
- powertools
64-
register: ofed_build_deps
65-
66-
- name: Run install script
67-
command:
68-
cmd: "{{ ofed_tmp_dir }}/MLNX_OFED_SRC-{{ ofed_version}}/install.pl --{{ ofed_packages }}"
69-
70-
# TODO: delete ofed temp stuff /tmp
1+
- include_tasks: install.yml

0 commit comments

Comments
 (0)