Generate and persist hostkeys for ondemand and login nodes

ansible/roles/persist_hostkeys/README.md

@@ -1,8 +1,5 @@
 # persist_hostkeys
 
-Save hostkeys to persistent storage and restore them after a rebuild/reimage.
+Idempotently generates a persistent set of hostkeys and restores them after a rebuild/reimage.
 
-Add hosts to the `persist_hostkeys` group to enable.
-
-This role has no variables but hosts in this group must have `appliances_state_dir`
-defined as a directory they can write to on persistent storage.
+Add hosts to the `persist_hostkeys` group to enable. All hosts in group will share the same set hostkeys.

ansible/roles/persist_hostkeys/defaults/main.yml

@@ -0,0 +1,2 @@
+persist_hostkeys_state_server: "{{ groups['control'] | first }}"
+persist_hostkeys_state_dir: "{{ hostvars[persist_hostkeys_state_server]['appliances_state_dir'] }}/hostkeys"

ansible/roles/persist_hostkeys/tasks/main.yml

@@ -1,33 +1,47 @@
 ---
 
-- name: Ensure hostkeys directory exists on persistent storage
-  file:
-    path: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}"
-    state: directory
-    owner: root
-    group: root
-    mode: 0600
+- name: Generate persistent hostkeys in state directory
+  delegate_to: "{{ persist_hostkeys_state_server }}"
+  block:
+  - name: Ensure hostkeys directory exists on persistent storage
+    file:
+      path: "{{ persist_hostkeys_state_dir }}"
+      state: directory
+      owner: root
+      group: root
+      mode: 0600
 
-- name: Copy hostkeys from persistent storage
-  # won't fail if no keys are in persistent storage
-  copy:
-    src: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
-    dest: /etc/ssh/
-    remote_src: true
+  - name: Check for existing hostkeys
+    find:
+      paths: "{{ persist_hostkeys_state_dir }}/"
+    register: _files_found
+
+  - name: Generate hostkeys
+    when: _files_found.matched == 0
+    shell:
+    # ssh-keygen -A needs a directory with an /etc/ssh suffix to write hostkeys into
+      cmd: |
+        mkdir -p {{ persist_hostkeys_state_dir }}/etc/ssh
+        ssh-keygen -A -N '' -f {{ persist_hostkeys_state_dir }}
+        mv {{ persist_hostkeys_state_dir }}/etc/ssh/* {{ persist_hostkeys_state_dir }}
+        rm -rf {{ persist_hostkeys_state_dir }}/etc/ssh
+
+  - name: Get created key names
+    find:
+      path: "{{ persist_hostkeys_state_dir }}/"
+    register: _find_ssh_keys
 
-- name: Find hostkeys
-  find:
-    path: /etc/ssh/
-    patterns: ssh_host_*_key*
-  register: _find_ssh_keys
+  - name: Create in-memory copies of keys
+    ansible.builtin.slurp:
+      src: "{{ item.path }}"
+    loop: "{{ _find_ssh_keys.files }}"
+    register: _slurp_keys
 
-- name: Persist hostkeys
+- name: Copy keys to hosts
+  no_log: true
   copy:
-    dest: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
-    src: "{{ item }}"
-    remote_src: true
-    mode: preserve
-  loop: "{{ _find_ssh_keys.files | map(attribute='path') }}"
+    content: "{{ item.content | b64decode }}"
+    dest: "/etc/ssh/{{ item.source | regex_search('[^/]+$') }}"
+  loop: "{{ _slurp_keys.results }}"
 
 - meta: reset_connection
-

docs/production.md

@@ -122,30 +122,6 @@ and referenced from the `site` and `production` environments, e.g.:
 - If floating IPs are required for login nodes, modify the OpenTofu configurations
   appropriately.
 
-- Enable persisting login node hostkeys so users do not get annoying ssh warning
-  messages on reimage:
-
-    ```yaml
-    # environments/site/inventory/groups:
-    [persist_hostkeys:children]
-    login
-    ```
-  And configure NFS to include exporting the state directory to these hosts:
-
-    ```yaml
-    # environments/common/inventory/group_vars/all/nfs.yml:
-    nfs_configurations:
-    # ... potentially, /home defintion from common environment
-    - comment: Export state directory to login nodes
-        nfs_enable:
-            server:  "{{ inventory_hostname in groups['control'] }}"
-            clients: "{{ inventory_hostname in groups['login'] }}"
-        nfs_server: "{{ nfs_server_default }}"
-        nfs_export: "/var/lib/state"
-        nfs_client_mnt_point: "/var/lib/state"
-    ```
-    See [issue 506](https://github.com/stackhpc/ansible-slurm-appliance/issues/506).
-
 - Consider whether mapping of baremetal nodes to ironic nodes is required. See
   [PR 485](https://github.com/stackhpc/ansible-slurm-appliance/pull/485).
 

environments/.caas/inventory/extra_groups

@@ -14,6 +14,3 @@ compute
 
 [podman:children]
 zenith
-
-[persist_hostkeys:children]
-openondemand

environments/.caas/inventory/group_vars/all/nfs.yml

@@ -1,14 +1,5 @@
 nfs_server: "{{ nfs_server_default }}"
 
-caas_nfs_ood_state:
-  - comment: Export /var/lib/state from Slurm control node to OOD
-    nfs_enable:
-        server:  "{{ inventory_hostname in groups['control'] }}"
-        clients: "{{ inventory_hostname in groups['openondemand'] }}"
-    nfs_export: "{{ appliances_state_dir }}"
-    nfs_client_mnt_point: "{{ appliances_state_dir }}"
-    nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
-
 caas_nfs_home:
   - comment: Export /exports/home from Slurm control node as /home
     nfs_enable:
@@ -17,4 +8,4 @@ caas_nfs_home:
     nfs_export: "/exports/home" # assumes skeleton TF is being used
     nfs_client_mnt_point: "/home"
 
-nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"
+nfs_configurations: "{{ caas_nfs_home if not cluster_home_manila_share | bool else [] }}"

environments/common/layouts/everything

@@ -71,8 +71,10 @@ openhpc
 [manila]
 # Hosts to configure for manila fileshares
 
-[persist_hostkeys]
-# Hosts to persist hostkeys for across reimaging. NB: Requires appliances_state_dir on hosts.
+[persist_hostkeys:children]
+# Hosts to use common set of hostkeys which persist across reimaging.
+login
+openondemand
 
 [squid]
 # Hosts to run squid proxy