complete cacerts, with compute-init support

Author: sjpb

ansible/bootstrap.yml

@@ -130,7 +130,7 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
-- hosts: cacerts
+- hosts: cacerts:!builder
   tags: cacerts
   gather_facts: false
   tasks:

ansible/roles/cacerts/tasks/runtime.yml renamed to ansible/roles/cacerts/tasks/configure.yml

@@ -3,13 +3,14 @@
 - name: Copy all certificates
   copy:
     src: "{{ item }}"
-    dest: /etc/pki/ca-trust/source/anchors
+    dest: /etc/pki/ca-trust/source/anchors/
     owner: root
+    group: root
     mode: 0644
   with_fileglob:
-    - "{{ appliances_environment_root }}/cacerts"
+    - "{{ cacerts_cert_dir }}/*"
   become: true
 
 - name: Update trust store
   command: update-ca-trust extract
-  become: true
+  become: true

ansible/roles/cacerts/tasks/install.yml

@@ -1,2 +1 @@
-- import_tasks: install.yml
-- import_tasks: runtime.yml
+- import_tasks: configure.yml

ansible/roles/cacerts/tasks/main.yml

@@ -48,6 +48,7 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | selinux                 | None required - use image build | Maybe [1]           |
 | bootstrap.yml            | sshd                    | None at present                 | No                  |
 | bootstrap.yml            | dnf_repos               | None at present [2]             | -                   |
+| bootstrap.yml            | cacerts                 | Supported [3]                   | -                   |
 | bootstrap.yml            | squid                   | Not relevant for compute nodes  | n/a                 |
 | bootstrap.yml            | tuned                   | Fully supported                 | No                  |         
 | bootstrap.yml            | freeipa_server          | Not relevant for compute nodes  | n/a                 |
@@ -61,23 +62,23 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | ansible_init (install)  | Not relevant during boot        | n/a                 |
 | bootstrap.yml            | k3s (install)           | Not relevant during boot        | n/a                 |
 | hooks/post-bootstrap.yml | ?                       | None at present                 | n/a                 |
-| iam.yml                  | freeipa_client          | None at present [3]             | Yes                 |
+| iam.yml                  | freeipa_client          | None at present [4]             | Yes                 |
 | iam.yml                  | freeipa_server          | Not relevant for compute nodes  | n/a                 |
 | iam.yml                  | sssd                    | None at present                 | No                  |
 | filesystems.yml          | block_devices           | None required - role deprecated | n/a                 |
 | filesystems.yml          | nfs                     | All client functionality        | No                  |
-| filesystems.yml          | manila                  | All functionality               | No [4]              |
+| filesystems.yml          | manila                  | All functionality               | No [5]              |
 | filesystems.yml          | lustre                  | None at present                 | Yes                 |
-| extras.yml               | basic_users             | All functionality [5]           | No                  |
-| extras.yml               | eessi                   | All functionality [6]           | No                  |
-| extras.yml               | cuda                    | None required - use image build | Yes [7]             |
+| extras.yml               | basic_users             | All functionality [6]           | No                  |
+| extras.yml               | eessi                   | All functionality [7]           | No                  |
+| extras.yml               | cuda                    | None required - use image build | Yes [8]             |
 | extras.yml               | persist_hostkeys        | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | compute_init (export)   | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | k9s (install)           | Not relevant during boot        | n/a                 |
-| extras.yml               | extra_packages          | None at present [8]             | -                   |
+| extras.yml               | extra_packages          | None at present [9]             | -                   |
 | slurm.yml                | mysql                   | Not relevant for compute nodes  | n/a                 |
 | slurm.yml                | rebuild                 | Not relevant for compute nodes  | n/a                 |
-| slurm.yml                | openhpc [9]             | All slurmd functionality        | No                  |
+| slurm.yml                | openhpc [10]            | All slurmd functionality        | No                  |
 | slurm.yml                | (set memory limits)     | None at present                 | -                   |
 | slurm.yml                | (block ssh)             | None at present                 | -                   |
 | portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
@@ -92,16 +93,17 @@ it also requires an image build with the role name added to the
 Notes:
 1. `selinux` is set to disabled in StackHPC images.
 2. Requirement for this functionality is TBD.
-3. FreeIPA client functionality would be better provided using a client fork
+3. `cacerts_cert_dir` must be the same on all nodes.
+4. FreeIPA client functionality would be better provided using a client fork
    which uses pkinit keys rather than OTP to reenrol nodes.
-4. Assuming default Ceph client version.
-5. Assumes home directory already exists on shared storage.
-6. Assumes `cvmfs_config` is the same on control node and all compute nodes.
-7. If `cuda` role was run during build, the nvidia-persistenced is enabled
+5. Assuming default Ceph client version.
+6. Assumes home directory already exists on shared storage.
+7. Assumes `cvmfs_config` is the same on control node and all compute nodes.
+8. If `cuda` role was run during build, the nvidia-persistenced is enabled
   and will start during boot.
-8. Would require `dnf_repos`.
-9. `openhpc` does not need to be added to `compute_init_enable`, this is
-   automatically enabled by adding `compute`.
+9. Would require `dnf_repos`.
+10. `openhpc` does not need to be added to `compute_init_enable`, this is
+    automatically enabled by adding `compute`.
 
 ## Approach
 This works as follows:

ansible/roles/cacerts/tasks/validate.yml

@@ -9,6 +9,7 @@
     enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}"
     enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}"
     enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}"
+    enable_cacerts: "{{ os_metadata.meta.cacerts | default(false) | bool }}"
     enable_tuned:  "{{ os_metadata.meta.tuned | default(false) | bool }}"
     enable_nfs: "{{ os_metadata.meta.nfs | default(false) | bool }}"
     enable_manila: "{{ os_metadata.meta.manila | default(false) | bool }}"
@@ -18,6 +19,8 @@
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
 
+    cacerts_cert_dir: "/mnt/cluster/cacerts"
+
     tuned_profile_baremetal: hpc-compute
     tuned_profile_vm: virtual-guest
     tuned_profile: "{{ tuned_profile_baremetal if ansible_virtualization_role != 'guest' else tuned_profile_vm }}"
@@ -132,6 +135,10 @@
         mode: 0644
       when: enable_etc_hosts
 
+    - name: Configure cacerts
+      include_tasks: tasks/cacerts.yml
+      when: enable_cacerts
+
     - name: Configure tuned
       include_tasks: tasks/tuned.yml
       when: enable_tuned

ansible/roles/compute_init/README.md

@@ -71,3 +71,16 @@
     remote_src: true
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
+
+- name: Copy cacerts from deploy host to /exports/cluster/cacerts/
+  copy:
+    src: "{{ item }}"
+    dest: /exports/cluster/cacerts/
+    owner: root
+    group: root
+    mode: 0644
+  with_fileglob:
+    - "{{ cacerts_cert_dir | default(appliances_environment_root + '/cacerts') }}/*" # role default
+  delegate_to: "{{ groups['control'] | first }}"
+  run_once: true
+  when: "'cacerts' in group_names"

ansible/roles/compute_init/files/compute-init.yml

@@ -32,6 +32,8 @@
       dest: files/NetworkManager-dns-none.conf
     - src: ../../basic_users/filter_plugins/filter_keys.py
       dest: filter_plugins/filter_keys.py
+    - src: ../../cacerts/tasks/configure.yml
+      dest: tasks/cacerts.yml
     - src: ../../tuned/tasks/configure.yml
       dest: tasks/tuned.yml
     - src: ../../stackhpc.nfs/tasks/nfs-clients.yml

ansible/roles/compute_init/tasks/export.yml

@@ -4,6 +4,8 @@
   tasks:
     - name: Output OS version
       command: cat /etc/redhat-release
+      changed_when: false
+
     - name: Write CI-generated inventory and secrets for debugging
       ansible.builtin.copy:
         dest: /etc/ci-config/

ansible/roles/compute_init/tasks/install.yml

@@ -165,3 +165,6 @@ extra_packages
 
 [pulp]
 # Add builder to this group to enable automatically syncing of pulp during image build
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on

environments/.stackhpc/hooks/pre.yml

@@ -111,3 +111,6 @@ control
 [extra_packages:children]
 # Hosts to install specified additional packages on
 builder
+
+[cacerts]
+# Hosts to configure CA certificates and trusts on