Merge pull request #823 from stackhpc/2023.1-zed-merge

2023.1: zed merge

Author: markgoddard

.github/workflows/stackhpc-container-image-build.yml

@@ -92,6 +92,7 @@ jobs:
     timeout-minutes: 720
     permissions: {}
     strategy:
+      fail-fast: false
       matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
     needs:
       - generate-tag

doc/source/configuration/index.rst

@@ -18,3 +18,4 @@ the various features provided.
    wazuh
    vault
    magnum-capi
+   security-hardening

doc/source/configuration/release-train.rst

@@ -101,6 +101,23 @@ default apt repositories. This can be done on a host-by host basis by defining
 the variables as host or group vars under ``etc/kayobe/inventory/host_vars`` or
 ``etc/kayobe/inventory/group_vars``.
 
+For Ubuntu-based deployments, Pulp currently `lacks support
+<https://github.com/pulp/pulp_deb/issues/419>`_ for certain types of content,
+including i18n files and command-not-found indices. This breaks APT when the
+``command-not-found`` package is installed:
+
+.. code:: console
+
+   E: Failed to fetch https://pulp.example.com/pulp/content/ubuntu/jammy-security/development/dists/jammy-security/main/cnf/Commands-amd64   404  Not Found
+
+The ``purge-command-not-found.yml`` custom playbook can be used to uninstall
+the package, prior to running any other APT commands. It may be installed as a
+:kayobe-doc:`pre-hook <custom-ansible-playbooks.html#hooks>` to the ``host
+configure`` commands. Note that if used as a hook, this playbook matches all
+hosts, so will run against the seed, even when running ``overcloud host
+configure``. Depending on the stage of deployment, some hosts may be
+unreachable.
+
 For Rocky Linux based systems, package manager configuration is provided by
 ``stackhpc_dnf_repos`` in ``etc/kayobe/dnf.yml``, which points to package
 repositories on the local Pulp server. To use this configuration, the

doc/source/configuration/security-hardening.rst

@@ -0,0 +1,42 @@
+==================
+Security Hardening
+==================
+
+CIS Benchmark Hardening
+-----------------------
+
+The roles from the `Ansible-Lockdown <https://github.com/ansible-lockdown>`_
+project are used to harden hosts in accordance with the CIS benchmark criteria.
+It won't get your benchmark score to 100%, but should provide a significant
+improvement over an unhardened system. A typical score would be 70%.
+
+The following operating systems are supported:
+
+- Ubuntu 22.04
+- Rocky 9
+
+Configuration
+--------------
+
+Some overrides to the role defaults are provided in
+``$KAYOBE_CONFIG_PATH/inventory/group_vars/overcloud/cis``. These may not be
+suitable for all deployments and so some fine tuning may be required. For
+instance, you may want different rules on a network node compared to a
+controller. It is best to consult the upstream role documentation for details
+about what each variable does. The documentation can be found here:
+
+- `Ubuntu 22.04 <https://github.com/ansible-lockdown/UBUNTU22-CIS>`__
+- `Rocky 9 <https://github.com/ansible-lockdown/RHEL9-CIS>`__
+
+Running the playbooks
+---------------------
+
+As there is potential for unintended side effects when applying the hardening
+playbooks, the playbooks are not currently enabled by default. It is recommended
+that they are first applied to a representative staging environment to determine
+whether or not workloads or API requests are affected by any configuration changes.
+
+.. code-block:: console
+
+    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cis.yml
+

etc/kayobe/ansible/cis.yml

@@ -4,12 +4,18 @@
   hosts: overcloud
   become: true
   tasks:
-    - name: Remove /etc/motd
-      # See remediation in:
-      # https://github.com/wazuh/wazuh/blob/bfa4efcf11e288c0a8809dc0b45fdce42fab8e0d/ruleset/sca/centos/8/cis_centos8_linux.yml#L777
-      file:
-        path: /etc/motd
-        state: absent
+    - name: Ensure the cron package is installed on ubuntu
+      package:
+        name: cron
+        state: present
+      when: ansible_facts.distribution == 'Ubuntu'
 
     - include_role:
-        name: ansible-lockdown.rhel8_cis
+        name: ansible-lockdown.rhel9_cis
+      when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '9'
+      tags: always
+
+    - include_role:
+        name: ansible-lockdown.ubuntu22_cis
+      when: ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_major_version == '22'
+      tags: always

etc/kayobe/ansible/nova-compute-disable.yml

@@ -11,6 +11,7 @@
     - name: Set up openstack cli virtualenv
       pip:
         virtualenv: "{{ venv }}"
+        virtualenv_command: "/usr/bin/python3 -m venv"
         name:
           - python-openstackclient
         state: latest

etc/kayobe/ansible/nova-compute-drain.yml

@@ -11,6 +11,7 @@
     - name: Set up openstack cli virtualenv
       pip:
         virtualenv: "{{ venv }}"
+        virtualenv_command: "/usr/bin/python3 -m venv"
         name:
           - python-openstackclient
         state: latest

etc/kayobe/ansible/nova-compute-enable.yml

@@ -11,6 +11,7 @@
     - name: Set up openstack cli virtualenv
       pip:
         virtualenv: "{{ venv }}"
+        virtualenv_command: "/usr/bin/python3 -m venv"
         name:
           - python-openstackclient
         state: latest

etc/kayobe/ansible/rabbitmq-reset.yml

@@ -1,6 +1,6 @@
 ---
 # Reset a broken RabbitMQ cluster.
-# Also restarts OpenStack services which may be broken.
+# Also restarts all OpenStack services using RabbitMQ.
 
 - name: Reset RabbitMQ
   hosts: controllers
@@ -65,7 +65,7 @@
   tags:
     - restart-openstack
   tasks:
-    # The following services can have problems if the cluster gets broken.
+    # The following services use RabbitMQ.
     - name: Restart OpenStack services
       shell: >-
-        systemctl -a | egrep '(cinder|heat|ironic|keystone|magnum|neutron|nova)' | awk '{ print $1 }' | xargs systemctl restart
+        systemctl -a | egrep '(barbican|blazar|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|neutron|nova|octavia)' | awk '{ print $1 }' | xargs systemctl restart

etc/kayobe/ansible/rekey-hosts.yml

@@ -0,0 +1,117 @@
+---
+# Playbook to rotate SSH keys across the cloud. By default it will rotate the
+# standard keys used by kayobe/kolla-ansible, but it can be configured for any
+# keys.
+
+- name: Rekey hosts
+  hosts: overcloud,seed,seed-hypervisor,infra-vms
+  gather_facts: false
+  vars:
+    # The existing key is the key that is currently used to access overcloud hosts
+    existing_private_key_path: "{{ ssh_private_key_path }}"
+    existing_public_key_path: "{{ ssh_public_key_path }}"
+    # The new key is the key that will be generated by this playbook
+    new_private_key_path: "{{ ssh_private_key_path }}"
+    new_public_key_path: "{{ ssh_public_key_path }}"
+    new_key_type: "{{ ssh_key_type }}"
+    # The existing key will locally be moved to deprecated_key_path once it is replaced
+    deprecated_key_path: ~/old_ssh_key
+    rekey_users:
+      - stack
+      - kolla
+    rekey_remove_existing_key: false
+  tasks:
+    - name: Stat existing private key file
+      ansible.builtin.stat:
+        path: "{{ existing_private_key_path }}"
+      register: stat_result
+      delegate_to: localhost
+      run_once: true
+
+    - name: Fail when existing private key does not exist
+      ansible.builtin.fail:
+        msg: "No existing private key file found. Check existing_private_key_path is set correctly."
+      when:
+        - not stat_result.stat.exists
+      delegate_to: localhost
+      run_once: true
+
+    - name: Stat existing public key file
+      ansible.builtin.stat:
+        path: "{{ existing_public_key_path }}"
+      register: stat_result
+      delegate_to: localhost
+      run_once: true
+
+    - name: Fail when existing public key does not exist
+      ansible.builtin.fail:
+        msg: "No existing public key file found. Check existing_public_key_path is set correctly."
+      when:
+        - not stat_result.stat.exists
+      delegate_to: localhost
+      run_once: true
+
+    - name: Generate a new SSH key
+      community.crypto.openssh_keypair:
+        path: "{{ existing_private_key_path }}_new"
+        type: "{{ new_key_type }}"
+      delegate_to: localhost
+      run_once: true
+
+    - name: Set new authorized keys
+      vars:
+        lookup_path: "{{ existing_private_key_path }}_new.pub"
+      ansible.posix.authorized_key:
+        user: "{{ item }}"
+        state: present
+        key: "{{ lookup('file', lookup_path) }}"
+      loop: "{{ rekey_users }}"
+      become: true
+
+    - name: Locally deprecate existing key (private)
+      command: "mv {{ existing_private_key_path }} {{ deprecated_key_path }}"
+      delegate_to: localhost
+      run_once: true
+
+    - name: Locally deprecate existing key (public)
+      command: "mv {{ existing_public_key_path }} {{ deprecated_key_path }}.pub"
+      delegate_to: localhost
+      run_once: true
+
+    - name: Locally promote new key (private)
+      command: "mv {{ existing_private_key_path }}_new {{ new_private_key_path }}"
+      delegate_to: localhost
+      run_once: true
+
+    - name: Locally promote new key (public)
+      command: "mv {{ existing_private_key_path }}_new.pub {{ new_public_key_path }}"
+      delegate_to: localhost
+      run_once: true
+
+    - block:
+        - name: Stat old key file
+          ansible.builtin.stat:
+            path: "{{ deprecated_key_path }}.pub"
+          register: stat_result
+          delegate_to: localhost
+          run_once: true
+
+        - name: Fail when deprecated public key does not exist
+          ansible.builtin.fail:
+            msg: "No deprecated public key file found. Check deprecated_key_path is set correctly."
+          when:
+            - not stat_result.stat.exists
+          delegate_to: localhost
+          run_once: true
+
+        - name: Remove old key from hosts
+          vars:
+            lookup_path: "{{ deprecated_key_path }}.pub"
+          ansible.posix.authorized_key:
+            user: "{{ item }}"
+            state: absent
+            key: "{{ lookup('file', lookup_path) }}"
+          loop: "{{ rekey_users }}"
+          become: true
+      tags: remove-key
+      when: rekey_remove_existing_key | bool

etc/kayobe/ansible/requirements.yml

@@ -12,9 +12,16 @@ collections:
     version: 2.4.0
 roles:
   - src: stackhpc.vxlan
-  - name: ansible-lockdown.rhel8_cis
-    src: https://github.com/ansible-lockdown/RHEL8-CIS
-    version: 1.3.0
+  - name: ansible-lockdown.ubuntu22_cis
+    src: https://github.com/stackhpc/UBUNTU22-CIS
+    #FIXME: Waiting for https://github.com/ansible-lockdown/UBUNTU22-CIS/pull/174
+    # to be in a tagged release
+    version: bugfix/inject-facts
+  - name: ansible-lockdown.rhel9_cis
+    src: https://github.com/stackhpc/RHEL9-CIS
+    #FIXME: Waiting for https://github.com/ansible-lockdown/RHEL9-CIS/pull/115
+    # to be in a tagged release.
+    version: bugfix/inject-facts
   - name: wazuh-ansible
     src: https://github.com/stackhpc/wazuh-ansible
     version: stackhpc

etc/kayobe/ansible/wazuh-agent.yml

@@ -28,7 +28,9 @@
         owner: wazuh
         group: wazuh
         block: sca.remote_commands=1
-      when: custom_sca_policies.files | length > 0
+      when:
+        - custom_sca_policies_folder.stat.exists
+        - custom_sca_policies.files | length > 0
       notify:
         - Restart wazuh-agent