Merge branch 'stackhpc/yoga' into add-kayobe-workflows

Author: jackhodgkiss

.github/workflows/overcloud-host-image-build.yml

@@ -23,10 +23,6 @@ on:
         description: Build Ubuntu 22.04 Jammy
         type: boolean
         default: true
-      SMS:
-        description: Push images to SMS
-        type: boolean
-        default: true
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -187,7 +183,7 @@ jobs:
         env:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' && inputs.sms
+        if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'
 
       - name: Build a Rocky Linux 8 overcloud host image
         id: build_rocky_8
@@ -231,7 +227,7 @@ jobs:
         env:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' && inputs.sms
+        if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'
 
       - name: Build a Rocky Linux 9 overcloud host image
         id: build_rocky_9
@@ -275,7 +271,7 @@ jobs:
         env:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' && inputs.sms
+        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
 
       - name: Build an Ubuntu Focal 20.04 overcloud host image
         id: build_ubuntu_focal
@@ -319,7 +315,7 @@ jobs:
         env:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' && inputs.sms
+        if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'
 
       - name: Build an Ubuntu Jammy 22.04 overcloud host image
         id: build_ubuntu_jammy
@@ -363,7 +359,7 @@ jobs:
         env:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' && inputs.sms
+        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload updated images artifact
         uses: actions/upload-artifact@v3

doc/source/configuration/index.rst

@@ -19,3 +19,4 @@ the various features provided.
    vault
    magnum-capi
    ci-cd
+   security-hardening

doc/source/configuration/security-hardening.rst

@@ -0,0 +1,44 @@
+==================
+Security Hardening
+==================
+
+CIS Benchmark Hardening
+-----------------------
+
+The roles from the `Ansible-Lockdown <https://github.com/ansible-lockdown>`_
+project are used to harden hosts in accordance with the CIS benchmark criteria.
+It won't get your benchmark score to 100%, but should provide a significant
+improvement over an unhardened system. A typical score would be 70%.
+
+The following operating systems are supported:
+
+- Rocky 8, RHEL 8, CentOS Stream 8
+- Ubuntu 22.04
+- Rocky 9
+
+Configuration
+--------------
+
+Some overrides to the role defaults are provided in
+``$KAYOBE_CONFIG_PATH/inventory/group_vars/overcloud/cis``. These may not be
+suitable for all deployments and so some fine tuning may be required. For
+instance, you may want different rules on a network node compared to a
+controller. It is best to consult the upstream role documentation for details
+about what each variable does. The documentation can be found here:
+
+- `Rocky 8, RHEL 8, CentOS Stream 8 <https://github.com/ansible-lockdown/RHEL8-CIS/tree/1.3.0>`__
+- `Ubuntu 22.04 <https://github.com/ansible-lockdown/UBUNTU22-CIS>`__
+- `Rocky 9 <https://github.com/ansible-lockdown/RHEL9-CIS>`__
+
+Running the playbooks
+---------------------
+
+As there is potential for unintended side effects when applying the hardening
+playbooks, the playbooks are not currently enabled by default. It is recommended
+that they are first applied to a representative staging environment to determine
+whether or not workloads or API requests are affected by any configuration changes.
+
+.. code-block:: console
+
+    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cis.yml
+

doc/source/operations/index.rst

@@ -10,3 +10,4 @@ This guide is for operators of the StackHPC Kayobe configuration project.
    rabbitmq
    octavia
    hotfix-playbook
+   rocky-linux-9

doc/source/operations/octavia.rst

@@ -38,3 +38,55 @@ when building new images.
 
 To rollback an image update, simply delete the old image. The next newest image with
 a tag matching ``amp_image_tag`` will be selected.
+
+Manually deleting broken load balancers
+=======================================
+
+Sometimes, a load balancer will get stuck in a broken state of ``PENDING_CREATE`` or ``PENDING_UPDATE``.
+When in this state, the load balancer cannot be deleted; you will see the error ``Invalid state PENDING_CREATE of loadbalancer resource``.
+To delete a load balancer in this state, you will need to manually update its provisioning status in the database.
+
+Find the database password:
+
+.. code-block:: console
+
+  ansible-vault view --vault-password-file <path-to-vault-pw> $KOLLA_CONFIG_PATH/passwords.yml
+
+  # Search for database_password with:
+  /^database
+
+Access the database from a controller:
+
+.. code-block:: console
+
+  docker exec -it mariadb bash
+  mysql -u root -p  octavia
+  # Enter the database password when promted.
+
+List the load balancers to find the ID of the broken one(s):
+
+.. code-block:: console
+
+  SELECT * FROM load_balancer;
+
+Set the provisioning status to ERROR for any broken load balancer:
+
+.. code-block:: console
+
+  UPDATE load_balancer SET provisioning_status='ERROR' WHERE id='<id>';
+
+Delete the load balancer from the OpenStack CLI, cascading if any stray
+Amphorae are hanging around:
+
+.. code-block:: console
+
+  openstack loadbalancer delete <id> --cascade
+
+
+Sometimes, Amphora may also fail to delete if they are stuck in state
+``BOOTING``. These can be resolved entirely from the OpenStack CLI:
+
+.. code-block:: console
+
+  openstack loadbalancer amphora configure <amphora-id>
+  openstack loadbalancer amphora delete <amphora-id>