2023.1: zed merge

.github/workflows/overcloud-host-image-build.yml

@@ -55,7 +55,7 @@ jobs:
           rm -f /tmp/updated_images.txt
 
       - name: Clone StackHPC Kayobe repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: stackhpc/kayobe
           ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}

.github/workflows/overcloud-host-image-promote.yml

@@ -21,9 +21,9 @@ jobs:
   overcloud-host-image-promote:
     name: Promote overcloud host image
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: src/kayobe-config
 
@@ -32,9 +32,10 @@ jobs:
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
           echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+        working-directory: src/kayobe-config
 
       - name: Clone StackHPC Kayobe repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: stackhpc/kayobe
           ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}

.github/workflows/overcloud-host-image-upload.yml

@@ -0,0 +1,173 @@
+---
+name: Upload overcloud host images
+on:
+  workflow_dispatch:
+    inputs:
+      rocky9:
+        description: Upload Rocky Linux 9
+        type: boolean
+        default: true
+      ubuntu-jammy:
+        description: Upload Ubuntu 22.04 Jammy
+        type: boolean
+        default: true
+      kayobe-environment:
+        description: Kayobe environment to use
+        type: string
+        default: "ci-builder"
+    secrets:
+      KAYOBE_VAULT_PASSWORD:
+        required: true
+      CLOUDS_YAML:
+        required: true
+      OS_APPLICATION_CREDENTIAL_ID:
+        required: true
+      OS_APPLICATION_CREDENTIAL_SECRET:
+        required: true
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+jobs:
+  overcloud-host-image-upload:
+    name: Upload overcloud host images
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: arc-skc-host-image-builder-runner
+    permissions: {}
+    steps:
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+
+      - uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+
+      - name: Clone StackHPC Kayobe repository
+        uses: actions/checkout@v4
+        with:
+          repository: stackhpc/kayobe
+          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: src/kayobe
+
+      - name: Install Kayobe
+        run: |
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install ../src/kayobe
+
+      - name: Bootstrap the control host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} &&
+          kayobe control host bootstrap --skip-tags bootstrap
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+
+      - name: Install OpenStack client
+        run: |
+          source venvs/kayobe/bin/activate &&
+          pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
+
+      - name: Output Rocky Linux 9 image tag
+        id: rocky_9_image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      - name: Check if image exists already
+        id: rocky_9_image_exists
+        run: |
+          source venvs/kayobe/bin/activate &&
+          openstack image show \
+          overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }}
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        continue-on-error: true
+
+      - name: Download Rocky Linux 9 overcloud host image from Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \
+          -e os_distribution="rocky" \
+          -e os_release="9"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure'
+
+      - name: Upload Rocky Linux 9 overcloud host image to Cloud
+        run: |
+          source venvs/kayobe/bin/activate &&
+          openstack image create \
+          overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} \
+          --container-format bare \
+          --disk-format qcow2 \
+          --file /tmp/rocky-9.qcow2 \
+          --private \
+          --progress
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure'
+
+      - name: Output Ubuntu Jammy image tag
+        id: ubuntu_jammy_image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      - name: Check if image exists already
+        id: ubuntu_jammy_image_exists
+        run: |
+          source venvs/kayobe/bin/activate &&
+          openstack image show \
+          overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }}
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        continue-on-error: true
+
+      - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \
+          -e os_distribution="ubuntu" \
+          -e os_release="jammy"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure'
+
+      - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud
+        run: |
+          source venvs/kayobe/bin/activate &&
+          openstack image create \
+          overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} \
+          --container-format bare \
+          --disk-format qcow2 \
+          --file /tmp/ubuntu-jammy.qcow2 \
+          --private \
+          --progress
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure'

.github/workflows/stackhpc-build-kayobe-image.yml

@@ -51,7 +51,7 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout kayobe config
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 

.github/workflows/stackhpc-container-image-build.yml

@@ -48,7 +48,7 @@ jobs:
       openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Determine OpenStack release
         id: openstack_release
@@ -97,12 +97,12 @@ jobs:
     needs:
       - generate-tag
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: src/kayobe-config
 
       - name: Clone StackHPC Kayobe repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: stackhpc/kayobe
           ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}

.github/workflows/stackhpc-pull-request.yml

@@ -22,7 +22,7 @@ jobs:
       aio: ${{ steps.changes.outputs.aio }}
     steps:
       - name: GitHub Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check changed files
         uses: dorny/paths-filter@v2
@@ -47,7 +47,7 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     steps:
       - name: GitHub Checkout 🛎
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Setup Python ${{ matrix.python-version }} 🐍

doc/source/configuration/wazuh.rst

@@ -226,11 +226,12 @@ You may need to modify some of the variables, including:
 
 .. note::
 
-    NOTE:
     If you are using multiple environments, and you need to customise Wazuh in
     each environment, create override files in an appropriate directory,
-    for example `etc/kayobe/environments/production/inventory/group_vars/`
+    for example ``etc/kayobe/environments/production/inventory/group_vars/``.
+
     Files which values can be overridden (in the context of Wazuh):
+
     - etc/kayobe/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager
     - etc/kayobe/wazuh-manager.yml
     - etc/kayobe/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent
@@ -328,6 +329,9 @@ If you are using the wazuh generated certificates,
 this will result in the creation of some certificates and keys (in case of custom certs adjust path to it).
 Encrypt the keys (and remember to commit to git):
 
+``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/<environment>/wazuh/wazuh-certificates/*.key``
+
+If using the kayobe environments feature, otherwise:
 
 ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key``
 

etc/kayobe/ansible/pulp-host-image-download.yml

@@ -0,0 +1,38 @@
+---
+- name: Download an overcloud host image from Ark
+  hosts: localhost
+  vars:
+    # This var is an edited version of stackhpc_overcloud_host_image_url
+    # without the auth credentials in it. Auth is handled by username and
+    # password in the get_url task of this playbook
+    stackhpc_overcloud_host_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/\
+      {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/\
+      {{ 'ofed/' if stackhpc_overcloud_host_image_is_ofed else '' }}\
+      {{ stackhpc_overcloud_host_image_version }}/\
+      overcloud-{{ os_distribution }}-{{ os_release }}\
+      {{ '-ofed' if stackhpc_overcloud_host_image_is_ofed else '' }}.qcow2"
+
+  tasks:
+    - name: Print image information
+      debug:
+        msg: |
+          OS Distribution: {{ os_distribution }}
+          OS Release: {{ os_release }}
+          Image tag: {{ stackhpc_overcloud_host_image_version }}
+          OFED: {{ stackhpc_overcloud_host_image_is_ofed }}
+
+    # TODO: Add checksum support
+    - name: Download image artifact
+      get_url:
+        url: "{{ stackhpc_overcloud_host_image_url_no_auth }}"
+        username: "{{ stackhpc_image_repository_username }}"
+        password: "{{ stackhpc_image_repository_password }}"
+        force_basic_auth: true
+        unredirected_headers:
+          - "Authorization"
+        dest: "/tmp/{{ os_distribution }}-{{ os_release }}.qcow2"
+        mode: "0644"
+      register: image_download_result
+      until: image_download_result.status_code == 200
+      retries: 3
+      delay: 60