Merge pull request #1687 from stackhpc/caracal-sync-master

Caracal sync master

Author: Alex-Welsh

.ansible-lint-ignore

@@ -5,3 +5,5 @@ etc/kayobe/ansible/vault-generate-internal-tls.yml fqcn[action-core]
 etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core]
 etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module
 etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file]
+etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module
+etc/kayobe/ansible/check-kolla-ansible-version.yml command-instead-of-module

.github/workflows/runner-selector.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Set output for container image build runner
         run: echo "Setting runner for ${{ inputs.runner_env }} -> ${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}"
-        
+
       - id: container-image-build-runner
         run: echo "runner_name_container_image_build=${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}" >> $GITHUB_OUTPUT
 

.github/workflows/stackhpc-all-in-one.yml

@@ -7,11 +7,10 @@ name: All in one
 on:
   workflow_call:
     inputs:
-      runner:
-        required: false
+      runner_env:
+        description: Which cloud to run on?
         type: string
-        description: 'Runner name'
-        default: 'arc-skc-aio-runner'
+        default: SMS Lab
       kayobe_image:
         description: Kayobe container image
         type: string
@@ -40,18 +39,6 @@ on:
         description: Default network interface name
         type: string
         default: ens3
-      vm_flavor:
-        description: Flavor for the all-in-one VM
-        type: string
-        default: en1.medium
-      vm_network:
-        description: Network for the all-in-one VM
-        type: string
-        default: stackhpc-ci
-      vm_subnet:
-        description: Subnet for the all-in-one VM
-        type: string
-        default: stackhpc-ci
       OS_CLOUD:
         description: Name of cloud in clouds.yaml
         type: string
@@ -87,11 +74,18 @@ on:
         required: true
 
 jobs:
+  runner-selection:
+    uses: ./.github/workflows/runner-selector.yml
+    with:
+      runner_env: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }}
   # NOTE: Runner needs unzip and nodejs packages.
   all-in-one:
     name: All in one
     if: ${{ inputs.if && !cancelled() }}
-    runs-on: ${{ inputs.runner }}
+    environment: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }}
+    runs-on: ${{ needs.runner-selection.outputs.runner_name_aio }}
+    needs:
+      - runner-selection
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
@@ -170,9 +164,9 @@ jobs:
           aio_vm_interface = "${{ env.VM_INTERFACE }}"
           aio_vm_name = "${{ env.VM_NAME }}"
           aio_vm_image = "${{ env.VM_IMAGE }}"
-          aio_vm_flavor = "${{ env.VM_FLAVOR }}"
-          aio_vm_network = "${{ env.VM_NETWORK }}"
-          aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
+          aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
+          aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
           aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
           aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
@@ -181,9 +175,6 @@ jobs:
           SSH_USERNAME: "${{ inputs.ssh_username }}"
           VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
           VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
-          VM_FLAVOR: ${{ inputs.vm_flavor }}
-          VM_NETWORK: ${{ inputs.vm_network }}
-          VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
           VM_VOLUME_SIZE: ${{ inputs.upgrade && '65' || '50' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
@@ -192,7 +183,7 @@ jobs:
         run: terraform plan
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
@@ -213,7 +204,7 @@ jobs:
           exit 1
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
@@ -471,7 +462,7 @@ jobs:
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: always()

.github/workflows/stackhpc-pull-request.yml

@@ -236,3 +236,21 @@ jobs:
       upgrade: true
     secrets: inherit
     if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-upgrade-rocky-9-ovs:
+    name: aio upgrade (Rocky 9 OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovs
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

.github/workflows/update-dependencies.yml

@@ -14,6 +14,7 @@ on:
 
 jobs:
   propose_github_release_updates:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-22.04
     strategy:
       matrix:

.github/workflows/upstream-sync.yml

@@ -0,0 +1,38 @@
+---
+name: Upstream Sync
+'on':
+  schedule:
+    - cron: "15 8 * * 1"
+  workflow_dispatch:
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  synchronise-2023-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2023.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2023.1
+      upstream: openstack/kayobe-config
+  synchronise-2024-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2024.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2024.1
+      upstream: openstack/kayobe-config
+  synchronise-2025-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2025.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2025.1
+      upstream: openstack/kayobe-config
+  synchronise-master:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise master
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: master
+      upstream: openstack/kayobe-config

doc/source/configuration/ipa.rst

@@ -11,7 +11,7 @@ StackHPC provides prebuilt Ironic Python Agent (IPA) images in Release Train
 through Ark.
 
 These images are built in CI using a GitHub workflow and are configured in this
-repository. See  :kayobe-doc: `Kayobe documentation
+repository. See :kayobe-doc:`Kayobe documentation
 <configuration/reference/ironic-python-agent.html>` for more details on IPA.
 
 Release Train IPA images are used by Bifrost and Overcloud Ironic by default in

doc/source/configuration/release-train.rst

@@ -52,16 +52,29 @@ The Pulp container is deployed on the seed by default, but may be disabled by
 setting ``seed_pulp_container_enabled`` to ``false`` in
 ``etc/kayobe/seed.yml``.
 
-The URL and credentials of the local Pulp server are configured in
-``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and
-``pulp_password``. In most cases, the default values should be sufficient.
-An admin password must be generated and set as the value of a
-``secrets_pulp_password`` variable, typically in an Ansible Vault encrypted
-``etc/kayobe/secrets.yml`` file. This password will be automatically set on
-Pulp startup.
-
-If a proxy is required to access the Internet from the seed, ``pulp_proxy_url``
-may be used.
+The URL for the local Pulp server is configured by ``pulp_url`` within
+``etc/kayobe/pulp.yml``.
+
+The Pulp service can be configured with two sets of credentials; one for
+administrator operations and another read-only for overcloud hosts
+to use.
+The administrator credentials can be configured ``pulp_username``,
+``pulp_password``
+The basic user account credentials can be configured with ``pulp_stack_username``
+and ``pulp_stack_password``.
+Both sets of credentials can be found within ``etc/kayobe/pulp.yml``.
+
+Both the ``pulp_password`` and ``pulp_stack_password`` are intended to be
+configured via their ``secrets_*`` counterparts, i.e.
+``secrets_pulp_password`` and ``secrets_pulp_stack_password``. These variables
+are expected to be set in an Ansible Vault encrypted
+``etc/kayobe/secrets.yml`` file.
+
+Passwords can be generated using ``OpenSSL``
+
+.. code-block:: console
+
+  openssl rand -base64 32
 
 Host images are not synchronised to the local Pulp server, since they should
 only be pulled to the seed node once. More information on host images can be

etc/kayobe/ansible/cephadm-gather-keys.yml

@@ -68,6 +68,7 @@
         # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf.
         content: |
           {{ cephadm_ceph_conf.stdout | regex_replace('\t') }}
+          {{ kolla_ceph_conf_append if kolla_ceph_conf_append is defined }}
         dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf"
       loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
       loop_control:

etc/kayobe/ansible/check-kayobe-version.yml

@@ -0,0 +1,62 @@
+---
+- name: Check Kayobe version
+  tags: kayobe-version-check
+  hosts: localhost
+  gather_facts: false
+  vars:
+    requirements_path: "{{ kayobe_config_path }}/../../requirements.txt"
+  tasks:
+    - name: Check version
+      when: stackhpc_enable_kayobe_check
+      check_mode: false
+      block:
+        - name: Get package info
+          community.general.pip_package_info:
+          register: packages
+
+        - name: Check if pip is version 24.0 or newer
+          ansible.builtin.assert:
+            that: "{{ packages.packages.pip.pip[0].version is version('24.0', '>=') }}"
+            fail_msg: |
+              Pip must be 24.0 or newer to run this check. Upgrade pip by running
+              pip install -U pip and reinstall Kayobe by running:
+              pip install --force-reinstall -r {{ requirements_path }}
+
+        - name: Get installed Kayobe commit
+          ansible.builtin.shell:
+            cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3
+            executable: /usr/bin/bash
+          register: kayobe_git_commit
+          failed_when: kayobe_git_commit.stdout == ""
+
+        - name: Clone Kayobe
+          ansible.builtin.git:
+            repo: https://github.com/stackhpc/kayobe.git
+            dest: /tmp/kayobe-git
+            version: stackhpc/{{ openstack_release }}
+
+        - name: Get tag from Kayobe commit
+          ansible.builtin.command:
+            cmd: git describe --tags {{ kayobe_git_commit.stdout }}
+            chdir: /tmp/kayobe-git
+          register: kayobe_current_version
+
+        - name: Get latest Kayobe version
+          ansible.builtin.shell:
+            cmd: set -o pipefail && grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2
+            executable: /usr/bin/bash
+          register: kayobe_latest_version
+
+        - name: Check installed Kayobe version is the latest
+          ansible.builtin.assert:
+            that: "kayobe_latest_version.stdout in kayobe_current_version.stdout"
+            fail_msg: |
+              Kayobe must use the expected version before continuing.
+
+              Current Kayobe version: {{ kayobe_current_version.stdout }}
+              Expected Kayobe version: {{ kayobe_latest_version.stdout }}
+
+              Recreate the Kayobe environment, or install the expected version
+              by running: pip install --force-reinstall -r {{ requirements_path }}
+            success_msg: |
+              Kayobe running at version: {{ kayobe_current_version.stdout }}

etc/kayobe/ansible/check-kolla-ansible-version.yml

@@ -0,0 +1,28 @@
+---
+- name: Check Kolla-Ansible version
+  tags: kolla-ansible-version-check
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Check version
+      when: stackhpc_enable_kolla_ansible_check
+      check_mode: false
+      block:
+        - name: Get current Kolla-Ansible tag
+          ansible.builtin.command:
+            cmd: git describe --tags
+            chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}"
+          register: kolla_ansible_current_version
+
+        - name: Check installed Kolla-Ansible version is the expected version
+          ansible.builtin.assert:
+            that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout"
+            fail_msg: |
+              Kolla-Ansible must use the expected version before continuing.
+
+              Current Kolla-Ansible version: {{ kolla_ansible_current_version.stdout }}
+              Expected Kolla-Ansible version: {{ stackhpc_kolla_ansible_source_version }}
+
+              Upgrade Kolla-Ansible by running: kayobe control host upgrade
+            success_msg: |
+              Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }}

etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml

@@ -107,7 +107,7 @@
         - name: Ensure radosgw_usage_exporter container is running
           community.docker.docker_container:
             name: radosgw_usage_exporter
-            image: ghcr.io/stackhpc/radosgw_usage_exporter:v0.1.1
+            image: ghcr.io/stackhpc/radosgw_usage_exporter:v0.1.3
             network_mode: host
             env:
               RADOSGW_SERVER: "{{ radosgw_server }}"

etc/kayobe/ansible/pci-passthrough.yml

@@ -11,7 +11,7 @@
     vfio_pci_ids: |-
       {% set gpu_list = [] %}
       {% set output = [] %}
-      {% for gpu_group in gpu_group_map | dict2items | default([]) %}
+      {% for gpu_group in (gpu_group_map | default({})) | dict2items %}
       {% if gpu_group.key in group_names %}
       {% set _ = gpu_list.append(gpu_group.value) %}
       {% endif %}

etc/kayobe/ansible/smartmon-tools.yml

@@ -15,10 +15,8 @@
 
     - name: Ensure Python 3, venv, and pip are installed
       ansible.builtin.package:
-        name:
-          - python3
-          - python3-venv
-          - python3-pip
+        name: >
+          {{ ['python3', 'python3-pip'] + (['python3-venv'] if ansible_facts['distribution'] == 'Ubuntu' else []) }}
         state: present
       become: true
 

etc/kayobe/cephadm.yml

@@ -133,3 +133,6 @@ kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}"
 
 # Whether to generate Ceph configuration for Nova.
 kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}"
+
+# A (multiline) string to append to all Ceph configuration files.
+#kolla_ceph_conf_append:

etc/kayobe/containers/pulp/post.yml

@@ -28,6 +28,18 @@
     - stackhpc_pulp_sync_for_local_container_build | bool
     - pulp_settings.changed
 
+- name: Ensure Pulp stack user exists
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_user
+  vars:
+    pulp_users:
+      - username: "{{ pulp_stack_username }}"
+        password: "{{ pulp_stack_password }}"
+        is_staff: false
+  when:
+    - pulp_stack_username is defined and pulp_stack_username | length > 0
+    - pulp_stack_password is defined and pulp_stack_password | length > 0
+
 - name: Login to docker registry
   docker_login:
     registry_url: "{{ kolla_docker_registry or omit }}"