Merge pull request #1286 from stackhpc/2023.1-zed-merge

2023.1: zed merge

Author: markgoddard

.github/path-filters.yml

@@ -1,5 +1,5 @@
 # This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml.
-aio:
+aio: &aio
   - '.automation'
   - '.automation.conf/config.sh'
   - '.automation.conf/tempest/load-lists/default'
@@ -20,6 +20,11 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
-check-tags:
+check-tags: &check-tags
   - '.github/workflows/stackhpc-check-tags.yml'
   - 'etc/kayobe/kolla-image-tags.yml'
+  - 'etc/kayobe/pulp.yml'
+  - 'tools/kolla-images.py'
+build-kayobe-image:
+  - *aio
+  - *check-tags

.github/workflows/stackhpc-check-tags.yml

@@ -46,6 +46,15 @@ jobs:
         run: |
           docker image pull $KAYOBE_IMAGE
 
+      - name: Check kolla-images.py image map and tag hierarchy
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
+            '$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml'
+
       - name: Check container image tags
         run: |
           docker run -t --rm \

.github/workflows/stackhpc-pull-request.yml

@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     outputs:
       aio: ${{ steps.changes.outputs.aio }}
+      build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }}
       check-tags: ${{ steps.changes.outputs.check-tags }}
     steps:
       - name: GitHub Checkout
@@ -74,7 +75,7 @@ jobs:
       - check-changes
     uses: ./.github/workflows/stackhpc-build-kayobe-image.yml
     with:
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      if: ${{ needs.check-changes.outputs.build-kayobe-image == 'true' }}
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
   check-tags:

etc/kayobe/ansible/check-kolla-images-py.yml

@@ -0,0 +1,35 @@
+---
+- name: Check kolla-images.py image map and tag hierarchy
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Create a temporary directory
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: kolla-ansible
+      register: tempdir_result
+
+    - name: Clone Kolla Ansible repository
+      ansible.builtin.git:
+        repo: "{{ stackhpc_kolla_ansible_source_url }}"
+        version: "{{ stackhpc_kolla_ansible_source_version }}"
+        dest: "{{ tempdir_result.path }}"
+
+    - name: Check image mapping
+      ansible.builtin.command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py
+          check-image-map
+          --kolla-ansible-path {{ tempdir_result.path }}
+
+    - name: Check tag hierarchy
+      ansible.builtin.command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py
+          check-hierarchy
+          --kolla-ansible-path {{ tempdir_result.path }}
+
+    - name: Remove temporary directory
+      ansible.builtin.file:
+        path: "{{ tempdir_result.path }}"
+        state: absent

etc/kayobe/ansible/check-tags.yml

@@ -4,6 +4,7 @@
 
 - name: Check whether tags exist in Pulp container registry
   hosts: localhost
+  gather_facts: false
   tasks:
     - name: Query images and tags
       command:

etc/kayobe/ansible/growroot.yml

@@ -75,7 +75,7 @@
           vars:
             pv: "{{ pvs.stdout | from_json }}"
             disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}"
-            disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' and disk_tmp[:9] == '/dev/nvme' else disk_tmp }}"
+            disk: "{{ disk_tmp[:-1] if pv.report[0].pv[0].pv_name | regex_search('[a-z0-9]+[0-9]+p[0-9]+') else disk_tmp }}"
             part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}"
           become: true
           failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout"

etc/kayobe/kolla-image-tags.yml

@@ -17,6 +17,9 @@ kolla_image_tags:
   ironic:
     rocky-9: 2023.1-rocky-9-20240906T144646
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240906T144646
+  ironic_dnsmasq:
+    rocky-9: 2023.1-rocky-9-20240709T132012
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20240621T104542
   kolla_toolbox:
     rocky-9: 2023.1-rocky-9-20240809T102431
   letsencrypt:

releasenotes/notes/fixes-growroot-for-software-raid-3852bdea5415a0be.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes a regression when using ``growroot.yml`` and software raid where the
+    playbook would fail to identify the correct disk.

tools/kolla-images.py

@@ -39,15 +39,20 @@
 
 # Maps a Kolla image to a list of containers that use the image.
 IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = {
+    "dnsmasq": [
+        "ironic_dnsmasq",
+    ],
     "haproxy": [
         "glance_tls_proxy",
+        "haproxy",
         "neutron_tls_proxy",
     ],
     "mariadb-server": [
         "mariadb",
         "mariabackup",
     ],
-    "neutron-eswitchd": [
+    "neutron-mlnx-agent": [
+        "neutron_eswitchd",
         "neutron_mlnx_agent",
     ],
     "neutron-metadata-agent": [
@@ -58,6 +63,15 @@
         "nova_super_conductor",
         "nova_conductor",
     ],
+    "openvswitch-db-server": [
+        "openvswitch_db",
+    ],
+    "ovn-nb-db-server": [
+        "ovn_nb_db",
+    ],
+    "ovn-sb-db-server": [
+        "ovn_sb_db",
+    ],
     "prometheus-v2-server": [
         "prometheus_server",
     ],
@@ -96,6 +110,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS)
     subparsers = parser.add_subparsers(dest="command", required=True)
 
+    subparser = subparsers.add_parser("check-image-map", help="Check image mapping against kolla-ansible")
+    subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")
+
     subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible")
     subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")
 
@@ -114,7 +131,7 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def get_abs_path(relative_path: str) -> str:
+def get_abs_path(relative_path: str) -> pathlib.Path:
     """Return the absolute path of a file in SKC."""
     script_path = pathlib.Path(inspect.getfile(inspect.currentframe()))
     return script_path.parent.parent / relative_path
@@ -277,6 +294,45 @@ def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, regist
         sys.exit(1)
 
 
+def check_image_map(kolla_ansible_path: str):
+    """Check the image mapping against Kolla Ansible variables.
+
+    The *_image variables in Kolla Ansible define the mapping between
+    containers and images. Ensure that the mapping defined in this script
+    matches the one in Kolla Ansible.
+    """
+    supported_images = read_images("etc/kayobe/pulp.yml")
+    assert supported_images
+    # Build a map from container to image name.
+    cmd = """git grep -h '^[a-z0-9_]*_image:' ansible/roles/*/defaults/main.yml"""
+    image_map_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path))
+    image_map = yaml.safe_load(image_map_str)
+    image_var_re = re.compile(r"^([a-z0-9_]+)_image$")
+    image_map = {
+        image_var_re.match(image_var).group(1): image.split("/")[-1]
+        for image_var, image in image_map.items()
+    }
+    # Filter out unsupported images.
+    image_map = {
+        container: image
+        for container, image in image_map.items()
+        if image in supported_images
+    }
+    assert image_map
+    errors = []
+    # Check that our mapping is correct.
+    for container, image in image_map.items():
+        containers = get_containers(image)
+        if container not in containers:
+            errors.append((container, image))
+    if errors:
+        print("Errors:")
+    for tag_var, image in errors:
+        print(f"Expected {tag_var} container to use {image} image")
+    if errors:
+        sys.exit(1)
+
+
 def check_hierarchy(kolla_ansible_path: str):
     """Check the tag variable hierarchy against Kolla Ansible variables."""
     cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml"""
@@ -352,7 +408,9 @@ def main():
 
     validate(kolla_image_tags)
 
-    if args.command == "check-hierarchy":
+    if args.command == "check-image-map":
+        check_image_map(args.kolla_ansible_path)
+    elif args.command == "check-hierarchy":
         check_hierarchy(args.kolla_ansible_path)
     elif args.command == "check-tags":
         check_tags(base_distros, kolla_image_tags, args.registry, args.namespace)

tools/merge.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python3
+
+DESCRIPTION = """
+This script merges one release branch of SKC into another.
+
+Example 1: Merge stackhpc/yoga into stackhpc/zed:
+
+    merge.py yoga zed
+
+Example 2: Merge the branch created in example 1 into stackhpc/2023.1:
+
+    merge.py zed 2023.1 zed-yoga-merge
+
+Example 3: Continue after manually resolving merge conflicts seen in example 2:
+
+    merge.py zed 2023.1 zed-yoga-merge --continue
+
+"""
+
+import argparse
+import os
+from subprocess import check_call, check_output
+import sys
+
+
+def command(cmd):
+    print("Running:", cmd)
+    check_call(cmd)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)
+    #"Merge one branch of SKC into the next")
+    parser.add_argument("previous", type=str, help="The previous version")
+    parser.add_argument("current", type=str, help="The current version")
+    parser.add_argument("previous_branch", type=str, nargs="?", default=None, help="Optional branch to use as the previous release. Allows merging multiple branches in parallel.")
+    parser.add_argument("--continue", dest="cont", action="store_true", help="Continue after merge conflicts have been resolved.")
+    parser.add_argument("--remote", type=str, default="origin", help="Git remote")
+    return parser.parse_args()
+
+
+def fetch(args):
+    command(["git", "fetch", args.remote])
+
+
+def checkout(args):
+    merge_branch = f"{args.current}-{args.previous}-merge"
+    current_branch = f"{args.remote}/stackhpc/{args.current}"
+    command(["git", "checkout", "-B", merge_branch, current_branch])
+
+
+def update_submodules():
+    command(["git", "submodule", "update"])
+
+
+def merge_in_progress():
+    repo_root = check_output(["git", "rev-parse", "--show-toplevel"])
+    repo_root = repo_root.decode().strip()
+    return os.path.isfile(os.path.join(repo_root, ".git", "MERGE_HEAD"))
+
+
+def uncommitted_changes():
+    unstaged = check_output(["git", "diff"])
+    staged = check_output(["git", "diff", "--cached"])
+    return unstaged or staged
+
+
+def continue_merge():
+    if merge_in_progress():
+        command(["git", "merge", "--continue"])
+    else:
+        print("No merge in progress")
+
+
+def merge(args):
+    if args.previous_branch:
+        previous_branch = args.previous_branch
+    else:
+        previous_branch = f"{args.remote}/stackhpc/{args.previous}"
+    commit_message = f"Merge stackhpc/{args.previous} into stackhpc/{args.current}"
+    command(["git", "merge", previous_branch, "-m", commit_message])
+
+
+def show_diff(args):
+    print("Proposed changes:")
+    current_branch = f"{args.remote}/stackhpc/{args.current}"
+    command(["git", "diff", current_branch])
+
+
+def create_pr(args):
+    current_branch = f"stackhpc/{args.current}"
+    pr_title = f"{args.current}: {args.previous} merge"
+    command(["gh", "pr", "create", "-f", "-a", "@me", "-B", current_branch, "-t", pr_title])
+
+
+def main():
+    args = parse_args()
+    if args.cont:
+        continue_merge()
+    else:
+        if merge_in_progress():
+            print("Merge in progress - did you miss the --continue argument?")
+            sys.exit(1)
+        if uncommitted_changes():
+            print("You have uncommitted changes - aborting")
+            sys.exit(1)
+        fetch(args)
+        checkout(args)
+        update_submodules()
+        merge(args)
+    show_diff(args)
+    create_pr(args)
+
+
+if __name__ == "__main__":
+    main()