Promote multiple tags simultaneously based on Kayobe config

.github/workflows/container-promote-old.yml

@@ -0,0 +1,59 @@
+---
+name: Promote container repositories (pre-Zed)
+on:
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: Space-separated list of regular expressions matching images to promote
+        type: string
+        required: false
+        default: ""
+      distros:
+        description: Space-separated list of base distributions to promote
+        type: string
+        required: false
+        default: ""
+      tag:
+        description: Container image tag to promote
+        required: true
+      promote-old-images:
+        description: Whether to promote images for Yoga and older
+        type: boolean
+        default: true
+      promote-new-images:
+        description: Whether to promote images for Zed and newer
+        type: boolean
+        default: true
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+  ANSIBLE_VAULT_PASSWORD_FILE: ${{ github.workspace }}/vault-pass
+jobs:
+  container-promote:
+    name: Promote container repositories
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Setup Release Train & dependencies
+      uses: ./.github/actions/setup
+      with:
+        vault-password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
+        vault-password-file: ${{ env.ANSIBLE_VAULT_PASSWORD_FILE }}
+
+    - name: Promote images from stackhpc-dev to stackhpc namespace in Ark
+      run: |
+        ansible-playbook -i ansible/inventory \
+        ansible/dev-pulp-container-promote-old.yml \
+        -e dev_pulp_repository_container_promotion_tag="$TAG" \
+        -e kolla_container_image_filter="'$FILTER'" \
+        -e kolla_base_distros_override="'$DISTROS'" \
+        -e sync_old_images="'$PROMOTE_OLD_IMAGES'" \
+        -e sync_new_images="'$PROMOTE_NEW_IMAGES'"
+      env:
+        TAG: ${{ github.event.inputs.tag }}
+        FILTER: ${{ github.event.inputs.filter }}
+        DISTROS: ${{ github.event.inputs.distros }}
+        PROMOTE_OLD_IMAGES: ${{ github.event.inputs.promote-old-images }}
+        PROMOTE_NEW_IMAGES: ${{ github.event.inputs.promote-new-images }}

.github/workflows/container-promote.yml

@@ -8,22 +8,14 @@ on:
         type: string
         required: false
         default: ""
-      distros:
-        description: Space-separated list of base distributions to promote
-        type: string
-        required: false
-        default: ""
-      tag:
-        description: Container image tag to promote
+      kayobe_config_branch:
         required: true
-      promote-old-images:
-        description: Whether to promote images for Yoga and older
-        type: boolean
-        default: true
-      promote-new-images:
-        description: Whether to promote images for Zed and newer
+        description: Branch of StackHPC Kayobe configuration to use
+      check_mode:
+        description: Check mode
         type: boolean
-        default: true
+        required: false
+        default: false
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -42,18 +34,25 @@ jobs:
         vault-password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
         vault-password-file: ${{ env.ANSIBLE_VAULT_PASSWORD_FILE }}
 
+    - name: Clone StackHPC Kayobe configuration repository
+      uses: actions/checkout@v3
+      with:
+        repository: stackhpc/stackhpc-kayobe-config
+        ref: refs/heads/${{ github.event.inputs.kayobe_config_branch }}
+        path: stackhpc-kayobe-config
+
     - name: Promote images from stackhpc-dev to stackhpc namespace in Ark
       run: |
+        args=""
+        if [[ $CHECK_MODE = true ]]; then
+          args="$args --check --diff"
+        fi
         ansible-playbook -i ansible/inventory \
+        ansible/dev-pulp-container-tag-query-kayobe.yml \
         ansible/dev-pulp-container-promote.yml \
-        -e dev_pulp_repository_container_promotion_tag="$TAG" \
         -e kolla_container_image_filter="'$FILTER'" \
-        -e kolla_base_distros_override="'$DISTROS'" \
-        -e sync_old_images="'$PROMOTE_OLD_IMAGES'" \
-        -e sync_new_images="'$PROMOTE_NEW_IMAGES'"
+        -e kayobe_config_repo_path=./stackhpc-kayobe-config/ \
+        $args
       env:
-        TAG: ${{ github.event.inputs.tag }}
         FILTER: ${{ github.event.inputs.filter }}
-        DISTROS: ${{ github.event.inputs.distros }}
-        PROMOTE_OLD_IMAGES: ${{ github.event.inputs.promote-old-images }}
-        PROMOTE_NEW_IMAGES: ${{ github.event.inputs.promote-new-images }}
+        CHECK_MODE: ${{ inputs.check_mode }}

.github/workflows/package-promote.yml

@@ -12,6 +12,11 @@ on:
         required: false
         description: Branch of StackHPC Kayobe configuration to use
         default: stackhpc/wallaby
+      check_mode:
+        description: Check mode
+        type: boolean
+        required: false
+        default: false
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -39,11 +44,18 @@ jobs:
 
     - name: Promote package repositories in Ark to release
       run: |
+        args=""
+        if [[ $CHECK_MODE = true ]]; then
+          args="$args --check --diff"
+        fi
+        source venv/bin/activate &&
         ansible-playbook -i ansible/inventory \
         ansible/dev-pulp-repo-version-query-kayobe.yml \
         ansible/dev-pulp-repo-promote.yml \
         -e deb_package_repo_filter="'$FILTER'" \
         -e rpm_package_repo_filter="'$FILTER'" \
-        -e kayobe_config_repo_path=./stackhpc-kayobe-config/
+        -e kayobe_config_repo_path=./stackhpc-kayobe-config/ \
+        $args
       env:
         FILTER: ${{ github.event.inputs.filter }}
+        CHECK_MODE: ${{ inputs.check_mode }}

ansible/dev-pulp-container-promote-old.yml

@@ -0,0 +1,65 @@
+---
+# This playbook promotes images in the development namespace (stackhpc-dev) to
+# the release namespace (stackhpc). This makes them available to clients.
+# Images with a tag defined by dev_pulp_repository_container_promotion_tag are
+# promoted.
+
+- name: Promote dev Pulp containers
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Fail if container image to promote is not defined
+      fail:
+        msg: >
+          The container image to promote must be specified via
+          'dev_pulp_repository_container_promotion_tag'
+      when: dev_pulp_repository_container_promotion_tag is not defined
+
+    - debug:
+        msg: "Promoting tag {{ dev_pulp_repository_container_promotion_tag }}"
+
+    # Copy tags from stackhpc-dev to stackhpc repositories.
+    - import_role:
+        name: stackhpc.pulp.pulp_container_content
+      vars:
+        pulp_url: "{{ dev_pulp_url }}"
+        pulp_username: "{{ dev_pulp_username }}"
+        pulp_password: "{{ dev_pulp_password }}"
+        pulp_container_content: >-
+          {%- set contents = [] -%}
+          {%- if sync_old_images | bool -%}
+          {%- for base_distro in kolla_base_distros -%}
+          {%- for image in kolla_container_images_filtered -%}
+          {%- if image not in kolla_unbuildable_images.old_scheme[base_distro] -%}
+          {%- set src_image_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%}
+          {%- set dest_image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%}
+          {%- set content = {
+            "allow_missing": True,
+            "src_repo": src_image_repo,
+            "src_is_push": true,
+            "repository": dest_image_repo,
+            "tags": [dev_pulp_repository_container_promotion_tag],
+          } -%}
+          {%- set _ = contents.append(content) -%}
+          {%- endif -%}
+          {%- endfor -%}
+          {%- endfor -%}
+          {%- endif -%}
+          {%- if sync_new_images | bool -%}
+          {%- for image in kolla_container_images_filtered -%}
+          {%- if image not in kolla_unbuildable_images.new_scheme -%}
+          {%- set src_image_repo = "stackhpc-dev/" ~ image -%}
+          {%- set dest_image_repo = "stackhpc/" ~ image -%}
+          {%- set content = {
+            "allow_missing": True,
+            "src_repo": src_image_repo,
+            "src_is_push": true,
+            "repository": dest_image_repo,
+            "tags": [dev_pulp_repository_container_promotion_tag],
+          } -%}
+          {%- set _ = contents.append(content) -%}
+          {%- endif -%}
+          {%- endfor -%}
+          {%- endif -%}
+          {{ contents }}
+        pulp_container_content_wait: false

ansible/dev-pulp-container-promote.yml

@@ -1,22 +1,29 @@
 ---
 # This playbook promotes images in the development namespace (stackhpc-dev) to
 # the release namespace (stackhpc). This makes them available to clients.
-# Images with a tag defined by dev_pulp_repository_container_promotion_tag are
-# promoted.
+# Images with a tag defined by dev_pulp_repository_container_promotion_tags are
+# promoted. dev_pulp_repository_container_promotion_tags is a dict with the
+# following format:
+#
+# dev_pulp_repository_container_promotion_tags:
+#   <image name>:
+#     - <image tag 1>
+#     - <image tag 2>
 
 - name: Promote dev Pulp containers
   hosts: localhost
   gather_facts: false
   tasks:
-    - name: Fail if container image to promote is not defined
+    - name: Fail if container images to promote are not defined
       fail:
         msg: >
-          The container image to promote must be specified via
-          'dev_pulp_repository_container_promotion_tag'
-      when: dev_pulp_repository_container_promotion_tag is not defined
+          The container images to promote must be specified via
+          'dev_pulp_repository_container_promotion_tags'
+      when: dev_pulp_repository_container_promotion_tags is not defined
 
-    - debug:
-        msg: "Promoting tag {{ dev_pulp_repository_container_promotion_tag }}"
+    - name: Display which tags are being promoted
+      debug:
+        var: "dev_pulp_repository_container_promotion_tags"
 
     # Copy tags from stackhpc-dev to stackhpc repositories.
     - import_role:
@@ -27,39 +34,17 @@
         pulp_password: "{{ dev_pulp_password }}"
         pulp_container_content: >-
           {%- set contents = [] -%}
-          {%- if sync_old_images | bool -%}
-          {%- for base_distro in kolla_base_distros -%}
-          {%- for image in kolla_container_images_filtered -%}
-          {%- if image not in kolla_unbuildable_images.old_scheme[base_distro] -%}
-          {%- set src_image_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%}
-          {%- set dest_image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%}
-          {%- set content = {
-            "allow_missing": True,
-            "src_repo": src_image_repo,
-            "src_is_push": true,
-            "repository": dest_image_repo,
-            "tags": [dev_pulp_repository_container_promotion_tag],
-          } -%}
-          {%- set _ = contents.append(content) -%}
-          {%- endif -%}
-          {%- endfor -%}
-          {%- endfor -%}
-          {%- endif -%}
-          {%- if sync_new_images | bool -%}
-          {%- for image in kolla_container_images_filtered -%}
-          {%- if image not in kolla_unbuildable_images.new_scheme -%}
+          {%- for image, tags in dev_pulp_repository_container_promotion_tags.items() -%}
           {%- set src_image_repo = "stackhpc-dev/" ~ image -%}
           {%- set dest_image_repo = "stackhpc/" ~ image -%}
           {%- set content = {
-            "allow_missing": True,
+            "allow_missing": False,
             "src_repo": src_image_repo,
             "src_is_push": true,
             "repository": dest_image_repo,
-            "tags": [dev_pulp_repository_container_promotion_tag],
+            "tags": tags,
           } -%}
           {%- set _ = contents.append(content) -%}
-          {%- endif -%}
           {%- endfor -%}
-          {%- endif -%}
           {{ contents }}
         pulp_container_content_wait: false

ansible/dev-pulp-container-tag-query-kayobe.yml

@@ -0,0 +1,32 @@
+---
+# This playbook queries the Pulp container image tags defined in a Kayobe
+# configuration repository (in etc/kayobe/kolla-image-tags.yml). It then sets
+# the 'dev_pulp_repository_container_promotion_tags' tag map variable based
+# upon those tags, which defines the set of container image tags that
+# will be promoted when the dev-pulp-container-promote.yml playbook is run.
+
+- import_playbook: kayobe-container-tag-query.yml
+
+- name: Set dev Pulp container image tags to promote from Kayobe config tags
+  hosts: localhost
+  gather_facts: true
+  tasks:
+    - name: Set a fact about container image tags to promote
+      set_fact:
+        dev_pulp_repository_container_promotion_tags: >-
+          {{ kayobe_kolla_image_tags |
+             dict2items |
+             selectattr('key', 'in', kolla_container_images_filtered) |
+             items2dict }}
+
+    - name: Display container promotion tags facts
+      debug:
+        var: dev_pulp_repository_container_promotion_tags
+
+    - name: Assert that tags variable is populated
+      assert:
+        that:
+          - dev_pulp_repository_container_promotion_tags | length > 0
+        msg: >-
+          Distribution promotion tag variable
+          'dev_pulp_repository_container_promotion_tags' is empty

ansible/kayobe-container-tag-query.yml

@@ -0,0 +1,42 @@
+---
+# This playbook queries the Pulp container image tags defined in a Kayobe
+# configuration repository (in etc/kayobe/kolla-image-tags.yml). It then sets
+# the 'kayobe_pulp_container_tags' tag map variable based upon those tags.
+#
+# The kayobe-config repository path should be specified via
+# kayobe_config_repo_path.
+
+- name: Query container image tags for Kayobe
+  hosts: localhost
+  gather_facts: True
+  vars:
+    kayobe_config_repo_path: ""
+  tasks:
+    - name: Fail if Kayobe config repo path is not specified
+      fail:
+        msg: >
+          Kayobe config git repository path must be specified via 'kayobe_config_repo_path'.
+      when: not kayobe_config_repo_path
+
+    - name: Fail if Kayobe config repo path is not a directory
+      fail:
+        msg: >
+          Kayobe config git repository path {{ kayobe_config_repo_path }} is not a directory.
+      when: kayobe_config_repo_path is not directory
+
+    - name: List Kayobe Kolla image tags
+      command:
+        cmd: >-
+          {{ kayobe_config_repo_path | realpath }}/tools/kolla-images.py list-tags
+        chdir: "{{ kayobe_config_repo_path | realpath }}"
+      register: list_tags
+      check_mode: false
+      changed_when: false
+
+    - name: Set a fact about Kayobe Kolla image tags
+      set_fact:
+        kayobe_kolla_image_tags: "{{ list_tags.stdout | from_yaml }}"
+
+    - name: Display Kayobe Kolla image tags
+      debug:
+        var: kayobe_kolla_image_tags