Merge pull request #325 from stackhpc/fix-nightly

Fix nightly package repo syncs

Author: markgoddard

.github/workflows/package-sync-nightly.yml

@@ -7,16 +7,52 @@ on:
 env:
   ANSIBLE_FORCE_COLOR: True
 jobs:
+  sync-matrix-build:
+    name: Build package matrix of package repo sync jobs
+    runs-on: arc-release-train-runner
+    outputs:
+      matrix: ${{ steps.matrix-build.outputs.matrix }}
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install wget 
+
+    - name: Install YQ
+      uses: dcarbone/[email protected]
+
+    - name: Create sync matrix
+      id: matrix-build
+      # Use YQ to make a list of all unique sync groups, and format as a GH
+      # actions matrix
+      run: |
+        groups=$(yq -o=json -I=0 \
+                   '([.rpm_package_repos[] | .sync_group | select(.) ]) + ([.deb_package_repos[] | .sync_group | select(.) ]) | unique' \
+                   ansible/inventory/group_vars/all/package-repos)
+        echo -n "matrix={\"sync_group\": " >> $GITHUB_OUTPUT
+        echo -n $groups >> $GITHUB_OUTPUT
+        echo "}" >> $GITHUB_OUTPUT
+
+    - name: Print sync matrix
+      run: |
+        echo "Package sync matrix:"
+        echo -n "${{ steps.matrix-build.outputs.matrix }}"
+
   sync-matrix-run:
     name: Sync
+    needs:
+      - sync-matrix-build
     strategy:
-      matrix:
-        filter: ['rocky','(centos|rhel|epel|^docker|opensearch|grafana|rabbitmq|^treasuredata|elasticsearch)', 'jammy', 'focal', 'ubuntu_cloud_archive']
+      matrix: ${{ fromJson(needs.sync-matrix-build.outputs.matrix) }}
       max-parallel: 1
       fail-fast: False
     uses: stackhpc/stackhpc-release-train/.github/workflows/package-sync.yml@main
     with:
       sync_ark: True
       sync_test: False
-      filter: ${{ matrix.filter }}
+      package_sync_group: ${{ matrix.sync_group }}
     secrets: inherit

.github/workflows/package-sync.yml

@@ -11,10 +11,11 @@ on:
         description: Sync package repositories in Test Pulp
         type: boolean
         required: true
-      filter:
-        description: Space-separated list of regular expressions matching short_name of repositories to sync
+      package_sync_group:
+        description: String matching sync_group of repositories to sync
         type: string
-        required: true
+        required: false
+        default: ""
   workflow_dispatch:
     inputs:
       sync_ark:
@@ -62,11 +63,13 @@ jobs:
           ansible/dev-pulp-repo-sync.yml \
           ansible/dev-pulp-repo-publication-cleanup.yml \
           ansible/dev-pulp-repo-publish.yml \
+          -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \
           -e deb_package_repo_filter="'$FILTER'" \
           -e rpm_package_repo_filter="'$FILTER'"
         retry_wait_seconds: 3600
       env:
         FILTER: ${{ inputs.filter }}
+        PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }}
 
   package-sync-test:
     name: Sync package repositories in test
@@ -95,8 +98,10 @@ jobs:
           ansible/test-pulp-repo-sync.yml \
           ansible/test-pulp-repo-publication-cleanup.yml \
           ansible/test-pulp-repo-publish.yml \
+          -e package_sync_group="'$PACKAGE_SYNC_GROUP'" \
           -e deb_package_repo_filter="'$FILTER'" \
           -e rpm_package_repo_filter="'$FILTER'"
         retry_wait_seconds: 3600
       env:
         FILTER: ${{ inputs.filter }}
+        PACKAGE_SYNC_GROUP: ${{ inputs.package_sync_group }}

ansible/filter_plugins/filters.py

@@ -15,18 +15,27 @@
 import re
 
 
-def select_repos(repos, filter_string):
-    """Select repositories that match a filter string.
+def select_repos(repos, filter_string, package_sync_group):
+    """Select repositories that match a filter string and package sync group.
 
     The filter string is a whitespace-separated list of regular expressions
     matching repository short names.
+
+    The package sync group is a string matching a repository sync group.
     """
-    if not filter_string:
-        return repos
-    regexes = filter_string.split()
-    patterns = re.compile(r"|".join(regexes).join('()'))
-    return [repo for repo in repos
-            if "short_name" in repo and re.search(patterns, repo["short_name"])]
+    if filter_string:
+        regexes = filter_string.split()
+        patterns = re.compile(r"|".join(regexes).join('()'))
+        filtered_repos = [repo for repo in repos
+                if "short_name" in repo and re.search(patterns, repo["short_name"])]
+    else:
+        filtered_repos = repos
+
+    if package_sync_group:
+        return [repo for repo in filtered_repos
+            if "sync_group" in repo and repo["sync_group"] == package_sync_group]
+    else:
+        return filtered_repos
 
 
 def select_images(images, filter_string):