Add a more robust script for automating the upstream/downstream sync process

Makefile

@@ -186,7 +186,10 @@ crc: crc-start crc-build crc-deploy
 clean:
 	rm -rf bin
 
+.PHONY: sync
+sync:
+	./scripts/sync.sh
+
 .PHONY: help
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-

scripts/common.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+export KNOWN_GENERATED_PATHS=(':!vendor' ':!pkg/manifests' ':!manifests' ':!go.sum' ':!go.mod')
+# TODO(tflannag): This is hacky but works in the current setup.
+export ROOT_GENERATED_PATHS=( "vendor" "pkg/manifests" "manifests" "go.mod" "go.sum" )
+export UPSTREAM_REMOTES=("api" "operator-registry" "operator-lifecycle-manager")

scripts/sync.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+ROOT_DIR=$(dirname "${BASH_SOURCE[0]}")/..
+# shellcheck disable=SC1091
+source "${ROOT_DIR}/scripts/common.sh"
+
+SYNC_BRANCH_NAME="sync-$(printf '%(%Y-%m-%d)T\n' -1)"
+
+add_remote() {
+    echo "Adding upstream remotes if they don't already exist"
+    git config remote.api.url >&- || git remote add api https://github.com/operator-framework/api
+    git config remote.operator-registry.url >&- || git remote add api https://github.com/operator-framework/operator-registry
+    git config remote.operator-lifecycle-manager.url >&- || git remote add api https://github.com/operator-framework/operator-lifecycle-manager
+    git config remote.upstream.url >&- || git remote add upstream https://github.com/openshift/operator-framework-olm
+}
+
+fetch_remote() {
+    git fetch upstream
+
+    echo "Fetching upstream remotes"
+    for remote in "${UPSTREAM_REMOTES[@]}"; do
+        git fetch "$remote"
+    done
+}
+
+new_candidate_branch() {
+    echo "Creating a sync branch if it doesn't already exist"
+    git checkout -b "$SYNC_BRANCH_NAME" upstream/master 2>/dev/null || git checkout "$SYNC_BRANCH_NAME"
+}
+
+candidates() {
+    # TODO: add support for only collecting a single remote.
+    echo "Collecting all upstream commits since last sync"
+    for remote in "${UPSTREAM_REMOTES[@]}"; do
+        "${ROOT_DIR}"/scripts/sync_get_candidates.sh "$remote"
+    done
+}
+
+pop() {
+    echo "Applying all upstream commit candidates"
+    for remote in "${UPSTREAM_REMOTES[@]}"; do
+        "${ROOT_DIR}"/scripts/sync_pop_candidate.sh "$remote" -a
+    done
+}
+
+check_local_branch_commit_diff() {
+    commits_ahead=$(git rev-list upstream/master..HEAD | wc -l)
+
+    if [[ "$commits_ahead" -gt 1 ]]; then
+        # TODO: automatically open a new pull request here.
+        echo "The local sync branch is $commits_ahead commits ahead of the upstream/master branch"
+    else
+        echo "No sync PR is needed as the upstream/master branch is up-to-date"
+    fi
+}
+
+main() {
+    add_remote
+    fetch_remote
+    new_candidate_branch
+    candidates
+    pop
+    check_local_branch_commit_diff
+}
+
+main

scripts/sync_pop_candidate.sh

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
 
-cph=$(git rev-list -n 1 CHERRY_PICK_HEAD 2> /dev/null)
-
 set -o errexit
 set -o pipefail
 
+ROOT_DIR=$(dirname "${BASH_SOURCE[@]}")/..
+# shellcheck disable=SC1091
+source "${ROOT_DIR}/scripts/common.sh"
+
 pop_all=true
 
 set +u
@@ -32,20 +34,37 @@ function pop() {
     fi
     printf 'popping: %s\n' "${rc}"
 
-    if [[ ! $cph ]]; then
-        git cherry-pick --allow-empty --keep-redundant-commits -Xsubtree="${subtree_dir}" "${rc}"
-    else
-        if [[ $cph != "${rc}" ]]; then
-            printf 'unexpected CHERRY_PICK_HEAD:\ngot %s\nexpected: %s\n' "${cph}" "${rc}"
-            exit
-        fi
-        printf 'cherry-pick in progress for %s\n' "${cph}"
-        git add .
+    if ! git cherry-pick --allow-empty --keep-redundant-commits -Xsubtree="${subtree_dir}" "${rc}"; then
+        # Always blast away the vendor directory given OLM/registry still commit it into source control.
+        git rm -rf "${subtree_dir}"/vendor 2>/dev/null || true
+
+        num_conflicts=$(git diff --name-only --diff-filter=U --relative | wc -l)
+        while [[ $num_conflicts != 0 ]] ; do
+            file=$(git diff --name-only --diff-filter=U --relative)
+
+            if [[ $file == *"go.mod"* ]]; then
+                git diff "${subtree_dir}"/go.mod
+
+                git checkout --theirs "${subtree_dir}"/go.mod
+                pushd "${subtree_dir}"
+                go mod tidy
+                git add go.mod go.sum
+                popd
+            else
+                git checkout --theirs "$file"
+                git diff "$file"
+                git add "$file"
+            fi
+
+            num_conflicts=$(git diff --name-only --diff-filter=U --relative | wc -l)
+            echo "Number of merge conflicts remaining: $num_conflicts"
+        done
 
         if [[ -z $(git status --porcelain) ]]; then
             git commit --allow-empty
         else
-            git cherry-pick --continue
+            echo "Current cherry pick status: $(git status --porcelain)"
+            git -c core.editor=true cherry-pick --continue
         fi
     fi
 
@@ -56,19 +75,21 @@ function pop() {
     # 4. Remove from cherrypick set
     make vendor
     make manifests
-    git add .
+    git add "${subtree_dir}" "${ROOT_GENERATED_PATHS[@]}"
     git status
     git commit --amend --allow-empty --no-edit --trailer "Upstream-repository: ${remote}" --trailer "Upstream-commit: ${rc}"
 
     tmp_set=$(mktemp)
     tail -n +2 "${cherrypick_set}" > "${tmp_set}"; cat "${tmp_set}" > "${cherrypick_set}"
 
-    (( --remaining ))
+    # Note: handle edge case where there's zero remaining to avoid
+    # returning a non-zero exit code.
+    (( --remaining )) || true
     printf '%d picks remaining (pop_all=%s)\n' "${remaining}" "${pop_all}"
 
     if [[ $pop_all == 'true' ]] && ((  remaining >  0 )); then
         pop
     fi
 }
 
-pop
+pop

scripts/verify_commits.sh

@@ -4,6 +4,10 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+ROOT_DIR=$(dirname "${BASH_SOURCE[@]}")/..
+# shellcheck disable=SC1091
+source "${ROOT_DIR}/scripts/common.sh"
+
 function err() {
     echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
 }
@@ -18,7 +22,7 @@ function verify_staging_sync() {
     local staging_dir="staging/${remote}"
 
     local outside_staging
-    outside_staging="$(git show --name-only "${downstream_commit}" -- ":!${staging_dir}" ':!vendor' ':!manifests' ':!go.sum' ':!go.mod')"
+    outside_staging="$(git show --name-only "${downstream_commit}" -- ":!${staging_dir}" "${KNOWN_GENERATED_PATHS[@]}")"
     if [[ -n "${outside_staging}" ]]; then
         err "downstream staging commit ${downstream_commit} changes files outside of ${staging_dir}, vendor, and manifests directories"
         err "${outside_staging}"
@@ -136,3 +140,4 @@ function main() {
 }
 
 main "$@"
+test $? -gt 0 || echo "Successfully validated all commits"