utils/update-checkout: Rework for more parallelism in updates

Reworks the update_all_repositories() and update_single_repository()
functions to benefit from more per-repo parallelism.

* Branch-scheme search loop is relocated from update_repository_to_scheme()
  to update_all_repositories().
* Functions update_repository_to_tag() and update_repository_to_scheme()
  are removed.
* Per-repo work of these functions is shifted to update_single_repository().
* Repeated work (repeated pushd's, etc.) has been eliminated.
* As much work as possible is performed within update_single_repository()'s
  outer try clause to catch more errors.
* Arguments passed to update_single_repository() have been changed
  appropriately.
* New helper functions confirm_tag_in_repo() and get_branch_for_repo() added
  to keep update_single_repository()'s length under control.
* Unnecessary setting of the cross_repo flag by the --tag argument has been
  removed, so repos which lack the tag are now properly rebased when updated.

Author: colinhowell

utils/update-checkout

@@ -35,14 +35,56 @@ sys.path.append(os.path.join(SCRIPT_DIR, 'swift_build_support'))
 from swift_build_support import shell  # noqa (E402)
 
 
+def confirm_tag_in_repo(tag, repo_name):
+    tag_exists = shell.capture(['git', 'ls-remote', '--tags',
+                                'origin', tag], echo=False)
+    if not tag_exists:
+        print("Tag '" + tag + "' does not exist for '"
+              + repo_name + "', just updating regularly")
+        tag = None
+    return tag
+
+
+def get_branch_for_repo(config, repo_name, scheme_name, scheme_map,
+                        cross_repos_pr):
+    cross_repo = False
+    repo_branch = scheme_name
+    if scheme_map:
+        scheme_branch = scheme_map[repo_name]
+        repo_branch = scheme_branch
+        remote_repo_id = config['repos'][repo_name]['remote']['id']
+        if remote_repo_id in cross_repos_pr:
+            cross_repo = True
+            pr_id = cross_repos_pr[remote_repo_id]
+            repo_branch = "ci_pr_{0}".format(pr_id)
+            shell.run(["git", "checkout", scheme_branch],
+                      echo=True)
+            shell.capture(["git", "branch", "-D", repo_branch],
+                          echo=True, allow_non_zero_exit=True)
+            shell.run(["git", "fetch", "origin",
+                       "pull/{0}/merge:{1}"
+                       .format(pr_id, repo_branch)], echo=True)
+    return repo_branch, cross_repo
+
+
 def update_single_repository(args):
-    repo_path, branch, reset_to_remote, should_clean, cross_repo = args
+    config, repo_name, scheme_name, scheme_map, tag, reset_to_remote, \
+        should_clean, cross_repos_pr = args
+    repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
     if not os.path.isdir(repo_path):
         return
 
     try:
         print("Updating '" + repo_path + "'")
         with shell.pushd(repo_path, dry_run=False, echo=False):
+            cross_repo = False
+            checkout_target = None
+            if tag:
+                checkout_target = confirm_tag_in_repo(tag, repo_name)
+            elif scheme_name:
+                checkout_target, cross_repo = get_branch_for_repo(
+                    config, repo_name, scheme_name, scheme_map, cross_repos_pr)
+
             # The clean option should restore a repository to pristine condition.
             if should_clean:
                 shell.run(['git', 'clean', '-fdx'], echo=True)
@@ -57,10 +99,10 @@ def update_single_repository(args):
                 except:
                     pass
 
-            if branch:
+            if checkout_target:
                 shell.run(['git', 'status', '--porcelain', '-uno'],
                                        echo=False)
-                shell.run(['git', 'checkout', branch], echo=True)
+                shell.run(['git', 'checkout', checkout_target], echo=True)
 
             # It's important that we checkout, fetch, and rebase, in order.
             # .git/FETCH_HEAD updates the not-for-merge attributes based on which
@@ -69,8 +111,8 @@ def update_single_repository(args):
 
             # If we were asked to reset to the specified branch, do the hard
             # reset and return.
-            if branch and reset_to_remote and not cross_repo:
-                shell.run(['git', 'reset', '--hard', "origin/%s" % branch],
+            if checkout_target and reset_to_remote and not cross_repo:
+                shell.run(['git', 'reset', '--hard', "origin/%s" % checkout_target],
                            echo=True)
                 return
 
@@ -110,80 +152,31 @@ def update_single_repository(args):
         return value
 
 
-def update_repository_to_tag(args, repo_name, repo_path, tag_name):
-    with shell.pushd(repo_path, dry_run=False, echo=False):
-        tag_exists = shell.capture(['git', 'ls-remote', '--tags',
-                                    'origin', tag_name], echo=False)
-        if not tag_exists:
-            print("Tag '" + tag_name + "' does not exist for '"
-                    + repo_name + "', just updating regularly")
-            tag_name = None
-
-        return [repo_path,
-                tag_name,
-                args.reset_to_remote,
-                args.clean,
-                True]
-
-
-def update_repository_to_scheme(
-        args, config, repo_name, repo_path, scheme_name, cross_repos_pr):
-    cross_repo = False
-    repo_branch = scheme_name
-    # This loop is only correct, since we know that each alias set has
-    # unique contents. This is checked by validate_config. Thus the first
-    # branch scheme data that has scheme_name as one of its aliases is
-    # the only possible correct answer.
-    for v in config['branch-schemes'].values():
-        if scheme_name not in v['aliases']:
-            continue
-        repo_branch = v['repos'][repo_name]
-        remote_repo_id = config['repos'][repo_name]['remote']['id']
-        if remote_repo_id in cross_repos_pr:
-            cross_repo = True
-            pr_id = cross_repos_pr[remote_repo_id]
-            repo_branch = "ci_pr_{0}".format(pr_id)
-            with shell.pushd(repo_path, dry_run=False, echo=False):
-                shell.run(["git", "checkout", v['repos'][repo_name]],
-                           echo=True)
-                shell.capture(["git", "branch", "-D", repo_branch],
-                              echo=True, allow_non_zero_exit=True)
-                shell.run(["git", "fetch", "origin",
-                            "pull/{0}/merge:{1}"
-                            .format(pr_id, repo_branch)], echo=True)
-        break
-    return [repo_path,
-            repo_branch,
-            args.reset_to_remote,
-            args.clean,
-            cross_repo]
-
-
 def update_all_repositories(args, config, scheme_name, cross_repos_pr):
+    scheme_map = None
+    if scheme_name:
+        # This loop is only correct, since we know that each alias set has
+        # unique contents. This is checked by validate_config. Thus the first
+        # branch scheme data that has scheme_name as one of its aliases is
+        # the only possible correct answer.
+        for v in config['branch-schemes'].values():
+            if scheme_name in v['aliases']:
+                scheme_map = v['repos']
+                break
     pool_args = []
     for repo_name in config['repos'].keys():
         if repo_name in args.skip_repository_list:
             print("Skipping update of '" + repo_name + "', requested by user")
             continue
-        repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
-        if args.tag:
-            my_args = update_repository_to_tag(args, repo_name, repo_path, args.tag)
-            pool_args.append(my_args)
-        elif scheme_name:
-            my_args = update_repository_to_scheme(args,
-                                        config,
-                                        repo_name,
-                                        repo_path,
-                                        scheme_name,
-                                        cross_repos_pr)
-            pool_args.append(my_args)
-        else:
-            my_args = [repo_path,
-                       None,
-                       args.reset_to_remote,
-                       args.clean,
-                       False]
-            pool_args.append(my_args)
+        my_args = [config,
+                   repo_name,
+                   scheme_name,
+                   scheme_map,
+                   args.tag,
+                   args.reset_to_remote,
+                   args.clean,
+                   cross_repos_pr]
+        pool_args.append(my_args)
 
     return shell.run_parallel(update_single_repository, pool_args, args.n_processes)