[update-checkout] Make SWIFT_SOURCE_ROOT customizable and add a basic clone test

utils/update_checkout/test_update_checkout.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# If the user did not specify a workspace dir, create one in /tmp/workspace and
+# export it for the tests.
+if [[ -n "${1}" ]]; then
+    export UPDATECHECKOUT_TEST_WORKSPACE_DIR="${1}"
+    echo "Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
+else
+    export UPDATECHECKOUT_TEST_WORKSPACE_DIR=/tmp/workspace
+    echo "No preset workspace dir! Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
+fi
+
+python -m unittest discover -s $DIR
+
+set +e
+

utils/update_checkout/tests/scheme_mock.py

@@ -0,0 +1,153 @@
+# ===--- SchemeMock.py ----------------------------------------------------===#
+#
+#  This source file is part of the Swift.org open source project
+#
+#  Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+#  Licensed under Apache License v2.0 with Runtime Library Exception
+#
+#  See https:#swift.org/LICENSE.txt for license information
+#  See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ===----------------------------------------------------------------------===#
+"""This file defines objects for mocking an update-checkout scheme. It creates
+a json .config file and a series of .git repos with "fake commits".
+"""
+
+import json
+import os
+import subprocess
+import unittest
+
+# For now we only use a config with a single scheme. We should add support for
+# handling multiple schemes.
+MOCK_REMOTE = {
+    'repo1': [
+        # This is a series of changes to repo1. (File, NewContents)
+        ('A.txt', 'A'),
+        ('B.txt', 'B'),
+        ('A.txt', 'a'),
+    ],
+    'repo2': [
+        # This is a series of changes to repo1. (File, NewContents)
+        ('X.txt', 'X'),
+        ('Y.txt', 'Y'),
+        ('X.txt', 'z'),
+    ],
+}
+
+MOCK_CONFIG = {
+    # We reset this value with our remote path when we process
+    'https-clone-pattern': '',
+    'repos': {
+        'repo1': {
+            'remote': {'id': 'repo1'},
+        },
+        'repo2': {
+            'remote': {'id': 'repo2'},
+        },
+    },
+    'default-branch-scheme': 'master',
+    'branch-schemes': {
+        'master': {
+            'aliases': ['master'],
+            'repos': {
+                'repo1': 'master',
+                'repo2': 'master',
+            }
+        }
+    }
+}
+
+
+def call_quietly(*args, **kwargs):
+    with open(os.devnull, 'w') as f:
+        kwargs['stdout'] = f
+        kwargs['stderr'] = f
+        subprocess.check_call(*args, **kwargs)
+
+
+def create_dir(d):
+    if not os.path.isdir(d):
+        os.makedirs(d)
+
+
+def teardown_mock_remote(base_dir):
+    call_quietly(['rm', '-rf', base_dir])
+
+
+def get_config_path(base_dir):
+    return os.path.join(base_dir, 'test-config.json')
+
+
+def setup_mock_remote(base_dir):
+    create_dir(base_dir)
+
+    # We use local as a workspace for creating commits.
+    LOCAL_PATH = os.path.join(base_dir, 'local')
+    # We use remote as a directory that simulates our remote unchecked out
+    # repo.
+    REMOTE_PATH = os.path.join(base_dir, 'remote')
+
+    create_dir(REMOTE_PATH)
+    create_dir(LOCAL_PATH)
+
+    for (k, v) in MOCK_REMOTE.items():
+        local_repo_path = os.path.join(LOCAL_PATH, k)
+        remote_repo_path = os.path.join(REMOTE_PATH, k)
+        create_dir(remote_repo_path)
+        create_dir(local_repo_path)
+        call_quietly(['git', 'init', '--bare', remote_repo_path])
+        call_quietly(['git', 'clone', '-l', remote_repo_path, local_repo_path])
+        for (i, (filename, contents)) in enumerate(v):
+            filename_path = os.path.join(local_repo_path, filename)
+            with open(filename_path, 'w') as f:
+                f.write(contents)
+            call_quietly(['git', 'add', filename], cwd=local_repo_path)
+            call_quietly(['git', 'commit', '-m', 'Commit %d' % i],
+                         cwd=local_repo_path)
+            call_quietly(['git', 'push', 'origin', 'master'],
+                         cwd=local_repo_path)
+
+    base_config = MOCK_CONFIG
+    https_clone_pattern = os.path.join('file://%s' % REMOTE_PATH, '%s')
+    base_config['https-clone-pattern'] = https_clone_pattern
+
+    with open(get_config_path(base_dir), 'w') as f:
+        json.dump(base_config, f)
+
+
+BASEDIR_ENV_VAR = 'UPDATECHECKOUT_TEST_WORKSPACE_DIR'
+CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+UPDATE_CHECKOUT_PATH = os.path.abspath(os.path.join(CURRENT_FILE_DIR,
+                                                    os.path.pardir,
+                                                    os.path.pardir,
+                                                    'update-checkout'))
+
+
+class SchemeMockTestCase(unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super(SchemeMockTestCase, self).__init__(*args, **kwargs)
+
+        self.workspace = os.getenv(BASEDIR_ENV_VAR)
+        if self.workspace is None:
+            raise RuntimeError('Misconfigured test suite! Environment '
+                               'variable %s must be set!' % BASEDIR_ENV_VAR)
+        self.config_path = get_config_path(self.workspace)
+        self.update_checkout_path = UPDATE_CHECKOUT_PATH
+        if not os.access(self.update_checkout_path, os.X_OK):
+            raise RuntimeError('Error! Could not find executable '
+                               'update-checkout at path: %s'
+                               % self.update_checkout_path)
+        self.source_root = os.path.join(self.workspace, 'source_root')
+
+    def setUp(self):
+        create_dir(self.source_root)
+        setup_mock_remote(self.workspace)
+
+    def tearDown(self):
+        teardown_mock_remote(self.workspace)
+
+    def call(self, *args, **kwargs):
+        kwargs['cwd'] = self.source_root
+        call_quietly(*args, **kwargs)

utils/update_checkout/tests/test_clone.py

@@ -0,0 +1,25 @@
+# ===--- test_clone.py ----------------------------------------------------===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https:#swift.org/LICENSE.txt for license information
+# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ===----------------------------------------------------------------------===#
+
+import scheme_mock
+
+
+class CloneTestCase(scheme_mock.SchemeMockTestCase):
+
+    def __init__(self, *args, **kwargs):
+        super(CloneTestCase, self).__init__(*args, **kwargs)
+
+    def test_simple_clone(self):
+        self.call([self.update_checkout_path,
+                   '--config', self.config_path,
+                   '--source-root', self.source_root,
+                   '--clone'])

utils/update_checkout/update_checkout/update_checkout.py

@@ -81,10 +81,10 @@ def get_branch_for_repo(config, repo_name, scheme_name, scheme_map,
     return repo_branch, cross_repo
 
 
-def update_single_repository(args):
-    config, repo_name, scheme_name, scheme_map, tag, timestamp, \
-        reset_to_remote, should_clean, cross_repos_pr = args
-    repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
+def update_single_repository(pool_args):
+    source_root, config, repo_name, scheme_name, scheme_map, tag, timestamp, \
+        reset_to_remote, should_clean, cross_repos_pr = pool_args
+    repo_path = os.path.join(source_root, repo_name)
     if not os.path.isdir(repo_path):
         return
 
@@ -175,7 +175,7 @@ def update_single_repository(args):
 def get_timestamp_to_match(args):
     if not args.match_timestamp:
         return None
-    with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, "swift"),
+    with shell.pushd(os.path.join(args.source_root, "swift"),
                      dry_run=False, echo=False):
         return shell.capture(["git", "log", "-1", "--format=%cI"],
                              echo=False).strip()
@@ -198,7 +198,7 @@ def update_all_repositories(args, config, scheme_name, cross_repos_pr):
         if repo_name in args.skip_repository_list:
             print("Skipping update of '" + repo_name + "', requested by user")
             continue
-        my_args = [config,
+        my_args = [args.source_root, config,
                    repo_name,
                    scheme_name,
                    scheme_map,
@@ -220,7 +220,7 @@ def obtain_additional_swift_sources(pool_args):
     env = dict(os.environ)
     env.update({'GIT_TERMINAL_PROMPT': 0})
 
-    with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
+    with shell.pushd(args.source_root, dry_run=False, echo=False):
 
         print("Cloning '" + repo_name + "'")
 
@@ -236,14 +236,14 @@ def obtain_additional_swift_sources(pool_args):
                       env=env,
                       echo=True)
         if scheme_name:
-            src_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name, ".git")
+            src_path = os.path.join(args.source_root, repo_name, ".git")
             shell.run(['git', '--git-dir',
                        src_path, '--work-tree',
-                       os.path.join(SWIFT_SOURCE_ROOT, repo_name),
+                       os.path.join(args.source_root, repo_name),
                        'checkout', repo_branch],
                       env=env,
                       echo=False)
-    with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, repo_name),
+    with shell.pushd(os.path.join(args.source_root, repo_name),
                      dry_run=False, echo=False):
         shell.run(["git", "submodule",
                    "update", "--recursive"],
@@ -255,7 +255,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
                                         skip_history, skip_repository_list):
 
     pool_args = []
-    with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
+    with shell.pushd(args.source_root, dry_run=False, echo=False):
         for repo_name, repo_info in config['repos'].items():
             if repo_name in skip_repository_list:
                 print("Skipping clone of '" + repo_name + "', requested by "
@@ -308,7 +308,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
                               args.n_processes)
 
 
-def dump_repo_hashes(config, branch_scheme_name='repro'):
+def dump_repo_hashes(args, config, branch_scheme_name='repro'):
     """
     Dumps the current state of the repo into a new config file that contains a
     master branch scheme with the relevant branches set to the appropriate
@@ -319,17 +319,17 @@ def dump_repo_hashes(config, branch_scheme_name='repro'):
     for config_copy_key in config_copy_keys:
         new_config[config_copy_key] = config[config_copy_key]
     repos = {}
-    repos = repo_hashes(config)
+    repos = repo_hashes(args, config)
     branch_scheme = {'aliases': [branch_scheme_name], 'repos': repos}
     new_config['branch-schemes'] = {branch_scheme_name: branch_scheme}
     json.dump(new_config, sys.stdout, indent=4)
 
 
-def repo_hashes(config):
+def repo_hashes(args, config):
     repos = {}
     for repo_name, repo_info in sorted(config['repos'].items(),
                                        key=lambda x: x[0]):
-        repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
+        repo_path = os.path.join(args.source_root, repo_name)
         if os.path.exists(repo_path):
             with shell.pushd(repo_path, dry_run=False, echo=False):
                 h = shell.capture(["git", "rev-parse", "HEAD"],
@@ -340,8 +340,8 @@ def repo_hashes(config):
     return repos
 
 
-def print_repo_hashes(config):
-    repos = repo_hashes(config)
+def print_repo_hashes(args, config):
+    repos = repo_hashes(args, config)
     for repo_name, repo_hash in sorted(repos.items(),
                                        key=lambda x: x[0]):
         print("{:<35}: {:<35}".format(repo_name, repo_hash))
@@ -484,6 +484,11 @@ def main():
         help="Number of threads to run at once",
         default=0,
         dest="n_processes")
+    parser.add_argument(
+        "--source-root",
+        help="The root directory to checkout repositories",
+        default=SWIFT_SOURCE_ROOT,
+        dest='source_root')
     args = parser.parse_args()
 
     if not args.scheme:
@@ -513,7 +518,7 @@ def main():
         return (None, None)
 
     if args.dump_hashes_config:
-        dump_repo_hashes(config, args.dump_hashes_config)
+        dump_repo_hashes(args, config, args.dump_hashes_config)
         return (None, None)
 
     cross_repos_pr = {}
@@ -540,7 +545,7 @@ def main():
                                                             skip_repo_list)
 
     # Quick check whether somebody is calling update in an empty directory
-    directory_contents = os.listdir(SWIFT_SOURCE_ROOT)
+    directory_contents = os.listdir(args.source_root)
     if not ('cmark' in directory_contents or
             'llvm' in directory_contents or
             'clang' in directory_contents):
@@ -556,5 +561,5 @@ def main():
         print("update-checkout failed, fix errors and try again")
     else:
         print("update-checkout succeeded")
-        print_repo_hashes(config)
+        print_repo_hashes(args, config)
     sys.exit(fail_count)

validation-test/Python/update_checkout.test-sh

@@ -1,10 +1,5 @@
-# RUN: %{python} -m unittest discover -s %swift_src_root/utils/update_checkout
+# RUN: %empty-directory(%t)
+# RUN: %swift_src_root/utils/update_checkout/test_update_checkout.sh %t
 
-# Do not run this when doing simulator or device testing on darwin.
-#
-# Otherwise, we are running these unittests multiple times without providing
-# value. We want to still run this on linux though.
-#
-# UNSUPPORTED: OS=ios
-# UNSUPPORTED: OS=watchos
-# UNSUPPORTED: OS=tvos
+# For now, only run this on macosx.
+# REQUIRES: OS=macosx