Skip to content

Commit 17d65dc

Browse files
gottesmmgottesmm
authored
Merge pull request #25673 from gottesmm/pr-346412d7746bf95e1c890610c95d7401bc4490c3
[update-checkout] Make SWIFT_SOURCE_ROOT customizable and add a basic clone test
2 parents 5c5de29 + a9d384d commit 17d65dc

File tree

5 files changed

+227
-29
lines changed

5 files changed

+227
-29
lines changed

‎utils/update_checkout/test_update_checkout.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
6+
7+
# If the user did not specify a workspace dir, create one in /tmp/workspace and
8+
# export it for the tests.
9+
if [[ -n "${1}" ]]; then
10+
export UPDATECHECKOUT_TEST_WORKSPACE_DIR="${1}"
11+
echo "Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
12+
else
13+
export UPDATECHECKOUT_TEST_WORKSPACE_DIR=/tmp/workspace
14+
echo "No preset workspace dir! Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
15+
fi
16+
17+
python -m unittest discover -s $DIR
18+
19+
set +e
20+

‎utils/update_checkout/tests/scheme_mock.py

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# ===--- SchemeMock.py ----------------------------------------------------===#
2+
#
3+
# This source file is part of the Swift.org open source project
4+
#
5+
# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
6+
# Licensed under Apache License v2.0 with Runtime Library Exception
7+
#
8+
# See https:#swift.org/LICENSE.txt for license information
9+
# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
#
11+
# ===----------------------------------------------------------------------===#
12+
"""This file defines objects for mocking an update-checkout scheme. It creates
13+
a json .config file and a series of .git repos with "fake commits".
14+
"""
15+
16+
import json
17+
import os
18+
import subprocess
19+
import unittest
20+
21+
# For now we only use a config with a single scheme. We should add support for
22+
# handling multiple schemes.
23+
MOCK_REMOTE = {
24+
'repo1': [
25+
# This is a series of changes to repo1. (File, NewContents)
26+
('A.txt', 'A'),
27+
('B.txt', 'B'),
28+
('A.txt', 'a'),
29+
],
30+
'repo2': [
31+
# This is a series of changes to repo1. (File, NewContents)
32+
('X.txt', 'X'),
33+
('Y.txt', 'Y'),
34+
('X.txt', 'z'),
35+
],
36+
}
37+
38+
MOCK_CONFIG = {
39+
# We reset this value with our remote path when we process
40+
'https-clone-pattern': '',
41+
'repos': {
42+
'repo1': {
43+
'remote': {'id': 'repo1'},
44+
},
45+
'repo2': {
46+
'remote': {'id': 'repo2'},
47+
},
48+
},
49+
'default-branch-scheme': 'master',
50+
'branch-schemes': {
51+
'master': {
52+
'aliases': ['master'],
53+
'repos': {
54+
'repo1': 'master',
55+
'repo2': 'master',
56+
}
57+
}
58+
}
59+
}
60+
61+
62+
def call_quietly(*args, **kwargs):
63+
with open(os.devnull, 'w') as f:
64+
kwargs['stdout'] = f
65+
kwargs['stderr'] = f
66+
subprocess.check_call(*args, **kwargs)
67+
68+
69+
def create_dir(d):
70+
if not os.path.isdir(d):
71+
os.makedirs(d)
72+
73+
74+
def teardown_mock_remote(base_dir):
75+
call_quietly(['rm', '-rf', base_dir])
76+
77+
78+
def get_config_path(base_dir):
79+
return os.path.join(base_dir, 'test-config.json')
80+
81+
82+
def setup_mock_remote(base_dir):
83+
create_dir(base_dir)
84+
85+
# We use local as a workspace for creating commits.
86+
LOCAL_PATH = os.path.join(base_dir, 'local')
87+
# We use remote as a directory that simulates our remote unchecked out
88+
# repo.
89+
REMOTE_PATH = os.path.join(base_dir, 'remote')
90+
91+
create_dir(REMOTE_PATH)
92+
create_dir(LOCAL_PATH)
93+
94+
for (k, v) in MOCK_REMOTE.items():
95+
local_repo_path = os.path.join(LOCAL_PATH, k)
96+
remote_repo_path = os.path.join(REMOTE_PATH, k)
97+
create_dir(remote_repo_path)
98+
create_dir(local_repo_path)
99+
call_quietly(['git', 'init', '--bare', remote_repo_path])
100+
call_quietly(['git', 'clone', '-l', remote_repo_path, local_repo_path])
101+
for (i, (filename, contents)) in enumerate(v):
102+
filename_path = os.path.join(local_repo_path, filename)
103+
with open(filename_path, 'w') as f:
104+
f.write(contents)
105+
call_quietly(['git', 'add', filename], cwd=local_repo_path)
106+
call_quietly(['git', 'commit', '-m', 'Commit %d' % i],
107+
cwd=local_repo_path)
108+
call_quietly(['git', 'push', 'origin', 'master'],
109+
cwd=local_repo_path)
110+
111+
base_config = MOCK_CONFIG
112+
https_clone_pattern = os.path.join('file://%s' % REMOTE_PATH, '%s')
113+
base_config['https-clone-pattern'] = https_clone_pattern
114+
115+
with open(get_config_path(base_dir), 'w') as f:
116+
json.dump(base_config, f)
117+
118+
119+
BASEDIR_ENV_VAR = 'UPDATECHECKOUT_TEST_WORKSPACE_DIR'
120+
CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__))
121+
UPDATE_CHECKOUT_PATH = os.path.abspath(os.path.join(CURRENT_FILE_DIR,
122+
os.path.pardir,
123+
os.path.pardir,
124+
'update-checkout'))
125+
126+
127+
class SchemeMockTestCase(unittest.TestCase):
128+
129+
def __init__(self, *args, **kwargs):
130+
super(SchemeMockTestCase, self).__init__(*args, **kwargs)
131+
132+
self.workspace = os.getenv(BASEDIR_ENV_VAR)
133+
if self.workspace is None:
134+
raise RuntimeError('Misconfigured test suite! Environment '
135+
'variable %s must be set!' % BASEDIR_ENV_VAR)
136+
self.config_path = get_config_path(self.workspace)
137+
self.update_checkout_path = UPDATE_CHECKOUT_PATH
138+
if not os.access(self.update_checkout_path, os.X_OK):
139+
raise RuntimeError('Error! Could not find executable '
140+
'update-checkout at path: %s'
141+
% self.update_checkout_path)
142+
self.source_root = os.path.join(self.workspace, 'source_root')
143+
144+
def setUp(self):
145+
create_dir(self.source_root)
146+
setup_mock_remote(self.workspace)
147+
148+
def tearDown(self):
149+
teardown_mock_remote(self.workspace)
150+
151+
def call(self, *args, **kwargs):
152+
kwargs['cwd'] = self.source_root
153+
call_quietly(*args, **kwargs)

‎utils/update_checkout/tests/test_clone.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# ===--- test_clone.py ----------------------------------------------------===#
2+
#
3+
# This source file is part of the Swift.org open source project
4+
#
5+
# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
6+
# Licensed under Apache License v2.0 with Runtime Library Exception
7+
#
8+
# See https:#swift.org/LICENSE.txt for license information
9+
# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
#
11+
# ===----------------------------------------------------------------------===#
12+
13+
import scheme_mock
14+
15+
16+
class CloneTestCase(scheme_mock.SchemeMockTestCase):
17+
18+
def __init__(self, *args, **kwargs):
19+
super(CloneTestCase, self).__init__(*args, **kwargs)
20+
21+
def test_simple_clone(self):
22+
self.call([self.update_checkout_path,
23+
'--config', self.config_path,
24+
'--source-root', self.source_root,
25+
'--clone'])

‎utils/update_checkout/update_checkout/update_checkout.py

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,10 @@ def get_branch_for_repo(config, repo_name, scheme_name, scheme_map,
8181
return repo_branch, cross_repo
8282

8383

84-
def update_single_repository(args):
85-
config, repo_name, scheme_name, scheme_map, tag, timestamp, \
86-
reset_to_remote, should_clean, cross_repos_pr = args
87-
repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
84+
def update_single_repository(pool_args):
85+
source_root, config, repo_name, scheme_name, scheme_map, tag, timestamp, \
86+
reset_to_remote, should_clean, cross_repos_pr = pool_args
87+
repo_path = os.path.join(source_root, repo_name)
8888
if not os.path.isdir(repo_path):
8989
return
9090

@@ -175,7 +175,7 @@ def update_single_repository(args):
175175
def get_timestamp_to_match(args):
176176
if not args.match_timestamp:
177177
return None
178-
with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, "swift"),
178+
with shell.pushd(os.path.join(args.source_root, "swift"),
179179
dry_run=False, echo=False):
180180
return shell.capture(["git", "log", "-1", "--format=%cI"],
181181
echo=False).strip()
@@ -198,7 +198,7 @@ def update_all_repositories(args, config, scheme_name, cross_repos_pr):
198198
if repo_name in args.skip_repository_list:
199199
print("Skipping update of '" + repo_name + "', requested by user")
200200
continue
201-
my_args = [config,
201+
my_args = [args.source_root, config,
202202
repo_name,
203203
scheme_name,
204204
scheme_map,
@@ -220,7 +220,7 @@ def obtain_additional_swift_sources(pool_args):
220220
env = dict(os.environ)
221221
env.update({'GIT_TERMINAL_PROMPT': 0})
222222

223-
with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
223+
with shell.pushd(args.source_root, dry_run=False, echo=False):
224224

225225
print("Cloning '" + repo_name + "'")
226226

@@ -236,14 +236,14 @@ def obtain_additional_swift_sources(pool_args):
236236
env=env,
237237
echo=True)
238238
if scheme_name:
239-
src_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name, ".git")
239+
src_path = os.path.join(args.source_root, repo_name, ".git")
240240
shell.run(['git', '--git-dir',
241241
src_path, '--work-tree',
242-
os.path.join(SWIFT_SOURCE_ROOT, repo_name),
242+
os.path.join(args.source_root, repo_name),
243243
'checkout', repo_branch],
244244
env=env,
245245
echo=False)
246-
with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, repo_name),
246+
with shell.pushd(os.path.join(args.source_root, repo_name),
247247
dry_run=False, echo=False):
248248
shell.run(["git", "submodule",
249249
"update", "--recursive"],
@@ -255,7 +255,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
255255
skip_history, skip_repository_list):
256256

257257
pool_args = []
258-
with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
258+
with shell.pushd(args.source_root, dry_run=False, echo=False):
259259
for repo_name, repo_info in config['repos'].items():
260260
if repo_name in skip_repository_list:
261261
print("Skipping clone of '" + repo_name + "', requested by "
@@ -308,7 +308,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
308308
args.n_processes)
309309

310310

311-
def dump_repo_hashes(config, branch_scheme_name='repro'):
311+
def dump_repo_hashes(args, config, branch_scheme_name='repro'):
312312
"""
313313
Dumps the current state of the repo into a new config file that contains a
314314
master branch scheme with the relevant branches set to the appropriate
@@ -319,17 +319,17 @@ def dump_repo_hashes(config, branch_scheme_name='repro'):
319319
for config_copy_key in config_copy_keys:
320320
new_config[config_copy_key] = config[config_copy_key]
321321
repos = {}
322-
repos = repo_hashes(config)
322+
repos = repo_hashes(args, config)
323323
branch_scheme = {'aliases': [branch_scheme_name], 'repos': repos}
324324
new_config['branch-schemes'] = {branch_scheme_name: branch_scheme}
325325
json.dump(new_config, sys.stdout, indent=4)
326326

327327

328-
def repo_hashes(config):
328+
def repo_hashes(args, config):
329329
repos = {}
330330
for repo_name, repo_info in sorted(config['repos'].items(),
331331
key=lambda x: x[0]):
332-
repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
332+
repo_path = os.path.join(args.source_root, repo_name)
333333
if os.path.exists(repo_path):
334334
with shell.pushd(repo_path, dry_run=False, echo=False):
335335
h = shell.capture(["git", "rev-parse", "HEAD"],
@@ -340,8 +340,8 @@ def repo_hashes(config):
340340
return repos
341341

342342

343-
def print_repo_hashes(config):
344-
repos = repo_hashes(config)
343+
def print_repo_hashes(args, config):
344+
repos = repo_hashes(args, config)
345345
for repo_name, repo_hash in sorted(repos.items(),
346346
key=lambda x: x[0]):
347347
print("{:<35}: {:<35}".format(repo_name, repo_hash))
@@ -484,6 +484,11 @@ def main():
484484
help="Number of threads to run at once",
485485
default=0,
486486
dest="n_processes")
487+
parser.add_argument(
488+
"--source-root",
489+
help="The root directory to checkout repositories",
490+
default=SWIFT_SOURCE_ROOT,
491+
dest='source_root')
487492
args = parser.parse_args()
488493

489494
if not args.scheme:
@@ -513,7 +518,7 @@ def main():
513518
return (None, None)
514519

515520
if args.dump_hashes_config:
516-
dump_repo_hashes(config, args.dump_hashes_config)
521+
dump_repo_hashes(args, config, args.dump_hashes_config)
517522
return (None, None)
518523

519524
cross_repos_pr = {}
@@ -540,7 +545,7 @@ def main():
540545
skip_repo_list)
541546

542547
# Quick check whether somebody is calling update in an empty directory
543-
directory_contents = os.listdir(SWIFT_SOURCE_ROOT)
548+
directory_contents = os.listdir(args.source_root)
544549
if not ('cmark' in directory_contents or
545550
'llvm' in directory_contents or
546551
'clang' in directory_contents):
@@ -556,5 +561,5 @@ def main():
556561
print("update-checkout failed, fix errors and try again")
557562
else:
558563
print("update-checkout succeeded")
559-
print_repo_hashes(config)
564+
print_repo_hashes(args, config)
560565
sys.exit(fail_count)

‎validation-test/Python/update_checkout.test-sh

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,5 @@
1-
# RUN: %{python} -m unittest discover -s %swift_src_root/utils/update_checkout
1+
# RUN: %empty-directory(%t)
2+
# RUN: %swift_src_root/utils/update_checkout/test_update_checkout.sh %t
23

3-
# Do not run this when doing simulator or device testing on darwin.
4-
#
5-
# Otherwise, we are running these unittests multiple times without providing
6-
# value. We want to still run this on linux though.
7-
#
8-
# UNSUPPORTED: OS=ios
9-
# UNSUPPORTED: OS=watchos
10-
# UNSUPPORTED: OS=tvos
4+
# For now, only run this on macosx.
5+
# REQUIRES: OS=macosx

0 commit comments

Comments
 (0)