Skip to content

Commit c4810ef

Browse files
graydongraydon
authored
Merge pull request swiftlang#37 from graydon/incremental-mode
Incremental mode
2 parents ceae856 + b0a325b commit c4810ef

File tree

3 files changed

+532
-1
lines changed

3 files changed

+532
-1
lines changed

‎build_incremental.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env python
2+
# ===--- build_incremental.py ---------------------------------------------===
3+
#
4+
# This source file is part of the Swift.org open source project
5+
#
6+
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See https://swift.org/LICENSE.txt for license information
10+
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
# ===----------------------------------------------------------------------===
13+
14+
"""Build a collection of Swift projects in incremental mode, collecting stats."""
15+
16+
import argparse
17+
import json
18+
import sys
19+
20+
import common
21+
import project
22+
23+
24+
def parse_args():
25+
"""Return parsed command line arguments."""
26+
parser = argparse.ArgumentParser()
27+
project.add_arguments(parser)
28+
return parser.parse_args()
29+
30+
31+
def main():
32+
"""Execute specified indexed project actions."""
33+
args = parse_args()
34+
index = json.loads(open(args.projects).read())
35+
result = project.ProjectListBuilder(
36+
args.include_repos,
37+
args.exclude_repos,
38+
args.verbose,
39+
project.ProjectBuilder.factory(
40+
args.include_actions,
41+
args.exclude_actions,
42+
args.verbose,
43+
project.IncrementalActionBuilder.factory(
44+
args.swiftc,
45+
args.swift_version,
46+
args.swift_branch,
47+
args.sandbox_profile_xcodebuild,
48+
args.sandbox_profile_package,
49+
args.add_swift_flags,
50+
args.check_stats,
51+
args.show_stats
52+
),
53+
),
54+
index
55+
).build()
56+
common.debug_print(str(result))
57+
return 0 if result.result in [project.ResultEnum.PASS,
58+
project.ResultEnum.XFAIL] else 1
59+
60+
if __name__ == '__main__':
61+
sys.exit(main())

‎project.py

Lines changed: 291 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import filecmp
2222
import sys
2323
import json
24+
import time
2425

2526
import common
2627

@@ -99,6 +100,8 @@ def get_build_command(self, incremental=False):
99100
dir_override = []
100101
if self._has_scheme:
101102
dir_override = ['-derivedDataPath', build_dir]
103+
else:
104+
dir_override = ['SYMROOT=' + build_dir]
102105
command = (['xcodebuild']
103106
+ build
104107
+ [project_param, self._project,
@@ -275,7 +278,8 @@ def dispatch(root_path, repo, action, swiftc, swift_version,
275278
if stats_path is not None:
276279
if os.path.exists(stats_path):
277280
shutil.rmtree(stats_path)
278-
common.check_execute(['mkdir', '-p', stats_path])
281+
common.check_execute(['mkdir', '-p', stats_path],
282+
stdout=stdout, stderr=stderr)
279283

280284
if action['action'] == 'BuildSwiftPackage':
281285
return build_swift_package(os.path.join(root_path, repo['path']),
@@ -955,3 +959,289 @@ def succeeded(self, identifier):
955959
result = ActionResult(Result.PASS, error_str)
956960
common.debug_print(error_str)
957961
return result
962+
963+
class EarlyExit(Exception):
964+
def __init__(self, value):
965+
self.value = value
966+
def __str__(self):
967+
return repr(self.value)
968+
969+
def ignore_missing(f):
970+
if (f.endswith('.dia') or
971+
f.endswith('~')):
972+
return True
973+
return False
974+
975+
def ignore_diff(f):
976+
if (f.endswith('-master.swiftdeps') or
977+
f.endswith('dependency_info.dat')):
978+
return True
979+
return False
980+
981+
def have_same_trees(full, incr, d):
982+
ok = True
983+
for f in d.left_only:
984+
if ignore_missing(f):
985+
continue
986+
ok = False
987+
common.debug_print("Missing 'incr' file: %s"
988+
% os.path.relpath(os.path.join(d.left, f), full))
989+
990+
for f in d.right_only:
991+
if ignore_missing(f):
992+
continue
993+
ok = False
994+
common.debug_print("Missing 'full' file: %s"
995+
% os.path.relpath(os.path.join(d.right, f), incr))
996+
997+
for f in d.diff_files:
998+
if ignore_diff(f):
999+
continue
1000+
ok = False
1001+
common.debug_print("File difference: %s"
1002+
% os.path.relpath(os.path.join(d.left, f), full))
1003+
1004+
for sub in d.subdirs.values():
1005+
ok = have_same_trees(full, incr, sub) and ok
1006+
return ok
1007+
1008+
class StatsSummary:
1009+
1010+
def __init__(self):
1011+
self.commits = {}
1012+
1013+
def add_stats_from_json(self, seq, sha, j):
1014+
key = (seq, sha)
1015+
if key not in self.commits:
1016+
self.commits[key] = {}
1017+
for (k, v) in j.items():
1018+
if k.startswith("time."):
1019+
continue
1020+
e = self.commits[key].get(k, 0)
1021+
self.commits[key][k] = int(v) + e
1022+
1023+
def check_against_expected(self, seq, sha, expected):
1024+
key = (seq, sha)
1025+
if key in self.commits:
1026+
for (k, ev) in expected.items():
1027+
if k in self.commits[key]:
1028+
gv = self.commits[key][k]
1029+
if ev < gv:
1030+
message = ("Expected %s of %s, got %s" %
1031+
(k, str(ev), str(gv)) )
1032+
raise EarlyExit(ActionResult(Result.FAIL, message))
1033+
1034+
def add_stats_from_file(self, seq, sha, f):
1035+
with open(f) as fp:
1036+
self.add_stats_from_json(seq, sha, json.load(fp))
1037+
1038+
def add_stats_from_dir(self, seq, sha, path):
1039+
for root, dirs, files in os.walk(path):
1040+
for f in files:
1041+
if not f.endswith(".json"):
1042+
continue
1043+
self.add_stats_from_file(seq, sha, os.path.join(root, f))
1044+
1045+
def dump(self, pattern):
1046+
return json.dumps([ {"commit": sha,
1047+
"stats": { k: v for (k, v) in self.commits[(seq, sha)].items()
1048+
if re.match(pattern, k) } }
1049+
for (seq, sha) in sorted(self.commits.keys()) ],
1050+
sort_keys=False,
1051+
indent=2)
1052+
1053+
class IncrementalActionBuilder(ActionBuilder):
1054+
1055+
def __init__(self, swiftc, swift_version, swift_branch,
1056+
sandbox_profile_xcodebuild,
1057+
sandbox_profile_package,
1058+
added_swift_flags,
1059+
check_stats,
1060+
show_stats,
1061+
project, action):
1062+
super(IncrementalActionBuilder,
1063+
self).__init__(swiftc, swift_version, swift_branch,
1064+
sandbox_profile_xcodebuild,
1065+
sandbox_profile_package,
1066+
added_swift_flags,
1067+
skip_clean=True,
1068+
project=project,
1069+
action=action)
1070+
self.check_stats = check_stats
1071+
self.show_stats = show_stats
1072+
self.stats_path = None
1073+
self.stats_summ = None
1074+
self.proj_path = os.path.join(self.root_path, self.project['path'])
1075+
self.incr_path = self.proj_path + "-incr"
1076+
if self.check_stats or (self.show_stats is not None):
1077+
self.stats_path = os.path.join(self.proj_path, "swift-stats")
1078+
self.stats_summ = StatsSummary()
1079+
1080+
def curr_build_state_path(self):
1081+
if self.action['action'] == 'BuildSwiftPackage':
1082+
return os.path.join(self.proj_path, ".build")
1083+
match = re.match(r'^(Build|Test)Xcode(Workspace|Project)(Scheme|Target)$',
1084+
self.action['action'])
1085+
if match:
1086+
project_path = os.path.join(self.proj_path,
1087+
self.action[match.group(2).lower()])
1088+
return os.path.join(os.path.dirname(project_path), "build")
1089+
else:
1090+
raise Exception("Unsupported action: " + self.action['action'])
1091+
1092+
def ignored_differences(self):
1093+
if self.action['action'] == 'BuildSwiftPackage':
1094+
return ['ModuleCache', 'build.db', 'master.swiftdeps', 'master.swiftdeps~']
1095+
elif re.match(r'^(Build|Test)Xcode(Workspace|Project)(Scheme|Target)$',
1096+
self.action['action']):
1097+
return ['ModuleCache', 'Logs', 'info.plist', 'dgph', 'dgph~',
1098+
'master.swiftdeps', 'master.swiftdeps~']
1099+
else:
1100+
raise Exception("Unsupported action: " + self.action['action'])
1101+
1102+
def expect_determinism(self):
1103+
# We're not seeing determinism in incremental builds yet, so
1104+
# for the time being disable the expectation.
1105+
return False
1106+
1107+
def saved_build_state_path(self, seq, flav, sha):
1108+
return os.path.join(self.incr_path, ("build-state-%03d-%s-%.7s" %
1109+
(seq, flav, sha)))
1110+
1111+
def saved_build_stats_path(self, seq, flav, sha):
1112+
return os.path.join(self.incr_path, ("build-stats-%03d-%s-%.7s" %
1113+
(seq, flav, sha)))
1114+
1115+
def restore_saved_build_state(self, seq, flav, sha, stdout=sys.stdout):
1116+
src = self.saved_build_state_path(seq, flav, sha)
1117+
dst = self.curr_build_state_path()
1118+
proj = self.project['path']
1119+
common.debug_print("Restoring %s build-state #%d of %s from %s" %
1120+
(flav, seq, proj, src), stderr=stdout)
1121+
if os.path.exists(dst):
1122+
shutil.rmtree(dst)
1123+
shutil.copytree(src, dst, symlinks=True)
1124+
1125+
def save_build_state(self, seq, flav, sha, stats, stdout=sys.stdout):
1126+
src = self.curr_build_state_path()
1127+
dst = self.saved_build_state_path(seq, flav, sha)
1128+
proj = self.project['path']
1129+
common.debug_print("Saving %s state #%d of %s to %s" %
1130+
(flav, seq, proj, dst), stderr=stdout)
1131+
if os.path.exists(dst):
1132+
shutil.rmtree(dst)
1133+
shutil.copytree(src, dst, symlinks=True)
1134+
if self.stats_summ is not None:
1135+
self.stats_summ.add_stats_from_dir(seq, sha, self.stats_path)
1136+
src = self.stats_path
1137+
dst = self.saved_build_stats_path(seq, flav, sha)
1138+
common.debug_print("Saving %s stats #%d of %s to %s" %
1139+
(flav, seq, proj, dst), stderr=stdout)
1140+
if os.path.exists(dst):
1141+
shutil.rmtree(dst)
1142+
shutil.copytree(src, dst, symlinks=True)
1143+
if stats is not None and self.check_stats:
1144+
self.stats_summ.check_against_expected(seq, sha, stats)
1145+
1146+
def check_full_vs_incr(self, seq, sha, stdout=sys.stdout):
1147+
full = self.saved_build_state_path(seq, 'full', sha)
1148+
incr = self.saved_build_state_path(seq, 'incr', sha)
1149+
common.debug_print("Comparing dirs %s vs. %s" % (os.path.relpath(full),
1150+
os.path.basename(incr)),
1151+
stderr=stdout)
1152+
d = filecmp.dircmp(full, incr, self.ignored_differences())
1153+
if not have_same_trees(full, incr, d):
1154+
message = ("Dirs differ: %s vs. %s" %
1155+
(os.path.relpath(full),
1156+
os.path.basename(incr)))
1157+
if self.expect_determinism():
1158+
raise EarlyExit(ActionResult(Result.FAIL, message))
1159+
else:
1160+
common.debug_print(message, stderr=stdout)
1161+
1162+
def excluded_by_limit(self, limits):
1163+
for (kind, value) in limits.items():
1164+
if self.action.get(kind) != value:
1165+
return True
1166+
return False
1167+
1168+
def build(self, stdout=sys.stdout):
1169+
action_result = ActionResult(Result.PASS, "")
1170+
try:
1171+
if 'incremental' in self.project:
1172+
for vers in self.project['incremental']:
1173+
incr = self.project['incremental'][vers]
1174+
if 'limit' in incr and self.excluded_by_limit(incr['limit']):
1175+
continue
1176+
ident = "%s-incr-%s" % (self.project['path'], vers)
1177+
action_result = self.build_incremental(ident,
1178+
incr['commits'],
1179+
stdout=stdout)
1180+
except EarlyExit as error:
1181+
action_result = error.value
1182+
if self.show_stats is not None:
1183+
common.debug_print("Stats summary:", stderr=stdout)
1184+
common.debug_print(self.stats_summ.dump(self.show_stats), stderr=stdout)
1185+
return action_result
1186+
1187+
def dispatch(self, identifier, incremental, stdout=sys.stdout, stderr=sys.stderr):
1188+
try:
1189+
dispatch(self.root_path, self.project, self.action,
1190+
self.swiftc,
1191+
self.swift_version,
1192+
self.sandbox_profile_xcodebuild,
1193+
self.sandbox_profile_package,
1194+
self.added_swift_flags,
1195+
should_strip_resource_phases=False,
1196+
stdout=stdout, stderr=stderr,
1197+
incremental=incremental,
1198+
stats_path=self.stats_path)
1199+
except common.ExecuteCommandFailure as error:
1200+
return self.failed(identifier, error)
1201+
else:
1202+
return self.succeeded(identifier)
1203+
1204+
def dispatch_or_raise(self, identifier, incremental,
1205+
stdout=sys.stdout, stderr=sys.stderr):
1206+
time.sleep(2)
1207+
action_result = self.dispatch(identifier, incremental=incremental,
1208+
stdout=stdout, stderr=stderr)
1209+
time.sleep(2)
1210+
if action_result.result not in [ResultEnum.PASS,
1211+
ResultEnum.XFAIL]:
1212+
raise EarlyExit(action_result)
1213+
return action_result
1214+
1215+
def build_incremental(self, identifier, commits, stdout=sys.stdout):
1216+
if os.path.exists(self.incr_path):
1217+
shutil.rmtree(self.incr_path)
1218+
os.makedirs(self.incr_path)
1219+
prev = None
1220+
seq = 0
1221+
action_result = ActionResult(Result.PASS, "")
1222+
for commit in commits:
1223+
sha = commit
1224+
stats = None
1225+
if type(commit) is dict:
1226+
sha = commit['commit']
1227+
stats = commit.get('stats', None)
1228+
proj = self.project['path']
1229+
ident = "%s-%03d-%.7s" % (identifier, seq, sha)
1230+
if prev is None:
1231+
common.debug_print("Doing full build #%03d of %s: %.7s" %
1232+
(seq, proj, sha), stderr=stdout)
1233+
self.checkout_sha(sha, stdout=stdout, stderr=stdout)
1234+
action_result = self.dispatch_or_raise(ident, incremental=False,
1235+
stdout=stdout, stderr=stdout)
1236+
self.save_build_state(seq, 'full', sha, None, stdout=stdout)
1237+
else:
1238+
common.debug_print("Doing incr build #%d of %s: %.7s -> %.7s" %
1239+
(seq, proj, prev, sha), stderr=stdout)
1240+
common.git_checkout(sha, self.proj_path, stdout=stdout, stderr=stdout)
1241+
common.git_submodule_update(self.proj_path, stdout=stdout, stderr=stdout)
1242+
action_result = self.dispatch_or_raise(ident, incremental=True,
1243+
stdout=stdout, stderr=stdout)
1244+
self.save_build_state(seq, 'incr', sha, stats, stdout=stdout)
1245+
prev = sha
1246+
seq += 1
1247+
return action_result

0 commit comments

Comments
 (0)