Skip to content

Commit 6ee3748

Browse files
graydongraydon
committed
Add run_cperf script
1 parent f838397 commit 6ee3748

File tree

1 file changed

+357
-0
lines changed

1 file changed

+357
-0
lines changed

‎run_cperf

Lines changed: 357 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,357 @@
1+
#!/usr/bin/env python
2+
# ===--- run --------------------------------------------------------------===
3+
#
4+
# This source file is part of the Swift.org open source project
5+
#
6+
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See https://swift.org/LICENSE.txt for license information
10+
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
# ===----------------------------------------------------------------------===
13+
14+
"""A run script to be executed as a pull-request test on Jenkins."""
15+
16+
import argparse
17+
import os
18+
import platform
19+
import shutil
20+
import sys
21+
22+
import common
23+
24+
25+
def setup_workspace(instance, workspace, args):
26+
if args.clean:
27+
shutil.rmtree(workspace)
28+
if not os.path.exists(workspace):
29+
os.makedirs(workspace)
30+
swift = os.path.join(workspace, "swift")
31+
if not os.path.exists(swift):
32+
common.git_clone(args.url, swift, tree=args.swift_branch)
33+
common.check_execute(['utils/update-checkout',
34+
'--clone', '--reset-to-remote',
35+
'--clean', '--scheme', args.swift_branch],
36+
cwd=swift)
37+
if instance == 'new':
38+
command_fetch = ['git', '-C', swift, 'fetch', 'origin',
39+
'pull/%d/merge' % args.setup_workspaces_for_pr]
40+
common.check_execute(command_fetch)
41+
common.git_checkout('FETCH_HEAD', swift)
42+
43+
44+
def validate_workspace(instance, workspace, args):
45+
if not os.path.exists(workspace):
46+
raise RuntimeError("Missing %s workspace dir %s"
47+
% (instance, workspace))
48+
swift = os.path.join(workspace, "swift")
49+
if not os.path.exists(swift):
50+
raise RuntimeError("Missing swift checkout in workspace dir %s"
51+
% workspace)
52+
53+
54+
def main():
55+
common.debug_print('** RUN PULL-REQUEST CPERF **')
56+
os.chdir(os.path.dirname(__file__))
57+
58+
args = parse_args()
59+
instances = ['old', 'new']
60+
configs = ['debug', 'wmo-onone', 'release']
61+
62+
for instance in instances:
63+
workspace = common.private_workspace(instance)
64+
65+
if args.setup_workspaces_for_pr:
66+
setup_workspace(instance, workspace, args)
67+
validate_workspace(instance, workspace, args)
68+
69+
if not args.skip_build:
70+
build_swift_toolchain(workspace, args)
71+
72+
if not args.skip_runner:
73+
execute_runner(instance, workspace, configs, args)
74+
75+
76+
def get_sandbox_profile_flags():
77+
sandbox_flags = []
78+
workspace = common.private_workspace('.')
79+
sscss = os.path.join(workspace, "swift-source-compat-suite-sandbox")
80+
if not os.path.exists(sscss):
81+
raise RuntimeError("Missing sandbox dir %s" % sscss)
82+
83+
if platform.system() == 'Darwin':
84+
sandbox_flags += [
85+
'--sandbox-profile-xcodebuild',
86+
os.path.join(sscss, 'sandbox_xcodebuild.sb'),
87+
'--sandbox-profile-package',
88+
os.path.join(sscss, 'sandbox_package.sb')
89+
]
90+
elif platform.system() == 'Linux':
91+
sandbox_flags += [
92+
'--sandbox-profile-package',
93+
os.path.join(sscss, 'sandbox_package_linux.profile')
94+
]
95+
else:
96+
raise common.UnsupportedPlatform
97+
return sandbox_flags
98+
99+
100+
def get_swiftc_path(workspace):
101+
if platform.system() == 'Darwin':
102+
swiftc_path = os.path.join(workspace, 'build/compat_macos/install/toolchain/usr/bin/swiftc')
103+
elif platform.system() == 'Linux':
104+
swiftc_path = os.path.join(workspace, 'build/compat_linux/install/usr/bin/swiftc')
105+
else:
106+
raise common.UnsupportedPlatform
107+
return swiftc_path
108+
109+
110+
def build_swift_toolchain(workspace, args):
111+
if platform.system() == 'Darwin':
112+
build_command = [
113+
os.path.join(workspace, 'swift/utils/build-script'),
114+
'--release',
115+
'--no-assertions',
116+
'--build-ninja',
117+
'--llbuild',
118+
'--swiftpm',
119+
'--ios',
120+
'--tvos',
121+
'--watchos',
122+
'--skip-build-benchmarks',
123+
'--build-subdir=compat_macos',
124+
'--compiler-vendor=apple',
125+
'--',
126+
'--darwin-install-extract-symbols',
127+
'--darwin-toolchain-alias=swift',
128+
'--darwin-toolchain-bundle-identifier=org.swift.compat-macos',
129+
'--darwin-toolchain-display-name-short=Swift Development Snapshot'
130+
'--darwin-toolchain-display-name=Swift Development Snapshot',
131+
'--darwin-toolchain-name=swift-DEVELOPMENT-SNAPSHOT',
132+
'--darwin-toolchain-version=3.999.999',
133+
'--install-llbuild',
134+
'--install-swift',
135+
'--install-swiftpm',
136+
'--install-destdir={}/build/compat_macos/install'.format(workspace),
137+
'--install-prefix=/toolchain/usr',
138+
'--install-symroot={}/build/compat_macos/symroot'.format(workspace),
139+
'--installable-package={}/build/compat_macos/root.tar.gz'.format(workspace),
140+
'--llvm-install-components=libclang;libclang-headers',
141+
'--swift-install-components=compiler;clang-builtin-headers;stdlib;sdk-overlay;license;sourcekit-xpc-service;swift-remote-mirror;swift-remote-mirror-headers',
142+
'--symbols-package={}/build/compat_macos/root-symbols.tar.gz'.format(workspace),
143+
'--verbose-build',
144+
'--reconfigure',
145+
]
146+
elif platform.system() == 'Linux':
147+
build_command = [
148+
os.path.join(workspace, 'swift/utils/build-script'),
149+
'--release',
150+
'--no-assertions',
151+
'--build-ninja',
152+
'--llbuild',
153+
'--swiftpm',
154+
'--foundation',
155+
'--libdispatch',
156+
'--xctest',
157+
'--skip-build-benchmarks',
158+
'--build-subdir=compat_linux',
159+
'--',
160+
'--install-foundation',
161+
'--install-libdispatch',
162+
'--install-llbuild',
163+
'--install-swift',
164+
'--install-swiftpm',
165+
'--install-xctest',
166+
'--install-destdir={}/build/compat_linux/install'.format(workspace),
167+
'--install-prefix=/usr',
168+
'--installable-package={}/build/compat_linux/root.tar.gz'.format(workspace),
169+
'--swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;swift-remote-mirror;sdk-overlay;license',
170+
'--verbose-build',
171+
'--reconfigure',
172+
]
173+
else:
174+
raise common.UnsupportedPlatform
175+
common.check_execute(build_command, timeout=9999999)
176+
177+
178+
def get_projects(suite):
179+
if suite == 'full':
180+
return 'projects.json'
181+
elif suite == 'smoketest':
182+
return 'projects-cperf-smoketest.json'
183+
else:
184+
raise ValueError("Unknown suite: " + suite)
185+
186+
187+
def get_stats_dir(instance, variant):
188+
return os.path.join(os.getcwd(),
189+
"-".join(["stats", instance, variant]))
190+
191+
192+
def get_actual_config_and_flags(config, stats):
193+
flags = ("-j 1 -num-threads 1 -stats-output-dir '%s'" % stats)
194+
# Handle wmo-onone as a pseudo-config
195+
if config == 'wmo-onone':
196+
flags += ' -wmo -Onone '
197+
config = 'release'
198+
return (config, flags)
199+
200+
201+
def get_variant(config, args):
202+
return '-'.join([args.suite, args.swift_branch, config])
203+
204+
205+
def execute_runner(instance, workspace, configs, args):
206+
projects = get_projects(args.suite)
207+
swiftc_path = get_swiftc_path(workspace)
208+
for config in configs:
209+
variant = get_variant(config, args)
210+
stats = get_stats_dir(instance, variant)
211+
(config, flags) = get_actual_config_and_flags(config, stats)
212+
runner_command = [
213+
'./runner.py',
214+
'--swiftc', swiftc_path,
215+
'--projects', projects,
216+
'--build-config', config,
217+
'--swift-version', '3',
218+
'--include-actions', 'action.startswith("Build")',
219+
'--swift-branch', args.swift_branch,
220+
'--add-swift-flags', flags,
221+
]
222+
if args.sandbox:
223+
runner_command += get_sandbox_profile_flags()
224+
if args.verbose:
225+
runner_command += ["--verbose"]
226+
common.check_execute(runner_command, timeout=9999999)
227+
228+
229+
def get_table_name(reference, subset, variant):
230+
return os.path.join(
231+
os.getcwd(),
232+
('-'.join([reference, subset, variant]) + '.md'))
233+
234+
235+
def get_baseline_name(variant):
236+
return os.path.join(os.cwd(),
237+
'cperf-baselines', variant + '.csv')
238+
239+
240+
def analyze_results(instances, configs, args):
241+
old_ws = common.private_workspace(instances[0])
242+
process_stats = os.path.join(old_ws, 'swift/utils/process-stats-dir.py')
243+
references = ('head', 'baseline')
244+
subsets = ('counters', 'timers')
245+
246+
common_args = [process_stats,
247+
'--markdown', '--group-by-module',
248+
'--sort-by-delta-pct', '--sort-descending']
249+
250+
returncodes = []
251+
for config in configs:
252+
variant = get_variant(config, args)
253+
stats_dirs = [get_stats_dir(i, variant)
254+
for i in instances]
255+
returncodes.append(
256+
common.execute(
257+
common_args +
258+
['--output', get_table_name('head', 'counters', variant),
259+
'--exclude-timers',
260+
'--compare-stats-dirs'] + stats_dirs))
261+
returncodes.append(
262+
common.execute(
263+
common_args +
264+
['--output', get_table_name('head', 'timers', variant),
265+
'--select-stat', 'driver.*user',
266+
'--compare-stats-dirs'] + stats_dirs))
267+
baseline = get_baseline_name(variant)
268+
if os.path.exists(baseline):
269+
(_, new) = stats_dirs
270+
returncodes.append(
271+
common.execute(
272+
common_args +
273+
['--output',
274+
get_table_name('baseline', 'counters', variant),
275+
'--exclude-timers',
276+
'--compare-to-csv-baseline', baseline, new]))
277+
returncodes.append(
278+
common.execute(
279+
common_args +
280+
['--output',
281+
get_table_name('baseline', 'timers', variant),
282+
'--select-stat', 'driver.*user',
283+
'--compare-to-csv-baseline', baseline, new]))
284+
285+
out = args.output
286+
regressions = any(x != 0 for x in returncodes)
287+
if regressions:
288+
args.output.write("**Regressions found (see below)**")
289+
else:
290+
args.output.write("No regressions above thresholds")
291+
for config in configs:
292+
variant = get_variant(config, args)
293+
for reference in references:
294+
out.write("\n")
295+
out.write("## PR vs. %s (%s)\n" % (reference, variant))
296+
out.write("\n")
297+
for subset in subsets:
298+
out.write("\n")
299+
out.write("### PR vs. %s %s changes (%s)\n" %
300+
(reference, subset, variant))
301+
out.write("\n")
302+
table = get_table_name('baseline', 'counters', variant)
303+
if os.path.exists(table):
304+
with open(table) as t:
305+
out.write(t.read())
306+
os.unlink(table)
307+
else:
308+
out.write("No analysis available\n")
309+
310+
out.write('<details>')
311+
for config in configs:
312+
variant = get_variant(config, args)
313+
baseline = get_baseline_name(variant)
314+
if os.path.exists(baseline):
315+
out.write("Last baseline commit on %s" %
316+
os.path.basename(baseline))
317+
out.write("\n\n<pre>\n")
318+
out.write(common.check_execute_output([
319+
'git', 'log', '--pretty=medium', '-1', baseline
320+
]))
321+
out.write("\n</pre>\n")
322+
else:
323+
out.write("No baseline file %s found" %
324+
os.path.basename(baseline))
325+
out.write('</details>')
326+
327+
return regressions
328+
329+
330+
def parse_args():
331+
parser = argparse.ArgumentParser()
332+
parser.add_argument('swift_branch')
333+
parser.add_argument('--sandbox', action='store_true')
334+
parser.add_argument('--url', type=str,
335+
default="https://github.com/apple/swift")
336+
parser.add_argument('--suite',
337+
metavar='(smoketest|full)',
338+
help='cperf suite to run',
339+
default='smoketest')
340+
parser.add_argument("--verbose",
341+
action='store_true')
342+
parser.add_argument('--skip-build',
343+
action='store_true')
344+
parser.add_argument('--skip-runner',
345+
action='store_true')
346+
parser.add_argument('--output',
347+
type=str,
348+
default='comment.md')
349+
parser.add_argument('--clean',
350+
action='store_true')
351+
parser.add_argument('--setup-workspaces-for-pr',
352+
type=int, default=None)
353+
return parser.parse_args()
354+
355+
356+
if __name__ == '__main__':
357+
sys.exit(main())

0 commit comments

Comments
 (0)