Skip to content

Commit b728b89

Browse files
gribozavrgribozavr
committed
Merge pull request #1126 from harlanhaskins/profdata-merge
[coverage] Automatic merger for LLVM profile data
2 parents a927dc8 + 2b22040 commit b728b89

File tree

22 files changed

+423
-84
lines changed

22 files changed

+423
-84
lines changed

‎CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ option(SWIFT_INCLUDE_DOCS
5858
"Create targets for building docs."
5959
TRUE)
6060

61+
set(SWIFT_ANALYZE_CODE_COVERAGE FALSE CACHE STRING
62+
"Build Swift with code coverage instrumenting enabled [FALSE, NOT-MERGED, MERGED]")
63+
set_property(CACHE SWIFT_ANALYZE_CODE_COVERAGE PROPERTY
64+
STRINGS FALSE "NOT-MERGED" "MERGED")
65+
6166
set(SWIFT_VERSION "3.0" CACHE STRING
6267
"The user-visible version of the Swift compiler")
6368
set(SWIFT_VENDOR "" CACHE STRING

‎cmake/modules/AddSwift.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ function(_add_variant_c_compile_link_flags
6464
"-m${SWIFT_SDK_${sdk}_VERSION_MIN_NAME}-version-min=${SWIFT_SDK_${sdk}_DEPLOYMENT_VERSION}")
6565

6666
if(analyze_code_coverage)
67-
list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
67+
list(APPEND result "-fprofile-instr-generate"
6868
"-fcoverage-mapping")
6969
endif()
7070
endif()
@@ -111,7 +111,7 @@ function(_add_variant_c_compile_flags
111111
endif()
112112

113113
if(analyze_code_coverage)
114-
list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
114+
list(APPEND result "-fprofile-instr-generate"
115115
"-fcoverage-mapping")
116116
endif()
117117

‎test/CMakeLists.txt

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ if(PYTHONINTERP_FOUND)
117117

118118
set(TEST_MODES optimize_none optimize optimize_unchecked)
119119

120+
120121
foreach(SDK ${SWIFT_SDKS})
121122
foreach(ARCH ${SWIFT_SDK_${SDK}_ARCHITECTURES})
122123
foreach(TEST_MODE ${TEST_MODES})
@@ -196,30 +197,52 @@ if(PYTHONINTERP_FOUND)
196197
"${CMAKE_CURRENT_SOURCE_DIR}/../validation-test/lit.site.cfg.in"
197198
"${validation_test_bin_dir}/lit.site.cfg"
198199
"validation-test${VARIANT_SUFFIX}.lit.site.cfg")
200+
set(profdata_merge_worker
201+
"${CMAKE_CURRENT_SOURCE_DIR}/../utils/profdata_merge/main.py")
202+
203+
if(SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "MERGED")
204+
set(command_profdata_merge_start
205+
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" start
206+
-o "${swift_test_results_dir}"
207+
-l "${swift_test_results_dir}/profdata_merge.log")
208+
set(command_profdata_merge_stop
209+
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" stop)
210+
else()
211+
set(command_profdata_merge_start)
212+
set(command_profdata_merge_stop)
213+
endif()
199214

200215
add_custom_target("check-swift${test_mode_target_suffix}${VARIANT_SUFFIX}"
201216
${command_upload_stdlib}
202217
${command_clean_test_results_dir}
218+
${command_profdata_merge_start}
203219
COMMAND ${lit_command} "${test_bin_dir}"
220+
${command_profdata_merge_stop}
204221
DEPENDS ${test_dependencies}
205222
COMMENT "Running Swift tests for ${VARIANT_TRIPLE}"
206223
${cmake_3_2_USES_TERMINAL})
207224

208225
add_custom_target("check-swift-validation${test_mode_target_suffix}${VARIANT_SUFFIX}"
209226
${command_upload_stdlib}
210227
${command_clean_test_results_dir}
228+
${command_profdata_merge_start}
211229
COMMAND ${lit_command} "${validation_test_bin_dir}"
230+
${command_profdata_merge_stop}
212231
DEPENDS ${test_dependencies} ${validation_test_dependencies}
213232
COMMENT "Running Swift validation tests for ${VARIANT_TRIPLE}"
214233
${cmake_3_2_USES_TERMINAL})
215234

216235
add_custom_target("check-swift-all${test_mode_target_suffix}${VARIANT_SUFFIX}"
217236
${command_upload_stdlib}
218237
${command_clean_test_results_dir}
238+
${command_profdata_merge_start}
219239
COMMAND ${lit_command} "${validation_test_bin_dir}" "${test_bin_dir}"
240+
${command_profdata_merge_stop}
220241
DEPENDS ${test_dependencies} ${validation_test_dependencies}
221242
COMMENT "Running all Swift tests for ${VARIANT_TRIPLE}"
222243
${cmake_3_2_USES_TERMINAL})
244+
245+
223246
endforeach()
224247
endforeach()
225248
endforeach()
@@ -245,6 +268,7 @@ if(PYTHONINTERP_FOUND)
245268

246269
add_custom_target(check-swift-all${test_mode_target_suffix}
247270
DEPENDS "check-swift-all${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
271+
248272
endforeach()
249273

250274
endif()

‎test/Unit/lit.site.cfg.in

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,7 @@ config.build_mode = lit_config.params.get('build_mode', "@LLVM_BUILD_MODE@")
88
config.swift_obj_root = "@SWIFT_BINARY_DIR@"
99
config.target_triple = "@TARGET_TRIPLE@"
1010

11+
config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
12+
1113
# Let the main config do the real work.
1214
lit_config.load_config(config, "@SWIFT_SOURCE_DIR@/test/Unit/lit.cfg")

‎test/lit.cfg

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ import re
2424
import subprocess
2525
import sys
2626
import tempfile
27+
import socket
28+
import glob
2729

30+
import lit
2831
import lit.formats
2932
import lit.util
3033

@@ -129,6 +132,43 @@ if config.test_exec_root is None:
129132

130133
###
131134

135+
class SwiftTest(lit.formats.ShTest, object):
136+
def __init__(self, coverage_mode=None, execute_external=True):
137+
super(SwiftTest, self).__init__(execute_external=execute_external)
138+
if coverage_mode == "FALSE":
139+
self.coverage_mode = None
140+
else:
141+
self.coverage_mode = coverage_mode
142+
143+
def profdir_for_test(self, test):
144+
_, tmp_base = lit.TestRunner.getTempPaths(test)
145+
return tmp_base + ".profdir"
146+
147+
def before_test(self, test, litConfig):
148+
if self.coverage_mode:
149+
profdir = self.profdir_for_test(test)
150+
if not os.path.exists(profdir):
151+
os.makedirs(profdir)
152+
153+
test.config.environment["LLVM_PROFILE_FILE"] = \
154+
os.path.join(profdir, "swift-%p.profraw")
155+
156+
def after_test(self, test, litConfig, result):
157+
if self.coverage_mode == "MERGED":
158+
profdir = self.profdir_for_test(test)
159+
files = glob.glob(os.path.join(profdir, "swift-*.profraw"))
160+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
161+
sock.connect(('localhost', 12400))
162+
sock.send("\n".join(files))
163+
sock.close()
164+
return result
165+
166+
167+
def execute(self, test, litConfig):
168+
self.before_test(test, litConfig)
169+
result = super(SwiftTest, self).execute(test, litConfig)
170+
return self.after_test(test, litConfig, result)
171+
132172
# name: The name of this test suite.
133173
config.name = 'Swift'
134174

@@ -138,7 +178,7 @@ if platform.system() == 'Darwin':
138178
config.environment['TOOLCHAINS'] = 'default'
139179

140180
# testFormat: The test format to use to interpret tests.
141-
config.test_format = lit.formats.ShTest(execute_external=True)
181+
config.test_format = SwiftTest(coverage_mode=config.coverage_mode)
142182

143183
# suffixes: A list of file extensions to treat as test files.
144184
config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m']

‎test/lit.site.cfg.in

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ config.variant_sdk = "@VARIANT_SDK@"
1818
config.swiftlib_dir = "@LIT_SWIFTLIB_DIR@"
1919
config.darwin_xcrun_toolchain = "@SWIFT_DARWIN_XCRUN_TOOLCHAIN@"
2020

21+
config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
22+
2123
if "@SWIFT_ASAN_BUILD@" == "TRUE":
2224
config.available_features.add("asan")
2325
else:
@@ -42,11 +44,6 @@ if "@SWIFT_OPTIMIZED@" == "TRUE":
4244
if "@SWIFT_HAVE_WORKING_STD_REGEX@" == "FALSE":
4345
config.available_features.add('broken_std_regex')
4446

45-
if "@SWIFT_ANALYZE_CODE_COVERAGE@" == "TRUE":
46-
lit_config.useValgrind = True
47-
lit_config.valgrindArgs = [os.path.join(config.swift_src_root,
48-
"utils/use_profdir.py")]
49-
5047
# Let the main config do the real work.
5148
if config.test_exec_root is None:
5249
config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

‎tools/SourceKit/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ macro(add_sourcekit_executable name)
241241

242242
if(SWIFT_ANALYZE_CODE_COVERAGE)
243243
set_property(TARGET "${name}" APPEND_STRING PROPERTY
244-
LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
244+
LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
245245
endif()
246246
endif()
247247
endif()

‎tools/SourceKit/tools/complete-test/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
1717

1818
if(SWIFT_ANALYZE_CODE_COVERAGE)
1919
set_property(TARGET complete-test APPEND_STRING PROPERTY
20-
LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
20+
LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
2121
endif()
2222
endif()
2323

‎tools/SourceKit/tools/sourcekitd-repl/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
1111

1212
if(SWIFT_ANALYZE_CODE_COVERAGE)
1313
set_property(TARGET sourcekitd-repl APPEND_STRING PROPERTY
14-
LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
14+
LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
1515
endif()
1616
endif()
1717

‎tools/SourceKit/tools/sourcekitd-test/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
2525

2626
if(SWIFT_ANALYZE_CODE_COVERAGE)
2727
set_property(TARGET sourcekitd-test APPEND_STRING PROPERTY
28-
LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
28+
LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
2929
endif()
3030
endif()
3131

‎unittests/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ function(add_swift_unittest test_dirname)
2929

3030
if(SWIFT_ANALYZE_CODE_COVERAGE)
3131
set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
32-
LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
32+
LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
3333
endif()
3434
endif()
3535
endfunction()

‎utils/build-script

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -512,11 +512,11 @@ also build for Apple watchos, but disallow tests that require an watchOS device"
512512
action="store_true")
513513

514514
parser.add_argument("--swift-analyze-code-coverage",
515-
help="enable code coverage analysis in Swift",
516-
action="store_const",
517-
const=True,
518-
dest="swift_analyze_code_coverage",
519-
default=False)
515+
help="""enable code coverage analysis in Swift (false, not-merged,
516+
merged).""",
517+
choices=["false", "not-merged", "merged"],
518+
default="false", # so CMake can see the inert mode as a false value
519+
dest="swift_analyze_code_coverage")
520520

521521
parser.add_argument("--build-subdir",
522522
help="""
@@ -726,7 +726,7 @@ the number of parallel build jobs to use""",
726726
swift_build_dir_label = args.swift_build_variant
727727
if args.swift_assertions:
728728
swift_build_dir_label += "Assert"
729-
if args.swift_analyze_code_coverage:
729+
if args.swift_analyze_code_coverage != "false":
730730
swift_build_dir_label += "Coverage"
731731

732732
swift_stdlib_build_dir_label = args.swift_stdlib_build_variant

‎utils/build-script-impl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ KNOWN_SETTINGS=(
7171
llvm-enable-assertions "1" "enable assertions in LLVM and Clang"
7272
swift-build-type "Debug" "the CMake build variant for Swift"
7373
swift-enable-assertions "1" "enable assertions in Swift"
74-
swift-analyze-code-coverage "0" "enable code coverage analysis in Swift"
74+
swift-analyze-code-coverage "not-merged" "Code coverage analysis mode for Swift (false, not-merged, merged). Defaults to false if the argument is not present, and not-merged if the argument is present without a modifier."
7575
swift-stdlib-build-type "Debug" "the CMake build variant for Swift"
7676
swift-stdlib-enable-assertions "1" "enable assertions in Swift"
7777
lldb-build-type "Debug" "the CMake build variant for LLDB"
@@ -1610,7 +1610,7 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_TOOLS_DEPLOYMENT_TARGETS[@]}"
16101610
-DCMAKE_CXX_FLAGS="$(swift_c_flags ${deployment_target})"
16111611
-DCMAKE_BUILD_TYPE:STRING="${SWIFT_BUILD_TYPE}"
16121612
-DLLVM_ENABLE_ASSERTIONS:BOOL=$(true_false "${SWIFT_ENABLE_ASSERTIONS}")
1613-
-DSWIFT_ANALYZE_CODE_COVERAGE:BOOL=$(true_false "${SWIFT_ANALYZE_CODE_COVERAGE}")
1613+
-DSWIFT_ANALYZE_CODE_COVERAGE:STRING=$(toupper "${SWIFT_ANALYZE_CODE_COVERAGE}")
16141614
-DSWIFT_STDLIB_BUILD_TYPE:STRING="${SWIFT_STDLIB_BUILD_TYPE}"
16151615
-DSWIFT_STDLIB_ASSERTIONS:BOOL=$(true_false "${SWIFT_STDLIB_ENABLE_ASSERTIONS}")
16161616
-DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="${native_llvm_tools_path}"

‎utils/profdata_merge/README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Profdata Merge
2+
3+
Because LLVM's instrumented builds produce a `profraw` file every time a
4+
they are executed, the Swift test suite, when run with coverage enabled,
5+
produces thousands of 30MB profile files.
6+
7+
When build-script is run with `--swift-analyze-code-coverage merged`, this
8+
module will get called before the tests run and spin up a small server that
9+
runs alongside the test suite. The server accepts filenames that it receives
10+
after each test directory is finished executing, and feeds them to 10 worker
11+
processes which concurrently merge those files together. Doing so means the
12+
test results directory doesn't balloon in size, as the files are merged together
13+
and removed often.
14+
15+
Once the test suite is finished, it sends a crafted message to the server that
16+
tells it to merge any files left in the queue, then merges all the workers
17+
together into one final file.

‎utils/profdata_merge/__init__.py

Whitespace-only changes.

‎utils/profdata_merge/config.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# utils/profdata_merge/config.py
2+
#
3+
# This source file is part of the Swift.org open source project
4+
#
5+
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
6+
# Licensed under Apache License v2.0 with Runtime Library Exception
7+
#
8+
# See http://swift.org/LICENSE.txt for license information
9+
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
11+
# This file contains the data structure that transforms arguments into usable
12+
# values
13+
14+
import tempfile
15+
import os
16+
17+
18+
class Config():
19+
"""A class to store configuration information specified by command-line
20+
arguments."""
21+
def __init__(self, out_dir, no_remove_files):
22+
self.out_dir = out_dir
23+
self.tmp_dir = tempfile.mkdtemp()
24+
self.pid_file_path = os.path.join(self.out_dir,
25+
"profdata_merge_worker.pid")
26+
self.final_profdata_path = os.path.join(self.out_dir, "swift.profdata")
27+
self.remove_files = not no_remove_files

‎utils/profdata_merge/main.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env python
2+
3+
# utils/profdata_merge/main.py
4+
#
5+
# This source file is part of the Swift.org open source project
6+
#
7+
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
8+
# Licensed under Apache License v2.0 with Runtime Library Exception
9+
#
10+
# See http://swift.org/LICENSE.txt for license information
11+
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
12+
13+
# This module is used to prevent profile data filling up available disk space
14+
# by listening for profile data and merging them into a universal profdata
15+
# file while tests are executing.
16+
# This file invokes the runner after parsing arguments.
17+
18+
from __future__ import print_function
19+
import sys
20+
import argparse
21+
import tempfile
22+
import logging
23+
24+
import runner
25+
26+
SERVER_ADDRESS = ('localhost', 12400)
27+
TESTS_FINISHED_SENTINEL = "PROFDATA_MERGE_WORKER_TESTS_FINISHED_SENTINEL"
28+
29+
if __name__ == "__main__":
30+
if sys.platform != "darwin":
31+
sys.exit("Error: The profile data merge worker requires OS X.")
32+
33+
parser = argparse.ArgumentParser()
34+
parser.add_argument("-l", "--log-file",
35+
help="The file to write logs in debug mode.")
36+
37+
subparsers = parser.add_subparsers()
38+
39+
start = subparsers.add_parser("start")
40+
start.add_argument("-d", "--debug",
41+
help="Run in foreground and report status.",
42+
action="store_true")
43+
start.add_argument("-o", "--output-dir",
44+
help=("The directory to write the PID file" +
45+
"and final profdata file."),
46+
default=tempfile.gettempdir())
47+
start.add_argument("--no-remove",
48+
action="store_true",
49+
help="Don't remove profraw files after merging them.")
50+
start.set_defaults(func=runner.start_server)
51+
52+
stop = subparsers.add_parser("stop")
53+
stop.set_defaults(func=runner.stop_server)
54+
55+
args = parser.parse_args()
56+
57+
log_args = {'level': logging.DEBUG}
58+
if args.log_file:
59+
log_args['filename'] = args.log_file
60+
61+
logging.basicConfig(**log_args)
62+
63+
args.func(args)

0 commit comments

Comments
 (0)