Add '--clang' flag to runner.py

'--clang' provides a way to build Xcode projects with a custom clang binary,
by adding 'CC=clangc' flag to xcode.

With '--clang-source-path', the 'run' script builds clang binary from
the clang source in the path and provides this binary path to 'runner.py'
using the '--clang' flag.

Author: rapidsna

project_future.py

@@ -584,6 +584,9 @@ def add_arguments(parser):
     parser.add_argument("--report-time-path",
                         help='export time for building each xcode build target to the specified json file',
                         type=os.path.abspath)
+    parser.add_argument("--clang",
+                        help='clang executable to build Xcode projects',
+                        type=os.path.abspath)
     parser.add_argument("--job-type",
                         help="The type of job to run. This influences which projects are XFailed, for example the stress tester tracks its XFails under a different job type. Defaults to 'source-compat'.",
                         default='source-compat')

run

@@ -37,11 +37,19 @@ def main():
     if not args.skip_build:
         build_swift_toolchain(workspace, args)
 
+    additional_runner_args = []
+    if args.clang_source_path or args.clang:
+        clang = args.clang
+        if args.clang_source_path:
+            clang = build_clang(workspace, args)
+        if clang:
+            additional_runner_args = ['--clang', clang]
+
     if not args.skip_runner:
         if args.test_incremental:
-            execute_build_incremental(workspace, args)
+            execute_build_incremental(workspace, args, additional_runner_args)
         else:
-            execute_runner(workspace, args)
+            execute_runner(workspace, args, additional_runner_args)
 
     return 0
 
@@ -57,10 +65,10 @@ def parse_args():
     parser.add_argument("--verbose",
                         action='store_true')
     parser.add_argument("--assertions",
-                        help='Build Swift with asserts',
+                        help='Build Swift and/or Clang with asserts',
                         action='store_true')
     parser.add_argument("--debug",
-                        help='Build Swift in debug mode',
+                        help='Build Swift and/or Clang in debug mode',
                         action='store_true')
     parser.add_argument("--filter-by-tag",
                         metavar='TAG',
@@ -74,6 +82,17 @@ def parse_args():
     parser.add_argument('--swiftc',
                         metavar='PATH',
                         help='swiftc executable')
+    clang_arguments = parser.add_mutually_exclusive_group()
+    clang_arguments.add_argument('--clang-source-path',
+                                 metavar="PATH",
+                                 help='Path to llvm-project source. Build a new clang '
+                                      'executable from the given path and uses it to '
+                                      'build Xcode projects',
+                                 type=os.path.abspath)
+    clang_arguments.add_argument('--clang',
+                                 metavar="PATH",
+                                 help='clang executable to build Xcode projects',
+                                 type=os.path.abspath)
     parser.add_argument('--skip-build',
                         action='store_true')
     parser.add_argument('--skip-ci-steps',
@@ -161,7 +180,7 @@ def get_sandbox_profile_flags_test():
     return sandbox_flags
 
 
-def execute_runner(workspace, args):
+def execute_runner(workspace, args, additional_runner_args):
     swiftc_path = get_swiftc_path(workspace, args.swiftc)
     if args.test:
         action_filter = 'action.startswith("TestSwiftPackage")'
@@ -193,10 +212,12 @@ def execute_runner(workspace, args):
     if args.build_config:
         runner_command += ['--build-config=%s' % args.build_config]
 
+    runner_command += additional_runner_args
+
     common.check_execute(runner_command, timeout=9999999)
 
 
-def execute_build_incremental(workspace, args):
+def execute_build_incremental(workspace, args, additional_runner_args):
     swiftc_path = get_swiftc_path(workspace, args.swiftc)
     runner_command = [
         './build_incremental.py',
@@ -211,6 +232,7 @@ def execute_build_incremental(workspace, args):
     ]
     if args.sandbox:
         runner_command += get_sandbox_profile_flags()
+    runner_command += additional_runner_args
     common.check_execute(runner_command, timeout=9999999)
 
 def get_preset_name(args):
@@ -270,6 +292,34 @@ def build_swift_toolchain(workspace, args):
         raise common.UnsupportedPlatform
     common.check_execute(build_command, timeout=9999999)
 
+def build_clang(workspace, args):
+    build_path = os.path.join(workspace, 'build_clang_source_compat')
+    source_path = os.path.join(args.clang_source_path, 'llvm')
+    common.check_execute(['mkdir', '-p', build_path])
+
+    with common.DirectoryContext(build_path):
+        # Get path to the ninja binary
+        ninja_path = common.check_execute_output(['xcrun', '--find', 'ninja']).strip().decode("utf-8")
+
+        build_type = "Debug" if args.debug else "Release"
+        assert_on = "True" if args.assertions or args.debug else "False"
+
+        # Generate a Ninja project with CMake
+        cmake_command = [
+            'xcrun', 'cmake', '-G', 'Ninja',
+            '-DCMAKE_MAKE_PROGRAM={}'.format(ninja_path),
+            '-DLLVM_ENABLE_PROJECTS=clang;llvm',
+            '-DCMAKE_BUILD_TYPE={}'.format(build_type),
+            '-DLLVM_ENABLE_ASSERTIONS={}'.format(assert_on),
+            '-DCLANG_APPLE_BUILD_VERSION_STRING=13000000',
+            '-DLLVM_TARGETS_TO_BUILD=X86;AArch64;ARM',
+            source_path]
+        common.check_execute(cmake_command)
+
+        # Build the Ninja project to produce the clang executable
+        common.check_execute(['xcrun', 'ninja'])
+
+    return os.path.join(build_path, 'bin', 'clang')
 
 if __name__ == '__main__':
     sys.exit(main())

runner.py

@@ -42,6 +42,10 @@ def main():
     xcodebuild_flags = args.add_xcodebuild_flags
     xcodebuild_flags += (' ' if xcodebuild_flags else '') + 'DEBUG_INFORMATION_FORMAT=dwarf'
 
+    # Use clang for building xcode projects.
+    if args.clang:
+        xcodebuild_flags += ' CC=%s' % args.clang
+
     swift_flags = args.add_swift_flags
 
     time_reporter = None