Add clangc flag to runner.py

'--clangc' provides a way to build projects with a custom clang binary.
With '--clang-source-path', the 'run' script builds clang binary from
the clang source in the path and provides this binary path to 'runner.py'
using the '--clangc' flag.

Author: rapidsna

clang_installer.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# ===--- clang_installer.py --------------------------------------------------------===
+#
+#  This source file is part of the Swift.org open source project
+#
+#  Copyright (c) 2021 Apple Inc. and the Swift project authors
+#  Licensed under Apache License v2.0 with Runtime Library Exception
+#
+#  See https://swift.org/LICENSE.txt for license information
+#  See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ===----------------------------------------------------------------------===
+
+import common
+import os
+import signal
+
+class ClangInstaller(object):
+    def __init__(self, prebuilt):
+        self.clang_paths = []
+        for sdk in ['macosx']:
+            try:
+                sdk_clang_path = common.check_execute_output(['xcrun', '--sdk', sdk, '--find', 'clang']).strip()
+            except:
+                pass
+            else:
+                self.clang_paths.append(sdk_clang_path)
+
+        self.local_clang_path = prebuilt
+        self.root_path = common.private_workspace('project_cache')
+        self.source_path = os.path.join(self.root_path, 'llvm-project')
+
+        self._default_sig_handlers = {
+            signal.SIGINT: signal.getsignal(signal.SIGINT),
+            signal.SIGTSTP: signal.getsignal(signal.SIGTSTP)
+        }
+
+        def exit_handler(signal_received, frame):
+            self.clear()
+            default_sig_handler = self._default_sig_handlers[signal_received]
+            signal.signal(signal_received, default_sig_handler)
+            default_sig_handler(signal_received, frame)
+
+        # run exit_handler() function when CTRL-C or CTRL-Z is received
+        signal.signal(signal.SIGINT, exit_handler)
+        signal.signal(signal.SIGTSTP, exit_handler)
+
+    @staticmethod
+    def orig_postfix():
+        return ".compatsuite.orig"
+
+    def install(self):
+        for clang_path in self.clang_paths:
+            clang_orig_path = clang_path + self.orig_postfix()
+            if os.path.isfile(clang_orig_path):
+                if os.path.islink(clang_path):
+                    common.check_execute(['rm', clang_path])
+                elif os.path.isfile(clang_path):
+                    continue
+            else:
+                common.check_execute(['mv', clang_path, clang_orig_path])
+
+            common.check_execute(['ln', '-s', self.local_clang_path, clang_path])
+
+    def clear(self):
+        if not self.clang_paths:
+            return
+        print('Restoring swapped clang binaries.')
+        for clang_path in self.clang_paths:
+            clang_orig_path = clang_path + self.orig_postfix()
+            if not os.path.isfile(clang_orig_path):
+                continue
+
+            if os.path.isfile(clang_path):
+                if os.path.islink(clang_path):
+                    common.check_execute(['rm', clang_path])
+                else:
+                    continue
+
+            common.check_execute(['mv', clang_orig_path, clang_path])
+        self.clang_paths = []
+
+    def __del__(self):
+        self.clear()

project_future.py

@@ -583,6 +583,9 @@ def add_arguments(parser):
     parser.add_argument("--report-time-path",
                         help='export time for building each xcode build target to the specified json file',
                         type=os.path.abspath)
+    parser.add_argument("--clangc",
+                        help='clang executable to temporarily be swapped with the clang in the toolchain',
+                        type=os.path.abspath)
 
 def add_minimal_arguments(parser):
     """Add common arguments to parser."""

run

@@ -37,11 +37,19 @@ def main():
     if not args.skip_build:
         build_swift_toolchain(workspace, args)
 
+    clang_flag = []
+    if args.clang_source_path or args.clangc:
+        clangc = args.clangc
+        if args.clang_source_path:
+            clangc = build_clang(workspace, args.clang_source_path)
+        if clangc:
+            clang_flag = ['--clangc', clangc]
+
     if not args.skip_runner:
         if args.test_incremental:
-            execute_build_incremental(workspace, args)
+            execute_build_incremental(workspace, args, clang_flag)
         else:
-            execute_runner(workspace, args)
+            execute_runner(workspace, args, clang_flag)
 
     return 0
 
@@ -74,6 +82,15 @@ def parse_args():
     parser.add_argument('--swiftc',
                         metavar='PATH',
                         help='swiftc executable')
+    clang_arguments = parser.add_mutually_exclusive_group()
+    clang_arguments.add_argument('--clang-source-path',
+                                 metavar="PATH",
+                                 help='path to llvm-project to build',
+                                 type=os.path.abspath)
+    clang_arguments.add_argument('--clangc',
+                                 metavar="PATH",
+                                 help='clang executable',
+                                 type=os.path.abspath)
     parser.add_argument('--skip-build',
                         action='store_true')
     parser.add_argument('--skip-ci-steps',
@@ -161,7 +178,7 @@ def get_sandbox_profile_flags_test():
     return sandbox_flags
 
 
-def execute_runner(workspace, args):
+def execute_runner(workspace, args, clang_flag):
     swiftc_path = get_swiftc_path(workspace, args.swiftc)
     if args.test:
         action_filter = 'action.startswith("TestSwiftPackage")'
@@ -193,10 +210,12 @@ def execute_runner(workspace, args):
     if args.build_config:
         runner_command += ['--build-config=%s' % args.build_config]
 
+    runner_command += clang_flag
+
     common.check_execute(runner_command, timeout=9999999)
 
 
-def execute_build_incremental(workspace, args):
+def execute_build_incremental(workspace, args, clang_flag):
     swiftc_path = get_swiftc_path(workspace, args.swiftc)
     runner_command = [
         './build_incremental.py',
@@ -211,6 +230,7 @@ def execute_build_incremental(workspace, args):
     ]
     if args.sandbox:
         runner_command += get_sandbox_profile_flags()
+    runner_command += clang_flag
     common.check_execute(runner_command, timeout=9999999)
 
 def get_preset_name(args):
@@ -270,6 +290,28 @@ def build_swift_toolchain(workspace, args):
         raise common.UnsupportedPlatform
     common.check_execute(build_command, timeout=9999999)
 
+def build_clang(workspace, source_path):
+    build_path = os.path.join(workspace, 'build_clang_source_compat')
+    source_path = os.path.join(source_path, 'llvm')
+    common.check_execute(['mkdir', '-p', build_path])
+
+    with common.DirectoryContext(build_path):
+        try:
+            ninja_path = common.check_execute_output(['xcrun', '--find', 'ninja']).strip().decode("utf-8")
+            cmake_command = [
+                'xcrun', 'cmake', '-G', 'Ninja',
+                '-DCMAKE_MAKE_PROGRAM={}'.format(ninja_path),
+                '-DLLVM_ENABLE_PROJECTS=clang;llvm',
+                '-DCMAKE_BUILD_TYPE=Release',
+                '-DCLANG_APPLE_BUILD_VERSION_STRING=13000000',
+                '-DLLVM_TARGETS_TO_BUILD=X86;AArch64;ARM',
+                source_path]
+            common.check_execute(cmake_command)
+            common.check_execute(['xcrun', 'ninja'])
+        except:
+            raise
+
+    return os.path.join(build_path, 'bin', 'clang')
 
 if __name__ == '__main__':
     sys.exit(main())

runner.py

@@ -19,6 +19,7 @@
 
 import common
 import project_future
+import clang_installer
 
 
 def parse_args():
@@ -48,6 +49,11 @@ def main():
     if args.report_time_path:
         time_reporter = project_future.TimeReporter(args.report_time_path)
 
+    clangc_installer = None
+    if args.clangc:
+        clangc_installer = clang_installer.ClangInstaller(args.clangc)
+        clangc_installer.install()
+
     index = json.loads(open(args.projects).read())
     result = project_future.ProjectListBuilder(
         args.include_repos,
@@ -81,6 +87,10 @@ def main():
         index
     ).build()
     common.debug_print(str(result))
+
+    if clangc_installer:
+        clangc_installer.clear()
+
     return 0 if result.result in [project_future.ResultEnum.PASS,
                                   project_future.ResultEnum.XFAIL] else 1