Merge pull request #24330 from apple/preset-for-llvm-tsan-libdispatch-tests

[build][Linux] Create preset to run LLVM's TSan libdispatch tests

Author: yln

utils/build-presets.ini

@@ -1039,6 +1039,21 @@ skip-build-swift
 skip-build-cmark
 llvm-include-tests
 
+
+[preset: buildbot_incremental_linux,tsan-libdispatch-test]
+mixin-preset=buildbot_incremental_linux,llvm-only
+build-subdir=buildbot_incremental_tsan_libdispatch_test
+
+llvm-cmake-options=-DLLVM_INSTALL_UTILS=ON
+llvm-install-components=all
+
+libdispatch-cmake-options=-DENABLE_SWIFT=OFF
+libdispatch
+install-libdispatch
+
+tsan-libdispatch-test
+
+
 [preset: buildbot_incremental_linux,asan]
 mixin-preset=buildbot_incremental_linux
 build-subdir=buildbot_incremental_asan

utils/build-script

@@ -342,6 +342,7 @@ class BuildScriptInvocation(object):
             'build_indexstoredb',
             'build_sourcekitlsp',
             'build_toolchainbenchmarks',
+            'tsan_libdispatch_test',
         ]
         has_target_needing_toolchain = \
             bool(sum(getattr(args, x) for x in targets_needing_toolchain))
@@ -897,6 +898,8 @@ class BuildScriptInvocation(object):
             product_classes.append(products.SourceKitLSP)
         if self.args.build_toolchainbenchmarks:
             product_classes.append(products.Benchmarks)
+        if self.args.tsan_libdispatch_test:
+            product_classes.append(products.TSanLibDispatch)
         return product_classes
 
     def execute(self):

utils/build-script-impl

@@ -2750,6 +2750,7 @@ for host in "${ALL_HOSTS[@]}"; do
                   call rm -rf "${LIBDISPATCH_BUILD_DIR}"
 
                   cmake_options=(
+                    -DENABLE_SWIFT=YES
                     ${cmake_options[@]}
                     -DCMAKE_BUILD_TYPE:STRING="${LIBDISPATCH_BUILD_TYPE}"
                     -DCMAKE_C_COMPILER:PATH="${LLVM_BIN}/clang"
@@ -2758,7 +2759,6 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
                     -DCMAKE_INSTALL_LIBDIR:PATH="lib"
 
-                    -DENABLE_SWIFT=YES
                     -DSwift_DIR="${SWIFT_BUILD_PATH}/lib/cmake/swift"
 
                     -DENABLE_TESTING=YES
@@ -3479,7 +3479,11 @@ for host in "${ALL_HOSTS[@]}"; do
                 if [[ -z "${LLVM_INSTALL_COMPONENTS}" ]] ; then
                     continue
                 fi
-                INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/;/ install-/g')
+                if [[ "${LLVM_INSTALL_COMPONENTS}" == "all" ]]; then
+                    INSTALL_TARGETS=install
+                else
+                    INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/;/ install-/g')
+                fi
                 ;;
             libcxx)
                 if [[ -z "${INSTALL_LIBCXX}" ]] ; then

utils/build_swift/driver_arguments.py

@@ -788,6 +788,15 @@ def create_argument_parser():
            help='if the Swift Benchmark Suite is run after building, run N '
                 'iterations with -Onone')
 
+    # We want to run the TSan (compiler-rt) libdispatch tests on Linux, where
+    # libdispatch is just another library and not available by default. To do
+    # so we build Clang/LLVM/libdispatch and use it to compile/run the TSan
+    # libdispatch tests.
+    option('--tsan-libdispatch-test', toggle_true,
+           help='Builds a new toolchain including the libdispatch C library. '
+                'Then re-builds the TSan runtime (compiler-rt) using this '
+                'freshly-built Clang and runs the TSan libdispatch tests.')
+
     option('--skip-test-osx', toggle_false('test_osx'),
            help='skip testing Swift stdlibs for Mac OS X')
     option('--skip-test-linux', toggle_false('test_linux'),

utils/build_swift/tests/expected_options.py

@@ -158,6 +158,7 @@
     'llvm_max_parallel_lto_link_jobs':
         host.max_lto_link_job_counts()['llvm'],
     'llvm_targets_to_build': 'X86;ARM;AArch64;PowerPC;SystemZ;Mips',
+    'tsan_libdispatch_test': False,
     'long_test': False,
     'lto_type': None,
     'show_sdks': False,
@@ -450,6 +451,7 @@ class IgnoreOption(_BaseOption):
     EnableOption('--indexstore-db', dest='build_indexstoredb'),
     EnableOption('--sourcekit-lsp', dest='build_sourcekitlsp'),
     EnableOption('--toolchain-benchmarks', dest='build_toolchainbenchmarks'),
+    EnableOption('--tsan-libdispatch-test'),
     EnableOption('--long-test'),
     EnableOption('--show-sdks'),
     EnableOption('--stress-test'),

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -27,6 +27,7 @@
 from .swiftevolve import SwiftEvolve
 from .swiftpm import SwiftPM
 from .swiftsyntax import SwiftSyntax
+from .tsan_libdispatch import TSanLibDispatch
 from .xctest import XCTest
 
 __all__ = [
@@ -49,4 +50,5 @@
     'IndexStoreDB',
     'SourceKitLSP',
     'Benchmarks',
+    'TSanLibDispatch',
 ]

utils/swift_build_support/swift_build_support/products/tsan_libdispatch.py

@@ -0,0 +1,68 @@
+# swift_build_support/products/tsan_libdispatch.py --------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import os
+
+from . import product
+from .. import shell
+
+
+def join_path(*paths):
+    return os.path.abspath(os.path.join(*paths))
+
+
+class TSanLibDispatch(product.Product):
+    @classmethod
+    def product_source_name(cls):
+        return "tsan-libdispatch-test"
+
+    @classmethod
+    def is_build_script_impl_product(cls):
+        return False
+
+    def build(self, host_target):
+        """Build TSan runtime (compiler-rt)."""
+        rt_source_dir = join_path(self.source_dir, os.pardir, 'compiler-rt')
+        toolchain_path = join_path(self.args.install_destdir, 'usr')
+        clang = join_path(toolchain_path, 'bin', 'clang')
+        clangxx = join_path(toolchain_path, 'bin', 'clang++')
+
+        config_cmd = [
+            'cmake',
+            '-GNinja',
+            '-DCMAKE_PREFIX_PATH=%s' % toolchain_path,
+            '-DCMAKE_C_COMPILER=%s' % clang,
+            '-DCMAKE_CXX_COMPILER=%s' % clangxx,
+            '-DCMAKE_BUILD_TYPE=Release',
+            '-DLLVM_ENABLE_ASSERTIONS=ON',
+            '-DCOMPILER_RT_INCLUDE_TESTS=ON',
+            '-DCOMPILER_RT_BUILD_XRAY=OFF',
+            '-DCOMPILER_RT_INTERCEPT_LIBDISPATCH=ON',
+            '-DCOMPILER_RT_LIBDISPATCH_INSTALL_PATH=%s' % toolchain_path,
+            rt_source_dir]
+        build_cmd = ['ninja', 'tsan']
+
+        # Always rebuild TSan runtime
+        shell.rmtree(self.build_dir)
+        shell.makedirs(self.build_dir)
+
+        with shell.pushd(self.build_dir):
+            shell.call(config_cmd)
+            shell.call(build_cmd)
+
+    def test(self, host_target):
+        """Run check-tsan target with a LIT filter for libdispatch."""
+        cmd = ['ninja', 'check-tsan']
+        env = {'LIT_FILTER': 'libdispatch'}
+
+        with shell.pushd(self.build_dir):
+            shell.call(cmd, env=env)