[build-script] Transform IndexStoreDB and SorcekitLSP to use ProductBuilder

utils/build-script

@@ -964,20 +964,16 @@ class BuildScriptInvocation(object):
 
         # Non-build-script-impl products...
         # Note: currently only supports building for the host.
-        for host_target in [self.args.host_target]:
+        for host_target in [
+                StdlibDeploymentTarget.get_target_for_name(
+                    self.args.host_target)]:
             for product_class in product_classes:
                 if product_class.is_build_script_impl_product():
                     continue
-                product_source = product_class.product_source_name()
-                product_name = product_class.product_name()
-                product = product_class(
-                    args=self.args,
-                    toolchain=self.toolchain,
-                    source_dir=self.workspace.source_dir(product_source),
-                    build_dir=self.workspace.build_dir(
-                        host_target, product_name))
-                product.build(host_target)
-                product.test(host_target)
+                builder = product_class.new_builder(
+                    self.args, self.toolchain, self.workspace, host_target)
+                builder.build()
+                builder.test()
 
         # Extract symbols...
         for host_target in all_hosts:

utils/swift_build_support/swift_build_support/products/benchmarks.py

@@ -28,48 +28,53 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        run_build_script_helper(host_target, self, self.args)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return BenchmarksBuilder(cls, args, toolchain, workspace, host)
+
+
+# NOTE: this is very similar to BuildScriptHelperBuilder, but also
+# different enough to justify a different builder altogether.
+class BenchmarksBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        # Our source directory is a subdirectory inside the swift source
+        # directory
+        self.__source_dir = os.path.join(workspace.source_dir('swift'),
+                                         'benchmark')
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+        self.__args = args
 
-    def test(self, host_target):
+    def build(self):
+        # We use a separate python helper to enable quicker iteration when
+        # working on this by avoiding going through build-script to test small
+        # changes.
+        script_path = os.path.join(
+            self.__source_dir, 'scripts', 'build_script_helper.py')
+        toolchain_path = self.__args.install_destdir
+        if platform.system() == 'Darwin':
+            # The prefix is an absolute path, so concatenate without os.path.
+            toolchain_path += \
+                targets.darwin_toolchain_prefix(self.__args.install_prefix)
+        helper_cmd = [
+            script_path,
+            '--verbose',
+            '--package-path', self.__source_dir,
+            '--build-path', self.__build_dir,
+            '--toolchain', toolchain_path,
+        ]
+        shell.call(helper_cmd)
+
+    def test(self):
         """Just run a single instance of the command for both .debug and
            .release.
         """
         cmdline = ['--num-iters=1', 'XorLoop']
-        bench_Onone = os.path.join(self.build_dir, 'bin', 'Benchmark_Onone')
+        bench_Onone = os.path.join(self.__build_dir, 'bin', 'Benchmark_Onone')
         shell.call([bench_Onone] + cmdline)
 
-        bench_O = os.path.join(self.build_dir, 'bin', 'Benchmark_O')
+        bench_O = os.path.join(self.__build_dir, 'bin', 'Benchmark_O')
         shell.call([bench_O] + cmdline)
 
-        bench_Osize = os.path.join(self.build_dir, 'bin', 'Benchmark_Osize')
+        bench_Osize = os.path.join(self.__build_dir, 'bin', 'Benchmark_Osize')
         shell.call([bench_Osize] + cmdline)
-
-
-def run_build_script_helper(host_target, product, args):
-    toolchain_path = args.install_destdir
-    if platform.system() == 'Darwin':
-        # The prefix is an absolute path, so concatenate without os.path.
-        toolchain_path += \
-            targets.darwin_toolchain_prefix(args.install_prefix)
-
-    # Our source_dir is expected to be './$SOURCE_ROOT/benchmarks'. That is due
-    # the assumption that each product is in its own build directory. This
-    # product is not like that and has its package/tools instead in
-    # ./$SOURCE_ROOT/swift/benchmark.
-    package_path = os.path.join(product.source_dir, '..', 'swift', 'benchmark')
-    package_path = os.path.abspath(package_path)
-
-    # We use a separate python helper to enable quicker iteration when working
-    # on this by avoiding going through build-script to test small changes.
-    helper_path = os.path.join(package_path, 'scripts',
-                               'build_script_helper.py')
-
-    build_cmd = [
-        helper_path,
-        '--verbose',
-        '--package-path', package_path,
-        '--build-path', product.build_dir,
-        '--toolchain', toolchain_path,
-    ]
-    shell.call(build_cmd)

utils/swift_build_support/swift_build_support/products/build_script_helper_builder.py

@@ -0,0 +1,61 @@
+# swift_build_support/product_builders/build_script_helper_b... -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import abc
+import os
+import platform
+
+from . import product
+from .. import shell, targets
+
+
+class BuildScriptHelperBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        self.__source_dir = workspace.source_dir(
+            product_class.product_source_name())
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+        self.__args = args
+
+    def build(self):
+        self.__run_build_script_helper('build')
+
+    def test(self):
+        if self._should_test():
+            self.__run_build_script_helper('test')
+
+    @abc.abstractmethod
+    def _should_test(self):
+        pass
+
+    def __run_build_script_helper(self, action):
+        script_path = os.path.join(
+            self.__source_dir, 'Utilities', 'build-script-helper.py')
+        toolchain_path = self.__args.install_destdir
+        if platform.system() == 'Darwin':
+            # The prefix is an absolute path, so concatenate without os.path.
+            toolchain_path += \
+                targets.darwin_toolchain_prefix(self.__args.install_prefix)
+        if self.__args.build_variant == 'Debug':
+            configuration = 'debug'
+        else:
+            configuration = 'release'
+        helper_cmd = [
+            script_path,
+            action,
+            '--verbose',
+            '--package-path', self.__source_dir,
+            '--build-path', self.__build_dir,
+            '--configuration', configuration,
+            '--toolchain', toolchain_path,
+        ]
+        shell.call(helper_cmd)

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -10,12 +10,8 @@
 #
 # ----------------------------------------------------------------------------
 
-import os
-import platform
-
 from . import product
-from .. import shell
-from .. import targets
+from .build_script_helper_builder import BuildScriptHelperBuilder
 
 
 class IndexStoreDB(product.Product):
@@ -27,30 +23,16 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        run_build_script_helper('build', host_target, self, self.args)
-
-    def test(self, host_target):
-        if self.args.test and self.args.test_indexstoredb:
-            run_build_script_helper('test', host_target, self, self.args)
-
-
-def run_build_script_helper(action, host_target, product, args):
-    script_path = os.path.join(
-        product.source_dir, 'Utilities', 'build-script-helper.py')
-    toolchain_path = args.install_destdir
-    if platform.system() == 'Darwin':
-        # The prefix is an absolute path, so concatenate without os.path.
-        toolchain_path += \
-            targets.darwin_toolchain_prefix(args.install_prefix)
-    configuration = 'debug' if args.build_variant == 'Debug' else 'release'
-    helper_cmd = [
-        script_path,
-        action,
-        '--verbose',
-        '--package-path', product.source_dir,
-        '--build-path', product.build_dir,
-        '--configuration', configuration,
-        '--toolchain', toolchain_path,
-    ]
-    shell.call(helper_cmd)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return IndexStoreDBBuilder(cls, args, toolchain, workspace, host)
+
+
+class IndexStoreDBBuilder(BuildScriptHelperBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        BuildScriptHelperBuilder.__init__(self, product_class, args, toolchain,
+                                          workspace, host)
+        self.__args = args
+
+    def _should_test(self):
+        return self.__args.test and self.__args.test_indexstoredb

utils/swift_build_support/swift_build_support/products/ninja.py

@@ -30,13 +30,14 @@ def is_build_script_impl_product(cls):
 
     @classmethod
     def new_builder(cls, args, toolchain, workspace, host):
-        return NinjaBuilder(cls, args, toolchain, workspace)
+        return NinjaBuilder(cls, args, toolchain, workspace, host)
 
 
 class NinjaBuilder(product.ProductBuilder):
-    def __init__(self, product_class, args, toolchain, workspace):
+    def __init__(self, product_class, args, toolchain, workspace, host):
         self.source_dir = workspace.source_dir(
             product_class.product_source_name())
+        # host is ignored. Ninja only builds for the build host.
         self.build_dir = workspace.build_dir('build',
                                              product_class.product_name())
         self.args = args

utils/swift_build_support/swift_build_support/products/product.py

@@ -40,20 +40,6 @@ def is_build_script_impl_product(cls):
         """
         return True
 
-    def build(self, host_target):
-        """build() -> void
-
-        Perform the build, for a non-build-script-impl product.
-        """
-        raise NotImplementedError
-
-    def test(self, host_target):
-        """test() -> void
-
-        Run the tests, for a non-build-script-impl product.
-        """
-        raise NotImplementedError
-
     def __init__(self, args, toolchain, source_dir, build_dir):
         self.args = args
         self.toolchain = toolchain
@@ -81,7 +67,7 @@ class ProductBuilder(object):
     """
 
     @abc.abstractmethod
-    def __init__(self, product_class, args, toolchain, workspace):
+    def __init__(self, product_class, args, toolchain, workspace, host):
         """
         Create a product builder for the given product class.
 
@@ -102,6 +88,10 @@ def __init__(self, product_class, args, toolchain, workspace):
             to be located. A builder should use the workspace to access its own
             source/build directory, as well as other products source/build
             directories.
+        host : `swift_build_support.targets.Target`
+            The target host for the product. The product is intended to be used
+            in the given target, even if the build machine is of a different OS
+            and/or architecture.
         """
         pass
 

utils/swift_build_support/swift_build_support/products/sourcekitlsp.py

@@ -10,8 +10,8 @@
 #
 # ----------------------------------------------------------------------------
 
-from . import indexstoredb
 from . import product
+from .build_script_helper_builder import BuildScriptHelperBuilder
 
 
 class SourceKitLSP(product.Product):
@@ -23,11 +23,16 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        indexstoredb.run_build_script_helper(
-            'build', host_target, self, self.args)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return SourceKitLSPBuilder(cls, args, toolchain, workspace, host)
+
+
+class SourceKitLSPBuilder(BuildScriptHelperBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        BuildScriptHelperBuilder.__init__(self, product_class, args, toolchain,
+                                          workspace, host)
+        self.__args = args
 
-    def test(self, host_target):
-        if self.args.test and self.args.test_sourcekitlsp:
-            indexstoredb.run_build_script_helper(
-                'test', host_target, self, self.args)
+    def _should_test(self):
+        return self.__args.test and self.__args.test_sourcekitlsp

utils/swift_build_support/swift_build_support/products/tsan_libdispatch.py

@@ -29,10 +29,24 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return TSanLibDispatchBuilder(cls, args, toolchain, workspace, host)
+
+
+class TSanLibDispatchBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        self.__args = args
+        self.__workspace = workspace
+
+        self.__source_dir = workspace.source_dir('compiler-rt')
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+
+    def build(self):
         """Build TSan runtime (compiler-rt)."""
-        rt_source_dir = join_path(self.source_dir, os.pardir, 'compiler-rt')
-        toolchain_path = join_path(self.args.install_destdir, 'usr')
+
+        toolchain_path = join_path(self.__args.install_destdir, 'usr')
         clang = join_path(toolchain_path, 'bin', 'clang')
         clangxx = join_path(toolchain_path, 'bin', 'clang++')
 
@@ -48,21 +62,21 @@ def build(self, host_target):
             '-DCOMPILER_RT_BUILD_XRAY=OFF',
             '-DCOMPILER_RT_INTERCEPT_LIBDISPATCH=ON',
             '-DCOMPILER_RT_LIBDISPATCH_INSTALL_PATH=%s' % toolchain_path,
-            rt_source_dir]
+            self.__source_dir]
         build_cmd = ['ninja', 'tsan']
 
         # Always rebuild TSan runtime
-        shell.rmtree(self.build_dir)
-        shell.makedirs(self.build_dir)
+        shell.rmtree(self.__build_dir)
+        shell.makedirs(self.__build_dir)
 
-        with shell.pushd(self.build_dir):
+        with shell.pushd(self.__build_dir):
             shell.call(config_cmd)
             shell.call(build_cmd)
 
-    def test(self, host_target):
+    def test(self):
         """Run check-tsan target with a LIT filter for libdispatch."""
         cmd = ['ninja', 'check-tsan']
         env = {'LIT_FILTER': 'libdispatch'}
 
-        with shell.pushd(self.build_dir):
+        with shell.pushd(self.__build_dir):
             shell.call(cmd, env=env)