[build-script] Transform IndexStoreDB and SorcekitLSP to use ProductBuilder.

Create a new helper builder for IndexStoreDB and SourceKitLSP called
BuildScriptHelperBuilder (because the invoked script is called
build-script-helper.py). Move most of the code that existed in a
function in indexstore.db into that helper class.

Create small subtypes of the helper paramtrized for IndexStoreDB and
SourceKitLSP.

Modify the main build-script to invoke the new build instead of the
Product class directly. This way, when all the products use builders
to be build, all of them will use the same Python invocations, but
some of them will use build-script-impl, and others will use
build-script-helper.py.

Author: drodriguez

utils/build-script

@@ -964,20 +964,16 @@ class BuildScriptInvocation(object):
 
         # Non-build-script-impl products...
         # Note: currently only supports building for the host.
-        for host_target in [self.args.host_target]:
+        for host_target in [
+                StdlibDeploymentTarget.get_target_for_name(
+                    self.args.host_target)]:
             for product_class in product_classes:
                 if product_class.is_build_script_impl_product():
                     continue
-                product_source = product_class.product_source_name()
-                product_name = product_class.product_name()
-                product = product_class(
-                    args=self.args,
-                    toolchain=self.toolchain,
-                    source_dir=self.workspace.source_dir(product_source),
-                    build_dir=self.workspace.build_dir(
-                        host_target, product_name))
-                product.build(host_target)
-                product.test(host_target)
+                builder = product_class.new_builder(
+                    self.args, self.toolchain, self.workspace, host_target)
+                builder.build()
+                builder.test()
 
         # Extract symbols...
         for host_target in all_hosts:

utils/swift_build_support/swift_build_support/products/benchmarks.py

@@ -28,48 +28,53 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        run_build_script_helper(host_target, self, self.args)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return BenchmarksBuilder(cls, args, toolchain, workspace, host)
+
+
+# NOTE: this is very similar to BuildScriptHelperBuilder, but also
+# different enough to justify a different builder altogether.
+class BenchmarksBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        # Our source directory is a subdirectory inside the swift source
+        # directory
+        self.__source_dir = os.path.join(workspace.source_dir('swift'),
+                                         'benchmark')
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+        self.__args = args
 
-    def test(self, host_target):
+    def build(self):
+        # We use a separate python helper to enable quicker iteration when
+        # working on this by avoiding going through build-script to test small
+        # changes.
+        script_path = os.path.join(
+            self.__source_dir, 'scripts', 'build_script_helper.py')
+        toolchain_path = self.__args.install_destdir
+        if platform.system() == 'Darwin':
+            # The prefix is an absolute path, so concatenate without os.path.
+            toolchain_path += \
+                targets.darwin_toolchain_prefix(self.__args.install_prefix)
+        helper_cmd = [
+            script_path,
+            '--verbose',
+            '--package-path', self.__source_dir,
+            '--build-path', self.__build_dir,
+            '--toolchain', toolchain_path,
+        ]
+        shell.call(helper_cmd)
+
+    def test(self):
         """Just run a single instance of the command for both .debug and
            .release.
         """
         cmdline = ['--num-iters=1', 'XorLoop']
-        bench_Onone = os.path.join(self.build_dir, 'bin', 'Benchmark_Onone')
+        bench_Onone = os.path.join(self.__build_dir, 'bin', 'Benchmark_Onone')
         shell.call([bench_Onone] + cmdline)
 
-        bench_O = os.path.join(self.build_dir, 'bin', 'Benchmark_O')
+        bench_O = os.path.join(self.__build_dir, 'bin', 'Benchmark_O')
         shell.call([bench_O] + cmdline)
 
-        bench_Osize = os.path.join(self.build_dir, 'bin', 'Benchmark_Osize')
+        bench_Osize = os.path.join(self.__build_dir, 'bin', 'Benchmark_Osize')
         shell.call([bench_Osize] + cmdline)
-
-
-def run_build_script_helper(host_target, product, args):
-    toolchain_path = args.install_destdir
-    if platform.system() == 'Darwin':
-        # The prefix is an absolute path, so concatenate without os.path.
-        toolchain_path += \
-            targets.darwin_toolchain_prefix(args.install_prefix)
-
-    # Our source_dir is expected to be './$SOURCE_ROOT/benchmarks'. That is due
-    # the assumption that each product is in its own build directory. This
-    # product is not like that and has its package/tools instead in
-    # ./$SOURCE_ROOT/swift/benchmark.
-    package_path = os.path.join(product.source_dir, '..', 'swift', 'benchmark')
-    package_path = os.path.abspath(package_path)
-
-    # We use a separate python helper to enable quicker iteration when working
-    # on this by avoiding going through build-script to test small changes.
-    helper_path = os.path.join(package_path, 'scripts',
-                               'build_script_helper.py')
-
-    build_cmd = [
-        helper_path,
-        '--verbose',
-        '--package-path', package_path,
-        '--build-path', product.build_dir,
-        '--toolchain', toolchain_path,
-    ]
-    shell.call(build_cmd)

utils/swift_build_support/swift_build_support/products/build_script_helper_builder.py

@@ -0,0 +1,61 @@
+# swift_build_support/product_builders/build_script_helper_b... -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import abc
+import os
+import platform
+
+from . import product
+from .. import shell, targets
+
+
+class BuildScriptHelperBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        self.__source_dir = workspace.source_dir(
+            product_class.product_source_name())
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+        self.__args = args
+
+    def build(self):
+        self.__run_build_script_helper('build')
+
+    def test(self):
+        if self._should_test():
+            self.__run_build_script_helper('test')
+
+    @abc.abstractmethod
+    def _should_test(self):
+        pass
+
+    def __run_build_script_helper(self, action):
+        script_path = os.path.join(
+            self.__source_dir, 'Utilities', 'build-script-helper.py')
+        toolchain_path = self.__args.install_destdir
+        if platform.system() == 'Darwin':
+            # The prefix is an absolute path, so concatenate without os.path.
+            toolchain_path += \
+                targets.darwin_toolchain_prefix(self.__args.install_prefix)
+        if self.__args.build_variant == 'Debug':
+            configuration = 'debug'
+        else:
+            configuration = 'release'
+        helper_cmd = [
+            script_path,
+            action,
+            '--verbose',
+            '--package-path', self.__source_dir,
+            '--build-path', self.__build_dir,
+            '--configuration', configuration,
+            '--toolchain', toolchain_path,
+        ]
+        shell.call(helper_cmd)

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -10,12 +10,8 @@
 #
 # ----------------------------------------------------------------------------
 
-import os
-import platform
-
 from . import product
-from .. import shell
-from .. import targets
+from .build_script_helper_builder import BuildScriptHelperBuilder
 
 
 class IndexStoreDB(product.Product):
@@ -27,30 +23,16 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        run_build_script_helper('build', host_target, self, self.args)
-
-    def test(self, host_target):
-        if self.args.test and self.args.test_indexstoredb:
-            run_build_script_helper('test', host_target, self, self.args)
-
-
-def run_build_script_helper(action, host_target, product, args):
-    script_path = os.path.join(
-        product.source_dir, 'Utilities', 'build-script-helper.py')
-    toolchain_path = args.install_destdir
-    if platform.system() == 'Darwin':
-        # The prefix is an absolute path, so concatenate without os.path.
-        toolchain_path += \
-            targets.darwin_toolchain_prefix(args.install_prefix)
-    configuration = 'debug' if args.build_variant == 'Debug' else 'release'
-    helper_cmd = [
-        script_path,
-        action,
-        '--verbose',
-        '--package-path', product.source_dir,
-        '--build-path', product.build_dir,
-        '--configuration', configuration,
-        '--toolchain', toolchain_path,
-    ]
-    shell.call(helper_cmd)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return IndexStoreDBBuilder(cls, args, toolchain, workspace, host)
+
+
+class IndexStoreDBBuilder(BuildScriptHelperBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        BuildScriptHelperBuilder.__init__(self, product_class, args, toolchain,
+                                          workspace, host)
+        self.__args = args
+
+    def _should_test(self):
+        return self.__args.test and self.__args.test_indexstoredb

utils/swift_build_support/swift_build_support/products/ninja.py

@@ -30,13 +30,14 @@ def is_build_script_impl_product(cls):
 
     @classmethod
     def new_builder(cls, args, toolchain, workspace, host):
-        return NinjaBuilder(cls, args, toolchain, workspace)
+        return NinjaBuilder(cls, args, toolchain, workspace, host)
 
 
 class NinjaBuilder(product.ProductBuilder):
-    def __init__(self, product_class, args, toolchain, workspace):
+    def __init__(self, product_class, args, toolchain, workspace, host):
         self.source_dir = workspace.source_dir(
             product_class.product_source_name())
+        # host is ignored. Ninja only builds for the build host.
         self.build_dir = workspace.build_dir('build',
                                              product_class.product_name())
         self.args = args

utils/swift_build_support/swift_build_support/products/product.py

@@ -40,20 +40,6 @@ def is_build_script_impl_product(cls):
         """
         return True
 
-    def build(self, host_target):
-        """build() -> void
-
-        Perform the build, for a non-build-script-impl product.
-        """
-        raise NotImplementedError
-
-    def test(self, host_target):
-        """test() -> void
-
-        Run the tests, for a non-build-script-impl product.
-        """
-        raise NotImplementedError
-
     def __init__(self, args, toolchain, source_dir, build_dir):
         self.args = args
         self.toolchain = toolchain
@@ -81,7 +67,7 @@ class ProductBuilder(object):
     """
 
     @abc.abstractmethod
-    def __init__(self, product_class, args, toolchain, workspace):
+    def __init__(self, product_class, args, toolchain, workspace, host):
         """
         Create a product builder for the given product class.
 
@@ -102,6 +88,10 @@ def __init__(self, product_class, args, toolchain, workspace):
             to be located. A builder should use the workspace to access its own
             source/build directory, as well as other products source/build
             directories.
+        host : `swift_build_support.targets.Target`
+            The target host for the product. The product is intended to be used
+            in the given target, even if the build machine is of a different OS
+            and/or architecture.
         """
         pass
 

utils/swift_build_support/swift_build_support/products/sourcekitlsp.py

@@ -10,8 +10,8 @@
 #
 # ----------------------------------------------------------------------------
 
-from . import indexstoredb
 from . import product
+from .build_script_helper_builder import BuildScriptHelperBuilder
 
 
 class SourceKitLSP(product.Product):
@@ -23,11 +23,16 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
-        indexstoredb.run_build_script_helper(
-            'build', host_target, self, self.args)
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return SourceKitLSPBuilder(cls, args, toolchain, workspace, host)
+
+
+class SourceKitLSPBuilder(BuildScriptHelperBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        BuildScriptHelperBuilder.__init__(self, product_class, args, toolchain,
+                                          workspace, host)
+        self.__args = args
 
-    def test(self, host_target):
-        if self.args.test and self.args.test_sourcekitlsp:
-            indexstoredb.run_build_script_helper(
-                'test', host_target, self, self.args)
+    def _should_test(self):
+        return self.__args.test and self.__args.test_sourcekitlsp

utils/swift_build_support/swift_build_support/products/tsan_libdispatch.py

@@ -29,10 +29,24 @@ def product_source_name(cls):
     def is_build_script_impl_product(cls):
         return False
 
-    def build(self, host_target):
+    @classmethod
+    def new_builder(cls, args, toolchain, workspace, host):
+        return TSanLibDispatchBuilder(cls, args, toolchain, workspace, host)
+
+
+class TSanLibDispatchBuilder(product.ProductBuilder):
+    def __init__(self, product_class, args, toolchain, workspace, host):
+        self.__args = args
+        self.__workspace = workspace
+
+        self.__source_dir = workspace.source_dir('compiler-rt')
+        self.__build_dir = workspace.build_dir(host.name,
+                                               product_class.product_name())
+
+    def build(self):
         """Build TSan runtime (compiler-rt)."""
-        rt_source_dir = join_path(self.source_dir, os.pardir, 'compiler-rt')
-        toolchain_path = join_path(self.args.install_destdir, 'usr')
+
+        toolchain_path = join_path(self.__args.install_destdir, 'usr')
         clang = join_path(toolchain_path, 'bin', 'clang')
         clangxx = join_path(toolchain_path, 'bin', 'clang++')
 
@@ -48,21 +62,21 @@ def build(self, host_target):
             '-DCOMPILER_RT_BUILD_XRAY=OFF',
             '-DCOMPILER_RT_INTERCEPT_LIBDISPATCH=ON',
             '-DCOMPILER_RT_LIBDISPATCH_INSTALL_PATH=%s' % toolchain_path,
-            rt_source_dir]
+            self.__source_dir]
         build_cmd = ['ninja', 'tsan']
 
         # Always rebuild TSan runtime
-        shell.rmtree(self.build_dir)
-        shell.makedirs(self.build_dir)
+        shell.rmtree(self.__build_dir)
+        shell.makedirs(self.__build_dir)
 
-        with shell.pushd(self.build_dir):
+        with shell.pushd(self.__build_dir):
             shell.call(config_cmd)
             shell.call(build_cmd)
 
-    def test(self, host_target):
+    def test(self):
         """Run check-tsan target with a LIT filter for libdispatch."""
         cmd = ['ninja', 'check-tsan']
         env = {'LIT_FILTER': 'libdispatch'}
 
-        with shell.pushd(self.build_dir):
+        with shell.pushd(self.__build_dir):
             shell.call(cmd, env=env)