[build-script] Turn on --no-legacy-impl by default

Turns on the `--no-legacy-impl` option to build-script by default; the
old behaviour is temporarily still available as `--legacy-impl`.

This causes build-script to invoke build-script-impl for every
individual build/test/install/etc. action rather than a single global
invocation. For example, a single invocation might be for
`macosx-swift-install`. This will enable the python code in build-script
to drive the overall process and add additional steps in between actions
without the involvement of build-script-impl. It also provides a path to
refactoring the existing actions out of build-script-impl individually.

Discussed as part of https://forums.swift.org/t/rfc-building-swift-packages-in-build-script/18920

The --no-legacy-impl flag was originally disabled by default because of
concerns about the performance of null builds due to the increased
number of script invocations. There is a small optimization in this
commit to use `tr` when processing command-line options instead of
bash's builtin substitution, which eliminates most of the overhead.
After this change, a null build of llvm+swift changes from 1.6 s to
2.1 s on Linux, and from 5 s to 6 s on macOS.  Non-null builds and
builds that involve more build products than just llvm+swift (e.g.
corelibs) are basically unaffected since they are not correctly
incremental to begin with.

The changes to build-script-impl in this commit are to fix the behaviour
of --no-legacy-impl, which had bitrotted since it was introduced. These
changes are to make various parts of the script not rely on variables
defined in "earlier" parts of the script, which is good hygiene in
general.

Author: benlangmuir

utils/build-script

@@ -527,9 +527,11 @@ class BuildScriptInvocation(object):
             # just invoke cmake from build-script directly rather than futzing
             # with build-script-impl. This makes even more sense since there
             # really isn't a security issue here.
-            impl_args += [
-                "--%s-cmake-options=%s" % (product_name, ' '.join(cmake_opts))
-            ]
+            if len(cmake_opts) > 0:
+                impl_args += [
+                    "--%s-cmake-options=%s" %
+                    (product_name, ' '.join(cmake_opts))
+                ]
 
         if args.build_stdlib_deployment_targets:
             impl_args += [
@@ -839,16 +841,20 @@ class BuildScriptInvocation(object):
         product_classes.append(products.Swift)
         if self.args.build_lldb:
             product_classes.append(products.LLDB)
-        if self.args.build_llbuild:
-            product_classes.append(products.LLBuild)
         if self.args.build_libdispatch:
             product_classes.append(products.LibDispatch)
         if self.args.build_foundation:
             product_classes.append(products.Foundation)
         if self.args.build_xctest:
             product_classes.append(products.XCTest)
+        if self.args.build_llbuild:
+            product_classes.append(products.LLBuild)
         if self.args.build_swiftpm:
             product_classes.append(products.SwiftPM)
+        if self.args.build_swiftsyntax:
+            product_classes.append(products.SwiftSyntax)
+        if self.args.build_skstresstester:
+            product_classes.append(products.SKStressTester)
         return product_classes
 
     def execute(self):
@@ -873,7 +879,7 @@ class BuildScriptInvocation(object):
                         name == "merged-hosts-lipo"), "invalid action"
                 action_name = name
             elif product_class is None:
-                assert name == "package", "invalid action"
+                assert name in ("package", "extractsymbols"), "invalid action"
                 action_name = "{}-{}".format(host.name, name)
             else:
                 assert name is not None, "invalid action"
@@ -925,6 +931,10 @@ class BuildScriptInvocation(object):
             for product_class in product_classes:
                 execute_one_impl_action(host_target, product_class, "install")
 
+        # Extract symbols...
+        for host_target in all_hosts:
+            execute_one_impl_action(host_target, name="extractsymbols")
+
         # Package...
         for host_target in all_hosts:
             execute_one_impl_action(host_target, name="package")

utils/build-script-impl

@@ -275,8 +275,8 @@ def create_argument_parser():
     option(['-n', '--dry-run'], store_true,
            help='print the commands that would be executed, but do not '
                 'execute them')
-    option('--no-legacy-impl', store_false('legacy_impl'),
-           help='avoid legacy implementation')
+    option('--legacy-impl', store_true('legacy_impl'),
+           help='use legacy implementation')
 
     option('--build-runtime-with-host-compiler', toggle_true,
            help='Use the host compiler, not the self-built one to compile the '

utils/build_swift/driver_arguments.py

@@ -138,7 +138,7 @@
     'install_symroot': None,
     'ios': False,
     'ios_all': False,
-    'legacy_impl': True,
+    'legacy_impl': False,
     'libdispatch_build_variant': 'Debug',
     'libicu_build_variant': 'Debug',
     'lit_args': '-sv',
@@ -411,7 +411,7 @@ class IgnoreOption(_BaseOption):
     SetTrueOption('-n', dest='dry_run'),
     SetTrueOption('-p', dest='build_swiftpm'),
 
-    SetFalseOption('--no-legacy-impl', dest='legacy_impl'),
+    SetTrueOption('--legacy-impl', dest='legacy_impl'),
 
     EnableOption('--android'),
     EnableOption('--build-external-benchmarks'),

utils/build_swift/tests/expected_options.py

@@ -18,8 +18,10 @@
 from .lldb import LLDB
 from .llvm import LLVM
 from .ninja import Ninja
+from .skstresstester import SKStressTester
 from .swift import Swift
 from .swiftpm import SwiftPM
+from .swiftsyntax import SwiftSyntax
 from .xctest import XCTest
 
 __all__ = [
@@ -35,4 +37,6 @@
     'Swift',
     'SwiftPM',
     'XCTest',
+    'SwiftSyntax',
+    'SKStressTester',
 ]

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -0,0 +1,23 @@
+# swift_build_support/products/skstresstester.py -----------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+from . import product
+
+
+class SKStressTester(product.Product):
+    @classmethod
+    def product_source_name(cls):
+        """product_source_name() -> str
+
+        The name of the source code directory of this product.
+        """
+        return "swift-stress-tester"

utils/swift_build_support/swift_build_support/products/skstresstester.py

@@ -0,0 +1,23 @@
+# swift_build_support/products/swiftsyntax.py --------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+from . import product
+
+
+class SwiftSyntax(product.Product):
+    @classmethod
+    def product_source_name(cls):
+        """product_source_name() -> str
+
+        The name of the source code directory of this product.
+        """
+        return "swift-syntax"