[build-script] Move building swiftevolve to swift_build_support

Author: ahoppen

utils/build-script

@@ -305,7 +305,6 @@ class BuildScriptInvocation(object):
             "--xctest-build-type", args.build_variant,
             "--swiftpm-build-type", args.build_variant,
             "--swiftsyntax-build-type", args.build_variant,
-            "--swiftevolve-build-type", args.build_variant,
             "--llbuild-build-type", args.build_variant,
             "--swift-enable-assertions", str(args.swift_assertions).lower(),
             "--swift-stdlib-enable-assertions", str(
@@ -436,8 +435,6 @@ class BuildScriptInvocation(object):
             impl_args += ["--skip-build-swiftpm"]
         if not args.build_swiftsyntax:
             impl_args += ["--skip-build-swiftsyntax"]
-        if not args.build_swiftevolve:
-            impl_args += ["--skip-build-swiftevolve"]
         if not args.build_playgroundsupport:
             impl_args += ["--skip-build-playgroundsupport"]
         if args.build_swift_dynamic_stdlib:
@@ -482,7 +479,6 @@ class BuildScriptInvocation(object):
                           "--skip-test-llbuild",
                           "--skip-test-swiftpm",
                           "--skip-test-swiftsyntax",
-                          "--skip-test-swiftevolve",
                           "--skip-test-xctest",
                           "--skip-test-foundation",
                           "--skip-test-libdispatch",

utils/build-script-impl

@@ -83,7 +83,6 @@ KNOWN_SETTINGS=(
     # When this flag is set, the build-script will only build/install the swift-syntax parser
     # This is a temporary workaround of having a separate build product for swift-syntax parser
     skip-swiftsyntax-swiftside  ""               "skip building/installing the swift side of swiftsyntax"
-    swiftevolve-build-type      "Debug"          "the build variant for the swift-evolve tool"
     llbuild-enable-assertions   "1"              "enable assertions in llbuild"
     enable-asan                 ""               "enable Address Sanitizer"
     enable-ubsan                ""               "enable Undefined Behavior Sanitizer"
@@ -128,7 +127,6 @@ KNOWN_SETTINGS=(
     skip-build-libcxx           ""               "set to skip building libcxx"
     skip-build-swiftpm          ""               "set to skip building swiftpm"
     skip-build-swiftsyntax      ""               "set to skip building swiftSyntax"
-    skip-build-swiftevolve      ""               "set to skip building the swift-evolve tool"
     skip-build-xctest           ""               "set to skip building xctest"
     skip-build-foundation       ""               "set to skip building foundation"
     skip-build-libdispatch      ""               "set to skip building libdispatch"
@@ -143,7 +141,6 @@ KNOWN_SETTINGS=(
     skip-test-llbuild           ""               "set to skip testing llbuild"
     skip-test-swiftpm           ""               "set to skip testing swiftpm"
     skip-test-swiftsyntax       ""               "set to skip testing swiftSyntax"
-    skip-test-swiftevolve       ""               "set to skip testing the swift-evolve tool"
     skip-test-xctest            ""               "set to skip testing xctest"
     skip-test-foundation        ""               "set to skip testing foundation"
     skip-test-libdispatch       ""               "set to skip testing libdispatch"
@@ -203,7 +200,6 @@ KNOWN_SETTINGS=(
     install-swiftpm             ""               "whether to install swiftpm"
     install-swiftsyntax         ""               "whether to install swiftsyntax"
     skip-install-swiftsyntax-module ""           "set to skip installing swiftsyntax modules"
-    install-swiftevolve         ""               "whether to install the swift-evolve tool"
     install-xctest              ""               "whether to install xctest"
     install-foundation          ""               "whether to install foundation"
     install-libcxx              ""               "whether to install libc++"
@@ -1196,7 +1192,6 @@ LLBUILD_SOURCE_DIR="${WORKSPACE}/llbuild"
 SWIFTPM_SOURCE_DIR="${WORKSPACE}/swiftpm"
 SWIFTSYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"
 STRESSTEST_PACKAGE_DIR="${WORKSPACE}/swift-stress-tester"
-SWIFTEVOLVE_SOURCE_DIR="${STRESSTEST_PACKAGE_DIR}/SwiftEvolve"
 XCTEST_SOURCE_DIR="${WORKSPACE}/swift-corelibs-xctest"
 FOUNDATION_SOURCE_DIR="${WORKSPACE}/swift-corelibs-foundation"
 FOUNDATION_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-foundation"
@@ -1279,14 +1274,9 @@ fi
 if [[ ! "${SKIP_BUILD_SWIFTPM}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" swiftpm)
 fi
-# swift-evolve is dependent on SwiftSyntax, so it must be added to the list of 
-# build products first.
 if [[ ! "${SKIP_BUILD_SWIFTSYNTAX}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" swiftsyntax)
 fi
-if [[ ! "${SKIP_BUILD_SWIFTEVOLVE}" ]] ; then
-     PRODUCTS=("${PRODUCTS[@]}" swiftevolve)
-fi
 
 # Checks if a given product is enabled (i.e. part of $PRODUCTS array)
 function contains_product() {
@@ -1595,9 +1585,6 @@ function build_directory_bin() {
             swiftsyntax)
                 echo "${root}/${SWIFTSYNTAX_BUILD_TYPE}/bin"
                 ;;
-            swiftevolve)
-                echo "${root}/${SWIFTEVOLVE_BUILD_TYPE}/bin"
-                ;;
             xctest)
                 echo "${root}/${XCTEST_BUILD_TYPE}/bin"
                 ;;
@@ -1743,9 +1730,6 @@ function cmake_config_opt() {
             swiftsyntax)
                 echo "--config ${SWIFTSYNTAX_BUILD_TYPE}"
                 ;;
-            swiftevolve)
-                echo "--config ${SWIFTEVOLVE_BUILD_TYPE}"
-                ;;
             xctest)
                 echo "--config ${XCTEST_BUILD_TYPE}"
                 ;;
@@ -1870,48 +1854,6 @@ function set_swiftsyntax_build_command() {
         --filecheck-exec="$(build_directory_bin ${LOCAL_HOST} llvm)/FileCheck")
 }
 
-function set_stresstester_build_script_helper_command() {
-    local package_name="$1"
-    local package_build_type="$2"
-    local package_build_dir="$3"
-
-    if [ "${SKIP_BUILD_SWIFTSYNTAX}" ]; then
-        echo "Error: Cannot build $package_name without SwiftSyntax."
-        exit 1
-    fi
-
-    local swiftsyntax_config="debug"
-    if [[ $(is_cmake_release_build_type "${SWIFTSYNTAX_BUILD_TYPE}") ]] ; then
-        swiftsyntax_config="release"
-    fi
-    local config="debug"
-    if [[ $(is_cmake_release_build_type "${package_build_type}") ]] ; then
-        config="release"
-    fi
-
-    stresstester_build_script_helper_command=("${STRESSTEST_PACKAGE_DIR}/build-script-helper.py")
-    if [[ "${VERBOSE_BUILD}" ]] ; then
-        stresstester_build_script_helper_command+=(-v)
-    fi
-
-    stresstester_build_script_helper_command+=(
-        --package-dir="${package_name}"
-        --build-dir="${package_build_dir}"
-        --swiftc-exec="$(build_directory_bin ${LOCAL_HOST} swift)/swiftc"
-        --swift-build-exec="$(swiftpm_find_tool swift-build)"
-        --swift-test-exec="$(swiftpm_find_tool swift-test)"
-        --syntax-parser-header-dir="${SWIFT_SOURCE_DIR}/include/swift-c/SyntaxParser"
-        --syntax-parser-lib-dir="$(build_directory ${host} swift)/lib"
-        --sourcekitd-dir="$(build_directory ${host} swift)/lib"
-        --swiftsyntax-dir="$(build_directory ${host} swiftsyntax)/${swiftsyntax_config}"
-        --config="${config}")
-}
-
-function set_swiftevolve_build_command() {
-    set_stresstester_build_script_helper_command SwiftEvolve "${SWIFTEVOLVE_BUILD_TYPE}" "$(build_directory ${host} swiftevolve)"
-    swiftevolve_build_command=("${stresstester_build_script_helper_command[@]}")
-}
-
 #
 # Configure and build each product
 #
@@ -2563,16 +2505,6 @@ for host in "${ALL_HOSTS[@]}"; do
                 set_swiftsyntax_build_command
                 call "${swiftsyntax_build_command[@]}"
 
-                continue
-                ;;
-            swiftevolve)
-                if [[ "$(uname -s)" != "Darwin" ]]; then
-                    echo "error: unable to build swift-evolve on this platform"
-                    continue
-                fi
-                set_swiftevolve_build_command
-                call "${swiftevolve_build_command[@]}"
-
                 continue
                 ;;
             xctest)
@@ -3087,15 +3019,6 @@ for host in "${ALL_HOSTS[@]}"; do
                 # As swiftSyntax tests itself, we break early here.
                 continue
                 ;;
-            swiftevolve)
-                if [[ "${SKIP_TEST_SWIFTEVOLVE}" ]]; then
-                    continue
-                fi
-                echo "--- Running tests for ${product} ---"
-                set_swiftevolve_build_command
-                call "${swiftevolve_build_command[@]}" test
-                continue
-                ;;
             xctest)
                 if [[ "${SKIP_TEST_XCTEST}" ]]; then
                     continue
@@ -3452,24 +3375,6 @@ for host in "${ALL_HOSTS[@]}"; do
                     call "${swiftsyntax_build_command[@]}" --dylib-dir="${DYLIB_DIR}" --install
                 fi
 
-                continue
-                ;;
-            swiftevolve)
-                if [[ -z "${INSTALL_SWIFTEVOLVE}" ]] ; then
-                    continue
-                fi
-                if [[ -z "${INSTALL_DESTDIR}" ]] ; then
-                    echo "--install-destdir is required to install products."
-                    exit 1
-                fi
-                if [[ -z "${INSTALL_SWIFTSYNTAX}" ]] ; then
-                    echo "--install-swiftsyntax is required to install swift-evolve"
-                    exit 1
-                fi
-
-                echo "--- Installing ${product} ---"
-                set_swiftevolve_build_command
-                call "${swiftevolve_build_command[@]}" --prefix="${host_install_destdir}${host_install_prefix}" install
                 continue
                 ;;
             xctest)

utils/build_swift/driver_arguments.py

@@ -206,6 +206,7 @@ def _apply_default_arguments(args):
         args.test_indexstoredb = False
         args.test_sourcekitlsp = False
         args.test_skstresstester = False
+        args.test_swiftevolve = False
 
     # --skip-test-ios is merely a shorthand for host and simulator tests.
     if not args.test_ios:
@@ -571,6 +572,8 @@ def create_argument_parser():
            help='install SourceKitLSP')
     option(['--install-skstresstester'], toggle_true('install_skstresstester'),
            help='install the SourceKit stress tester')
+    option(['--install-swiftevolve'], toggle_true('install_swiftevolve'),
+           help='install SwiftEvolve')
     option(['--toolchain-benchmarks'],
            toggle_true('build_toolchainbenchmarks'),
            help='build Swift Benchmarks using swiftpm against the just built '
@@ -960,6 +963,8 @@ def create_argument_parser():
            help='skip testing sourcekit-lsp')
     option('--skip-test-skstresstester', toggle_false('test_skstresstester'),
            help='skip testing the SourceKit Stress tester')
+    option('--skip-test-swiftevolve', toggle_false('test_swiftevolve'),
+           help='skip testing SwiftEvolve')
 
     # -------------------------------------------------------------------------
     in_group('Build settings specific for LLVM')
@@ -1031,6 +1036,17 @@ def create_argument_parser():
     option('--skip-test-optimize-none-with-implicit-dynamic', unsupported)
     option('--skip-test-optimized', unsupported)
 
+    # -------------------------------------------------------------------------
+    in_group('Build-script-impl arguments (for disambiguation)')
+    # We need to list --skip-build-swift explicitly because otherwise argparse
+    # will auto-expand arguments like --skip-build-swift to the only known
+    # argument --skip-build-swiftevolve.
+    # These arguments are forwarded to impl_args in migration.py
+
+    option('--install-swift', toggle_true('impl_install_swift'))
+    option('--skip-test-swift', toggle_true('impl_skip_test_swift'))
+
+
     # -------------------------------------------------------------------------
     return builder.build()
 
@@ -1073,7 +1089,8 @@ def create_argument_parser():
 
 
 Any arguments not listed are forwarded directly to Swift's
-'build-script-impl'.  See that script's help for details.
+'build-script-impl'. See that script's help for details. The listed
+build-script-impl arguments are only for disambiguation in the argument parser.
 
 Environment variables
 ---------------------

utils/build_swift/tests/expected_options.py

@@ -90,6 +90,7 @@
     'build_sourcekitlsp': False,
     'install_sourcekitlsp': False,
     'install_skstresstester': False,
+    'install_swiftevolve': False,
     'build_toolchainbenchmarks': False,
     'build_tvos': True,
     'build_tvos_device': False,
@@ -207,6 +208,7 @@
     'test_indexstoredb': False,
     'test_sourcekitlsp': False,
     'test_skstresstester': False,
+    'test_swiftevolve': False,
     'tvos': False,
     'tvos_all': False,
     'validation_test': None,
@@ -344,6 +346,14 @@ class IgnoreOption(_BaseOption):
     pass
 
 
+class BuildScriptImplOption(_BaseOption):
+    """Option that gets forwarded to build-script-impl by migration.py and is
+    only listed for disambiguation by argparse.
+    """
+
+    pass
+
+
 # -----------------------------------------------------------------------------
 
 EXPECTED_OPTIONS = [
@@ -467,6 +477,7 @@ class IgnoreOption(_BaseOption):
     EnableOption('--sourcekit-lsp', dest='build_sourcekitlsp'),
     EnableOption('--install-sourcekit-lsp', dest='install_sourcekitlsp'),
     EnableOption('--install-skstresstester', dest='install_skstresstester'),
+    EnableOption('--install-swiftevolve', dest='install_swiftevolve'),
     EnableOption('--toolchain-benchmarks', dest='build_toolchainbenchmarks'),
     EnableOption('--tsan-libdispatch-test'),
     EnableOption('--long-test'),
@@ -524,6 +535,7 @@ class IgnoreOption(_BaseOption):
     DisableOption('--skip-test-indexstore-db', dest='test_indexstoredb'),
     DisableOption('--skip-test-sourcekit-lsp', dest='test_sourcekitlsp'),
     DisableOption('--skip-test-skstresstester', dest='test_skstresstester'),
+    DisableOption('--skip-test-swiftevolve', dest='test_swiftevolve'),
 
     ChoicesOption('--android-ndk-gcc-version',
                   choices=['4.8', '4.9']),
@@ -590,6 +602,10 @@ class IgnoreOption(_BaseOption):
     UnsupportedOption('--skip-test-optimize-none-with-implicit-dynamic'),
     UnsupportedOption('--skip-test-optimized'),
 
+    # Options forwared to build-script-impl
+    BuildScriptImplOption('--skip-test-swift', dest='impl_skip_test_swift'),
+    BuildScriptImplOption('--install-swift', dest='impl_install_swift'),
+
     # NOTE: LTO flag is a special case that acts both as an option and has
     # valid choices
     SetOption('--lto', dest='lto_type'),

utils/build_swift/tests/test_driver_arguments.py

@@ -313,6 +313,23 @@ def test(self):
 
         return test
 
+    @classmethod
+    def _generate_build_script_impl_option_test(cls, option):
+        def test(self):
+            with self.assertNotRaises(ParserError):
+                namespace, unknown_args = self.parse_args_and_unknown_args([])
+                self.assertFalse(hasattr(namespace, option.dest))
+                self.assertEqual(unknown_args, [])
+
+                namespace, unknown_args = self.parse_args_and_unknown_args(
+                    [option.option_string])
+                # The argument should never show up in the namespace
+                self.assertFalse(hasattr(namespace, option.dest))
+                # It should instead be forwareded to unkown_args
+                self.assertEqual(unknown_args, [option.option_string])
+
+        return test
+
     @classmethod
     def generate_option_test(cls, option):
         generate_test_funcs = {
@@ -328,6 +345,8 @@ def generate_option_test(cls, option):
             eo.PathOption: cls._generate_path_option_test,
             eo.AppendOption: cls._generate_append_option_test,
             eo.UnsupportedOption: cls._generate_unsupported_option_test,
+            eo.BuildScriptImplOption:
+                cls._generate_build_script_impl_option_test,
 
             # IgnoreOptions should be manually tested
             eo.IgnoreOption: lambda self: None,
@@ -383,7 +402,7 @@ def _check_impl_args(self, namespace):
             raise ParserError('failed to parse impl arguments: ' +
                               str(namespace.build_script_impl_args), e)
 
-    def parse_args(self, args, namespace=None):
+    def parse_args_and_unknown_args(self, args, namespace=None):
         if namespace is None:
             namespace = argparse.Namespace()
 
@@ -392,9 +411,18 @@ def parse_args(self, args, namespace=None):
                 namespace, unknown_args =\
                     super(self.parser.__class__, self.parser)\
                     .parse_known_args(args, namespace)
+                namespace, unknown_args =\
+                    migration.process_disambiguation_arguments(namespace,
+                                                               unknown_args)
             except (SystemExit, argparse.ArgumentError) as e:
                 raise ParserError('failed to parse arguments: ' + str(args), e)
 
+        return namespace, unknown_args
+
+    def parse_args(self, args, namespace=None):
+        namespace, unknown_args = self.parse_args_and_unknown_args(args,
+            namespace)
+
         if unknown_args:
             raise ParserError('unknown arguments: ' + str(unknown_args))
 
@@ -452,6 +480,7 @@ def test_expected_options_have_default_values(self):
             eo.HelpOption,
             eo.IgnoreOption,
             eo.UnsupportedOption,
+            eo.BuildScriptImplOption,
         ]
 
         missing_defaults = set()