[build-script] Presets Module (#14422) (#14834)

* Implemented a presets module which includes a more robust and easy to understand parser. Moved the swift-sdks migration code to a new migration module and added testing for both the presets and migration modules. Also converted build-script to use the new presets parser.

* Switched the expansion algorithm to resolve mixins in-place (little known feature) and also changed the parser to skip all non-preset sections. Tests are included for these two behaviors.

* Re-worked the presets error hierarchy to have more descriptive and information packed exception classes. Also re-worked the PresetParser to catch duplicate preset declarations and duplicate options in a single preset. There's some special shim-code to handle the disconnect between the Python 2 ConfigParser module and the Python 3 update which adds DuplicateOptionError.

Author: Rostepher

utils/build-script

@@ -20,6 +20,8 @@ import time
 
 from build_swift import defaults
 from build_swift import driver_arguments
+from build_swift import presets
+from build_swift.migration import migrate_swift_sdks
 
 from swift_build_support.swift_build_support import (
     arguments,
@@ -32,16 +34,12 @@ from swift_build_support.swift_build_support import (
     targets,
     workspace
 )
-
 from swift_build_support.swift_build_support.SwiftBuildSupport import (
     HOME,
     SWIFT_BUILD_ROOT,
     SWIFT_REPO_NAME,
     SWIFT_SOURCE_ROOT,
-    get_all_preset_names,
-    get_preset_options,
 )
-
 from swift_build_support.swift_build_support.cmake import CMake
 from swift_build_support.swift_build_support.targets import \
     StdlibDeploymentTarget
@@ -974,42 +972,56 @@ def main_preset():
 
     if len(args.preset_file_names) == 0:
         args.preset_file_names = [
-            os.path.join(HOME, ".swift-build-presets"),
             os.path.join(
                 SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
                 "build-presets.ini")
         ]
 
+        user_presets_file = os.path.join(HOME, '.swift-build-presets')
+        if os.path.isfile(user_presets_file):
+            args.preset_file_names.append(user_presets_file)
+
+    preset_parser = presets.PresetParser()
+
+    try:
+        preset_parser.read(args.preset_file_names)
+    except presets.Error as e:
+        diagnostics.fatal(e.message)
+
     if args.show_presets:
-        for name in sorted(get_all_preset_names(args.preset_file_names),
-                           key=str.lower):
+        for name in sorted(preset_parser.preset_names(),
+                           key=lambda name: name.lower()):
             print(name)
         return 0
 
     if not args.preset:
         diagnostics.fatal("missing --preset option")
 
     args.preset_substitutions = {}
-
     for arg in args.preset_substitutions_raw:
         name, value = arg.split("=", 1)
         args.preset_substitutions[name] = value
 
-    preset_args = get_preset_options(
-        args.preset_substitutions, args.preset_file_names, args.preset)
+    try:
+        preset = preset_parser.get_preset(args.preset,
+                                          vars=args.preset_substitutions)
+    except presets.Error as e:
+        diagnostics.fatal(e.message)
+
+    preset_args = migrate_swift_sdks(preset.format_args())
 
     build_script_args = [sys.argv[0]]
     if args.dry_run:
         build_script_args += ["--dry-run"]
+
     build_script_args += preset_args
     if args.distcc:
         build_script_args += ["--distcc"]
     if args.build_jobs:
         build_script_args += ["--jobs", str(args.build_jobs)]
 
-    diagnostics.note(
-        "using preset '" + args.preset + "', which expands to \n\n" +
-        shell.quote_command(build_script_args) + "\n")
+    diagnostics.note('using preset "{}", which expands to \n\n{}\n'.format(
+        args.preset, shell.quote_command(build_script_args)))
 
     if args.expand_build_script_invocation:
         return 0

utils/build_swift/migration.py

@@ -0,0 +1,84 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+
+"""
+Temporary module with functionaly used to migrate away from build-script-impl.
+"""
+
+from __future__ import absolute_import, unicode_literals
+
+from swift_build_support.swift_build_support.targets import \
+    StdlibDeploymentTarget
+
+
+__all__ = [
+    'UnknownSDKError',
+    'migrate_swift_sdks',
+]
+
+
+_SDK_TARGETS = {
+    'OSX': StdlibDeploymentTarget.OSX.targets,
+    'IOS': StdlibDeploymentTarget.iOS.targets,
+    'IOS_SIMULATOR': StdlibDeploymentTarget.iOSSimulator.targets,
+    'TVOS': StdlibDeploymentTarget.AppleTV.targets,
+    'TVOS_SIMULATOR': StdlibDeploymentTarget.AppleTVSimulator.targets,
+    'WATCHOS': StdlibDeploymentTarget.AppleWatch.targets,
+    'WATCHOS_SIMULATOR': StdlibDeploymentTarget.AppleWatchSimulator.targets,
+}
+
+
+# -----------------------------------------------------------------------------
+
+class UnknownSDKError(Exception):
+    """Error indicating an unknown SDK was encountered when migrating to target
+    triples.
+    """
+
+    pass
+
+
+def _swift_sdks_to_stdlib_targets(swift_sdks):
+    stdlib_targets = []
+    for sdk in swift_sdks:
+        sdk_targets = _SDK_TARGETS.get(sdk, None)
+        if sdk_targets is None:
+            raise UnknownSDKError(sdk)
+        stdlib_targets += sdk_targets
+
+    return stdlib_targets
+
+
+def migrate_swift_sdks(args):
+    """Migrate usages of the now deprecated `--swift-sdks` option to the new
+    `--stdlib-deployment-targets` option, converting Swift SDKs to the
+    corresponding targets. Since argument parsing is a last-wins scenario, only
+    the last `--swift-sdks` option is migrated, all others are removed.
+
+    This function is a stop-gap to replacing all instances of `--swift-sdks`.
+    """
+
+    swift_sdks_args = [arg for arg in args if arg.startswith('--swift-sdks')]
+
+    if len(swift_sdks_args) < 1:
+        return args
+
+    # Only get the last --swift-sdks arg since last-wins
+    swift_sdks_arg = swift_sdks_args[-1]
+
+    sdk_list = swift_sdks_arg.split('=')[1].split(';')
+    stdlib_targets = _swift_sdks_to_stdlib_targets(sdk_list)
+
+    target_names = ' '.join([target.name for target in stdlib_targets])
+    stdlib_targets_arg = '--stdlib-deployment-targets=' + target_names
+
+    args = [arg for arg in args if not arg.startswith('--swift-sdks')]
+    args.append(stdlib_targets_arg)
+
+    return args