[WIP][Build Script Helper] On macOS make install action use CMake and install libSwiftDriver

Previously only the x86 `swift-driver` and `swift-help` executables were copied over to the specified toolchain's `bin` directory.
With this change, libSwiftDriver shared library is also packaged and installed into the toolchain, as well as libSwiftOptions.
To do so, we must build SwiftDriver and all of its dependencies using CMake for arm64 and x86_64 and create universal binaries, as documented here:
https://developer.apple.com/documentation/xcode/building_a_universal_macos_binary

Author: artemcm

Utilities/build-script-helper.py

@@ -3,16 +3,35 @@
 from __future__ import print_function
 
 import argparse
+from distutils import file_util
 import os
+import json
 import platform
 import shutil
 import subprocess
 import sys
+import errno
+
+if platform.system() == 'Darwin':
+    shared_lib_ext = '.dylib'
+else:
+    shared_lib_ext = '.so'
+macos_deployment_target = '10.15'
+macos_target_architectures = ['x86_64','arm64']
 
 # Tools constructed as a part of the a development build toolchain
 driver_toolchain_tools = ['swift', 'swift-frontend', 'clang', 'swift-help',
                           'swift-autolink-extract', 'lldb']
 
+def mkdir_p(path):
+    """Create the given directory, if it does not exist."""
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        # Ignore EEXIST, which may occur during a race condition.
+        if e.errno != errno.EEXIST:
+            raise
+
 def call_output(cmd, cwd=None, stderr=False, verbose=False):
     """Calls a subprocess for its return data."""
     if verbose:
@@ -76,26 +95,26 @@ def get_swiftpm_options(args):
 
   return swiftpm_args
 
-def install(swiftpm_bin_path, toolchain):
-  toolchain_bin = os.path.join(toolchain, 'bin')
-  for exe in ['swift-driver', 'swift-help']:
-    install_binary(exe, swiftpm_bin_path, toolchain_bin, toolchain, is_executable=True)
-
-def install_binary(file, source_dir, install_dir, toolchain, is_executable):
+def install_binary(file, source_dir, install_dir, verbose):
+  print('Installing %s into: %s' % (file, install_dir))
   cmd = ['rsync', '-a', os.path.join(source_dir.decode('UTF-8'), file), install_dir]
-  print(' '.join(cmd))
-  subprocess.check_call(cmd)
-
-  if platform.system() == 'Darwin' and is_executable:
-    result_path = os.path.join(install_dir, file)
-    stdlib_rpath = os.path.join(toolchain, 'lib', 'swift', 'macosx')
-    delete_rpath(stdlib_rpath, result_path)
-
-def delete_rpath(rpath, binary):
-  cmd = ["install_name_tool", "-delete_rpath", rpath, binary]
-  print(' '.join(cmd))
+  if verbose:
+    print(' '.join(cmd))
   subprocess.check_call(cmd)
 
+def delete_rpath(rpath, binary, verbose):
+  cmd = ['install_name_tool', '-delete_rpath', rpath, binary]
+  if verbose:
+    print(' '.join(cmd))
+  installToolProcess = subprocess.Popen(cmd,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+  stdout, stderr = installToolProcess.communicate()
+  if installToolProcess.returncode != 0:
+    print('install_name_tool command failed: ')
+    print(stderr)
+  if verbose:
+    print(stdout)
 
 def should_test_parallel():
   if platform.system() == 'Linux':
@@ -105,7 +124,6 @@ def should_test_parallel():
       return False
   return True
 
-
 def handle_invocation(toolchain_bin, args):
   swiftpm_args = get_swiftpm_options(args)
 
@@ -138,19 +156,255 @@ def handle_invocation(toolchain_bin, args):
       test_args += ['--parallel']
     swiftpm('test', swift_exec, test_args, env)
   elif args.action == 'install':
-    bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, env)
-    swiftpm('build', swift_exec, swiftpm_args, env)
-    install(bin_path, args.toolchain)
+    if platform.system() == 'Darwin':
+      distribution_build_dir = os.path.join(args.build_path, 'dist')
+      build_for_distribution(args, toolchain_bin, distribution_build_dir)
+      install(args, distribution_build_dir)
+    else:
+      bin_path = swiftpm_bin_path(swift_exec, swiftpm_args, env)
+      swiftpm('build', swift_exec, swiftpm_args, env)
+      non_darwin_install(bin_path, args.toolchain, args.verbose)
   else:
     assert False, 'unknown action \'{}\''.format(args.action)
 
 
+# Installation flow for non-darwin platforms, only copies over swift-driver and swift-help
+# TODO: Unify CMake-based installation flow used on Darwin with this
+def non_darwin_install(swiftpm_bin_path, toolchain, verbose):
+  toolchain_bin = os.path.join(toolchain, 'bin')
+  for exe in ['swift-driver', 'swift-help']:
+    install_binary(exe, swiftpm_bin_path, toolchain_bin, verbose)
+
+def install(args, build_dir):
+  # Construct and install universal swift-driver, swift-help executables
+  # and libSwiftDriver, libSwiftOptions libraries
+  toolchain_bin = os.path.join(args.toolchain, 'bin')
+  toolchain_lib = os.path.join(args.toolchain, 'lib', 'swift', 'macosx')
+  universal_dir = os.path.join(build_dir, 'universal-apple-macos%s' % macos_deployment_target)
+  bin_dir = os.path.join(universal_dir, 'bin')
+  lib_dir = os.path.join(universal_dir, 'lib')
+  mkdir_p(universal_dir)
+  mkdir_p(bin_dir)
+  mkdir_p(lib_dir)
+
+  # swift-driver and swift-help
+  install_executables(args, build_dir, bin_dir, toolchain_bin)
+
+  # libSwiftDriver and libSwiftOptions
+  install_libraries(args, build_dir, lib_dir, toolchain_lib)
+
+  # SwiftDriver.swiftmodule and SwiftOptions.swiftmodule
+  install_modules(args, build_dir, toolchain_lib)
+
+def install_executables(args, build_dir, universal_bin_dir, toolchain_bin_dir):
+  for exe in ['swift-driver', 'swift-help']:
+    # Fixup rpaths
+    for arch in macos_target_architectures:
+      exe_bin_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                     'swift-driver', 'bin', exe)
+      help_bin_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                     'swift-driver', 'bin', 'swift-help')
+      driver_lib_dir_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                         'swift-driver', 'lib')
+      delete_rpath(driver_lib_dir_path, exe_bin_path, args.verbose)
+      # Only swift-driver requires libllbuild
+      if exe == 'swift-driver':
+        llbuild_lib_dir_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                          'llbuild', 'lib')
+        delete_rpath(llbuild_lib_dir_path, exe_bin_path, args.verbose)
+
+    # Merge the multiple architecture binaries into a universal binary and install
+    output_bin_path = os.path.join(universal_bin_dir, exe)
+    lipo_cmd = ['lipo']
+    # Inputs
+    for arch in macos_target_architectures:
+      input_bin_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                    'swift-driver', 'bin', exe)
+      lipo_cmd.append(input_bin_path)
+    lipo_cmd.extend(['-create', '-output', output_bin_path])
+    subprocess.check_call(lipo_cmd)
+    install_binary(exe, universal_bin_dir, toolchain_bin_dir, args.verbose)
+
+def install_libraries(args, build_dir, universal_lib_dir, toolchain_lib_dir):
+  # Fixup the rpath for libSwiftDriver (libSwiftOptions does not link against these libraries)
+  for arch in macos_target_architectures:
+    lib_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                   'swift-driver', 'lib', 'libSwiftDriver' + shared_lib_ext)
+    driver_lib_dir_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                       'swift-driver', 'lib')
+    llbuild_lib_dir_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                        'llbuild', 'lib')
+    delete_rpath(driver_lib_dir_path, lib_path, args.verbose)
+    delete_rpath(llbuild_lib_dir_path, lib_path, args.verbose)
+
+  # Install the libSwiftDriver and libSwiftOptions shared libraries into the toolchain lib
+  for lib in ['libSwiftDriver', 'libSwiftOptions']:
+    dylib_file = lib + shared_lib_ext
+    output_dylib_path = os.path.join(universal_lib_dir, dylib_file)
+    lipo_cmd = ['lipo']
+    for arch in macos_target_architectures:
+      input_lib_path = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                                    'swift-driver', 'lib', dylib_file)
+      lipo_cmd.append(input_lib_path)
+    lipo_cmd.extend(['-create', '-output', output_dylib_path])
+    subprocess.check_call(lipo_cmd)
+    install_binary(dylib_file, universal_lib_dir, toolchain_lib_dir, args.verbose)
+
+def install_modules(args, build_dir, toolchain_lib_dir):
+  for module in ['SwiftDriver', 'SwiftOptions']:
+    toolchain_module_dir = os.path.join(toolchain_lib_dir, module + '.swiftmodule')
+    mkdir_p(toolchain_module_dir)
+    for arch in macos_target_architectures:
+      swift_dir = os.path.join(build_dir, arch + '-apple-macos' + macos_deployment_target,
+                               'swift-driver', 'swift')
+      for fileext in ['.swiftmodule', '.swiftdoc']:
+        install_binary(module + fileext, swift_dir, toolchain_module_dir, args.verbose)
+        os.rename(os.path.join(toolchain_module_dir, module + fileext),
+                  os.path.join(toolchain_module_dir, arch + '-apple-macos' + fileext))
+
+def build_for_distribution(args, toolchain_bin, build_dir):
+  print('Preparing SwiftDriver for distribution using CMake.')
+  swiftc_exec = os.path.join(toolchain_bin, 'swiftc')
+
+  # Targets for which we must build swift-driver to distribute
+  if platform.system() == 'Darwin':
+    targets = [x + '-apple-macos' + macos_deployment_target for x in macos_target_architectures]
+  else:
+    targets = [get_build_target(swiftc_path, args)]
+
+  for target in targets:
+    base_cmake_flags = [
+      '-DCMAKE_Swift_FLAGS=-target %s' % target,
+    ]
+    if platform.system() == 'Darwin':
+      base_cmake_flags.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=%s' % macos_deployment_target)
+
+    # Target directory to build distribution artifacts
+    cmake_target_dir = os.path.join(build_dir, target)
+    # LLBuild
+    build_llbuild_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags)
+    # TSC
+    build_tsc_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags)
+    # Argument Parser
+    build_argument_parser_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags)
+    # Yams
+    build_yams_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags)
+    # SwiftDriver
+    build_swift_driver_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags)
+
+def build_llbuild_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags):
+  print('Building llbuild for target: %s' % target)
+  llbuild_source_dir = os.path.join(args.package_path, '..', 'llbuild')
+  llbuild_build_dir = os.path.join(cmake_target_dir, 'llbuild')
+  llbuild_api_dir = os.path.join(llbuild_build_dir, '.cmake/api/v1/query')
+  mkdir_p(llbuild_api_dir)
+  subprocess.check_call(['touch', os.path.join(llbuild_api_dir, 'codemodel-v2')])
+  flags = [
+        '-DBUILD_SHARED_LIBS=OFF',
+        '-DCMAKE_C_COMPILER:=clang',
+        '-DCMAKE_CXX_COMPILER:=clang++',
+        '-DCMAKE_CXX_FLAGS=-target %s' % target,
+        '-DLLBUILD_SUPPORT_BINDINGS:=Swift'
+    ]
+  if platform.system() == 'Darwin':
+    flags.append('-DCMAKE_OSX_ARCHITECTURES=%s' % target.split('-')[0])
+  llbuild_cmake_flags = base_cmake_flags + flags
+  if args.sysroot:
+    llbuild_cmake_flags.append('-DSQLite3_INCLUDE_DIR=%s/usr/include' % args.sysroot)
+  cmake_build(args, swiftc_exec, llbuild_cmake_flags, llbuild_source_dir, llbuild_build_dir)
+
+def build_tsc_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags):
+  print('Building TSC for target: %s' % target)
+  tsc_source_dir = os.path.join(args.package_path, '..', 'swift-tools-support-core')
+  tsc_build_dir = os.path.join(cmake_target_dir, 'swift-tools-support-core')
+  tsc_flags = base_cmake_flags + ['-DBUILD_SHARED_LIBS=OFF']
+  cmake_build(args, swiftc_exec, tsc_flags, tsc_source_dir, tsc_build_dir)
+
+def build_yams_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags):
+  print('Building Yams for target: %s' % target)
+  yams_source_dir = os.path.join(args.package_path, '..', 'yams')
+  yams_build_dir = os.path.join(cmake_target_dir, 'yams')
+  yams_flags = base_cmake_flags + ['-DBUILD_SHARED_LIBS=OFF', '-DCMAKE_C_FLAGS=-target %s' % target]
+  cmake_build(args, swiftc_exec, yams_flags, yams_source_dir, yams_build_dir)
+
+def build_argument_parser_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags):
+  print('Building Argument Parser for target: %s' % target)
+  parser_source_dir = os.path.join(args.package_path, '..', 'swift-argument-parser')
+  parser_build_dir = os.path.join(cmake_target_dir, 'swift-argument-parser')
+  custom_flags = ['-DBUILD_SHARED_LIBS=OFF', '-DBUILD_TESTING=NO', '-DBUILD_EXAMPLES=NO']
+  parser_flags = base_cmake_flags + custom_flags
+  cmake_build(args, swiftc_exec, parser_flags, parser_source_dir, parser_build_dir)
+  return
+
+def build_swift_driver_using_cmake(args, target, swiftc_exec, cmake_target_dir, base_cmake_flags):
+  print('Building Swift Driver for target: %s' % target)
+  driver_source_dir = args.package_path
+  driver_build_dir = os.path.join(cmake_target_dir, 'swift-driver')
+  # TODO: Enable Library Evolution for swift-driver
+  #swift_flags = '-DCMAKE_Swift_FLAGS=' + '-enable-library-evolution' + ' -emit-module-interface-path ' + os.path.join(cmake_target_dir, 'swift-driver', 'SwiftDriver.swiftinterface') + ' -Xfrontend -module-interface-preserve-types-as-written'
+  swift_flags = ''
+  flags = [
+        '-DLLBuild_DIR=' + os.path.join(os.path.join(cmake_target_dir, 'llbuild'), 'cmake/modules'),
+        '-DTSC_DIR=' + os.path.join(os.path.join(cmake_target_dir, 'swift-tools-support-core'), 'cmake/modules'),
+        '-DYams_DIR=' + os.path.join(os.path.join(cmake_target_dir, 'yams'), 'cmake/modules'),
+        '-DArgumentParser_DIR=' + os.path.join(os.path.join(cmake_target_dir, 'swift-argument-parser'), 'cmake/modules'),
+        swift_flags
+    ]
+  driver_cmake_flags = base_cmake_flags + flags
+  cmake_build(args, swiftc_exec, driver_cmake_flags, driver_source_dir, driver_build_dir)
+
+def cmake_build(args, swiftc_exec, cmake_args, source_path, build_dir):
+  """Configure with CMake and build with Ninja"""
+  swift_flags = ''
+  if args.sysroot:
+    swift_flags = '-sdk %s' % args.sysroot
+  cmd = [
+    args.cmake_bin, '-G', 'Ninja',
+    '-DCMAKE_MAKE_PROGRAM=%s' % args.ninja_bin,
+    '-DCMAKE_BUILD_TYPE:=Release',
+    '-DCMAKE_Swift_FLAGS=' + swift_flags,
+    '-DCMAKE_Swift_COMPILER:=%s' % (swiftc_exec),
+  ] + cmake_args + [source_path]
+  if args.verbose:
+    print(' '.join(cmd))
+  mkdir_p(build_dir)
+  subprocess.check_output(cmd, cwd=build_dir)
+
+  # Invoke Ninja
+  ninja_cmd = [args.ninja_bin]
+  if args.verbose:
+    ninja_cmd.append('-v')
+    print(' '.join(ninja_cmd))
+  ninjaProcess = subprocess.Popen(ninja_cmd, cwd=build_dir,
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE,
+                                  env = os.environ)
+  stdout, stderr = ninjaProcess.communicate()
+  if ninjaProcess.returncode != 0:
+    print('Ninja invocation failed: ')
+    print(stderr)
+    sys.exit(ninjaProcess.returncode)
+  if args.verbose:
+    print(stdout)
+
+def get_build_target(swiftc_path, args):
+    """Returns the target-triple of the current machine."""
+    try:
+        target_info_json = subprocess.check_output([swiftc_path, '-print-target-info'],
+                                                   stderr=subprocess.PIPE,
+                                                   universal_newlines=True).strip()
+        args.target_info = json.loads(target_info_json)
+        return args.target_info['target']['unversionedTriple']
+    except Exception as e:
+        error(str(e))
+
 def main():
   parser = argparse.ArgumentParser(description='Build along with the Swift build-script.')
   def add_common_args(parser):
     parser.add_argument('--package-path', metavar='PATH', help='directory of the package to build', default='.')
     parser.add_argument('--toolchain', required=True, metavar='PATH', help='build using the toolchain at PATH')
     parser.add_argument('--ninja-bin', metavar='PATH', help='ninja binary to use for testing')
+    parser.add_argument('--cmake-bin', metavar='PATH', help='cmake binary to use for building')
     parser.add_argument('--build-path', metavar='PATH', default='.build', help='build in the given path')
     parser.add_argument('--configuration', '-c', default='debug', help='build using configuration (release|debug)')
     parser.add_argument('--no-local-deps', action='store_true', help='use normal remote dependencies when building')