[build] Allow cross-compiling build-script products for non-Darwin hosts too

To that end, move the --cross-compile-deps-path flag from build-script-impl to a
publicly documented build-script flag and use it for build-script products'
library dependencies too. Generate a SPM destination JSON file that can be used
both for cross-compiling these build-script products and by users for their own
Swift packages.

Also, add a native_toolchain_path() method, that uses a prebuilt toolchain if
available, and pass an install prefix in to swift-driver and sourcekit-lsp.

Author: finagolfin

utils/build_swift/build_swift/driver_arguments.py

@@ -563,6 +563,11 @@ def create_argument_parser():
            help='A space separated list of targets to cross-compile host '
                 'Swift tools for. Can be used multiple times.')
 
+    option('--cross-compile-deps-path', store_path,
+           help='The path to a directory that contains prebuilt cross-compiled '
+                'library dependencies of the corelibs and other Swift repos, '
+                'such as the libcurl dependency of FoundationNetworking')
+
     option('--stdlib-deployment-targets', store,
            type=argparse.ShellSplitType(),
            default=None,

utils/build_swift/tests/expected_options.py

@@ -131,6 +131,7 @@
     'cmark_build_variant': 'Debug',
     'compiler_vendor': defaults.COMPILER_VENDOR,
     'coverage_db': None,
+    'cross_compile_deps_path': None,
     'cross_compile_hosts': [],
     'darwin_deployment_version_ios':
         defaults.DARWIN_DEPLOYMENT_VERSION_IOS,
@@ -694,6 +695,7 @@ class BuildScriptImplOption(_BaseOption):
     PathOption('--clang-profile-instr-use'),
     PathOption('--cmake'),
     PathOption('--coverage-db'),
+    PathOption('--cross-compile-deps-path'),
     PathOption('--host-cc'),
     PathOption('--host-cxx'),
     PathOption('--host-libtool'),

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -181,6 +181,10 @@ def convert_to_impl_arguments(self):
         if args.cross_compile_hosts:
             impl_args += [
                 "--cross-compile-hosts", " ".join(args.cross_compile_hosts)]
+        if args.cross_compile_deps_path is not None:
+            impl_args += [
+                "--cross-compile-deps-path=%s" % args.cross_compile_deps_path
+            ]
 
         if args.test_paths:
             impl_args += ["--test-paths", " ".join(args.test_paths)]
@@ -664,12 +668,14 @@ def execute(self):
                 self._execute_impl(pipeline, all_hosts, perform_epilogue_opts)
             else:
                 assert(index != last_impl_index)
-                # Once we have performed our last impl pipeline, we no longer
-                # support cross compilation.
-                #
-                # This just maintains current behavior.
                 if index > last_impl_index:
-                    self._execute(pipeline, [self.args.host_target])
+                    non_darwin_cross_compile_hostnames = [
+                        target for target in self.args.cross_compile_hosts if not
+                        StdlibDeploymentTarget.get_target_for_name(
+                            target).platform.is_darwin
+                    ]
+                    self._execute(pipeline, [self.args.host_target] +
+                                  non_darwin_cross_compile_hostnames)
                 else:
                     self._execute(pipeline, all_host_names)
 
@@ -727,6 +733,8 @@ def _execute_impl(self, pipeline, all_hosts, should_run_epilogue_operations):
 
     def _execute(self, pipeline, all_host_names):
         for host_target in all_host_names:
+            if self.args.skip_local_build and host_target == self.args.host_target:
+                continue
             for product_class in pipeline:
                 # Execute clean, build, test, install
                 self.execute_product_build_steps(product_class, host_target)

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -85,8 +85,7 @@ def run_build_script_helper(action, host_target, product, args,
         install_destdir = swiftpm.SwiftPM.get_install_destdir(args,
                                                               host_target,
                                                               product.build_dir)
-    toolchain_path = targets.toolchain_path(install_destdir,
-                                            args.install_prefix)
+    toolchain_path = product.native_toolchain_path(host_target)
     is_release = product.is_release()
     configuration = 'release' if is_release else 'debug'
     helper_cmd = [
@@ -110,4 +109,22 @@ def run_build_script_helper(action, host_target, product, args,
     elif args.enable_tsan:
         helper_cmd.extend(['--sanitize', 'thread'])
 
+    if not product.is_darwin_host(
+            host_target) and product.is_cross_compile_target(host_target):
+        helper_cmd.extend(['--cross-compile-host', host_target])
+        build_toolchain_path = install_destdir + args.install_prefix
+        resource_dir = '%s/lib/swift' % build_toolchain_path
+        helper_cmd += [
+            '--cross-compile-config',
+            targets.StdlibDeploymentTarget.get_target_for_name(host_target).platform
+            .swiftpm_config(args, output_dir=build_toolchain_path,
+                            swift_toolchain=toolchain_path,
+                            resource_path=resource_dir)
+        ]
+
+    if action == 'install' and product.product_name() == "sourcekitlsp":
+        helper_cmd.extend([
+            '--prefix', install_destdir + args.install_prefix
+        ])
+
     shell.call(helper_cmd)

utils/swift_build_support/swift_build_support/products/product.py

@@ -195,18 +195,25 @@ def install_toolchain_path(self, host_target):
         """toolchain_path() -> string
 
         Returns the path to the toolchain that is being created as part of this
-        build, or to a native prebuilt toolchain that was passed in.
+        build
         """
-        if self.args.native_swift_tools_path is not None:
-            return os.path.split(self.args.native_swift_tools_path)[0]
-
         install_destdir = self.args.install_destdir
         if self.args.cross_compile_hosts:
-            build_root = os.path.dirname(self.build_dir)
-            install_destdir = '%s/intermediate-install/%s' % (build_root, host_target)
+            if self.is_darwin_host(host_target):
+                build_root = os.path.dirname(self.build_dir)
+                install_destdir = '%s/intermediate-install/%s' % (build_root,
+                                                                  host_target)
+            else:
+                install_destdir = os.path.join(install_destdir, self.args.host_target)
         return targets.toolchain_path(install_destdir,
                                       self.args.install_prefix)
 
+    def native_toolchain_path(self, host_target):
+        if self.args.native_swift_tools_path is not None:
+            return os.path.split(self.args.native_swift_tools_path)[0]
+        else:
+            return self.install_toolchain_path(host_target)
+
     def is_darwin_host(self, host_target):
         return host_target.startswith("macosx") or \
             host_target.startswith("iphone") or \

utils/swift_build_support/swift_build_support/products/swiftdriver.py

@@ -91,8 +91,7 @@ def run_build_script_helper(action, host_target, product, args):
         install_destdir = swiftpm.SwiftPM.get_install_destdir(args,
                                                               host_target,
                                                               product.build_dir)
-    toolchain_path = targets.toolchain_path(install_destdir,
-                                            args.install_prefix)
+    toolchain_path = product.native_toolchain_path(host_target)
 
     # Pass Dispatch directory down if we built it
     dispatch_build_dir = os.path.join(
@@ -134,10 +133,26 @@ def run_build_script_helper(action, host_target, product, args):
         ]
     # Pass Cross compile host info
     if swiftpm.SwiftPM.has_cross_compile_hosts(args):
-        helper_cmd += ['--cross-compile-hosts']
-        for cross_compile_host in args.cross_compile_hosts:
-            helper_cmd += [cross_compile_host]
+        if product.is_darwin_host(host_target):
+            helper_cmd += ['--cross-compile-hosts']
+            for cross_compile_host in args.cross_compile_hosts:
+                helper_cmd += [cross_compile_host]
+        elif product.is_cross_compile_target(host_target):
+            helper_cmd += ['--cross-compile-hosts', host_target]
+            build_toolchain_path = install_destdir + args.install_prefix
+            resource_dir = '%s/lib/swift' % build_toolchain_path
+            helper_cmd += [
+                '--cross-compile-config',
+                targets.StdlibDeploymentTarget.get_target_for_name(
+                    host_target).platform.swiftpm_config(
+                    args, output_dir=build_toolchain_path,
+                    swift_toolchain=toolchain_path, resource_path=resource_dir)]
     if args.verbose_build:
         helper_cmd.append('--verbose')
 
+    if action == 'install':
+        helper_cmd += [
+            '--prefix', install_destdir + args.install_prefix
+        ]
+
     shell.call(helper_cmd)

utils/swift_build_support/swift_build_support/products/swiftpm.py

@@ -23,6 +23,7 @@
 from . import swift
 from . import xctest
 from .. import shell
+from ..targets import StdlibDeploymentTarget
 
 
 class SwiftPM(product.Product):
@@ -44,7 +45,8 @@ def should_build(self, host_target):
     def run_bootstrap_script(self, action, host_target, additional_params=[]):
         script_path = os.path.join(
             self.source_dir, 'Utilities', 'bootstrap')
-        toolchain_path = self.install_toolchain_path(host_target)
+
+        toolchain_path = self.native_toolchain_path(host_target)
         swiftc = os.path.join(toolchain_path, "bin", "swiftc")
 
         # FIXME: We require llbuild build directory in order to build. Is
@@ -92,9 +94,22 @@ def run_bootstrap_script(self, action, host_target, additional_params=[]):
 
         # Pass Cross compile host info
         if self.has_cross_compile_hosts(self.args):
-            helper_cmd += ['--cross-compile-hosts']
-            for cross_compile_host in self.args.cross_compile_hosts:
-                helper_cmd += [cross_compile_host]
+            if self.is_darwin_host(host_target):
+                helper_cmd += ['--cross-compile-hosts']
+                for cross_compile_host in self.args.cross_compile_hosts:
+                    helper_cmd += [cross_compile_host]
+            elif self.is_cross_compile_target(host_target):
+                helper_cmd += ['--cross-compile-hosts', host_target,
+                               '--skip-cmake-bootstrap']
+                build_toolchain_path = self.get_install_destdir(
+                    self.args, host_target, self.build_dir) + self.args.install_prefix
+                resource_dir = '%s/lib/swift' % build_toolchain_path
+                helper_cmd += [
+                    '--cross-compile-config',
+                    StdlibDeploymentTarget.get_target_for_name(host_target).platform
+                    .swiftpm_config(self.args, output_dir=build_toolchain_path,
+                                    swift_toolchain=toolchain_path,
+                                    resource_path=resource_dir)]
 
         helper_cmd.extend(additional_params)
 
@@ -126,8 +141,12 @@ def has_cross_compile_hosts(self, args):
     def get_install_destdir(self, args, host_target, build_dir):
         install_destdir = args.install_destdir
         if self.has_cross_compile_hosts(args):
-            build_root = os.path.dirname(build_dir)
-            install_destdir = '%s/intermediate-install/%s' % (build_root, host_target)
+            if self.is_darwin_host(self, host_target):
+                build_root = os.path.dirname(build_dir)
+                install_destdir = '%s/intermediate-install/%s' % (
+                                  build_root, host_target)
+            else:
+                install_destdir = os.path.join(install_destdir, host_target)
         return install_destdir
 
     def install(self, host_target):

utils/swift_build_support/swift_build_support/targets.py

@@ -80,6 +80,13 @@ def cmake_options(self, args):
         """
         return ''
 
+    def swiftpm_config(self, args, output_dir, swift_toolchain, resource_path):
+        """
+        Generate a JSON file that SPM can use to cross-compile
+        """
+        raise NotImplementedError('Generating a SwiftPM cross-compilation JSON file '
+                                  'for %s is not supported yet' % self.name)
+
 
 class DarwinPlatform(Platform):
     def __init__(self, name, archs, sdk_name=None, is_simulator=False):
@@ -155,8 +162,7 @@ def swift_flags(self, args):
         flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % (
                  args.build_root, self.name, args.android_arch)
 
-        android_toolchain_path = '%s/toolchains/llvm/prebuilt/%s' % (
-            args.android_ndk, StdlibDeploymentTarget.host_target().name)
+        android_toolchain_path = self.ndk_toolchain_path(args)
 
         flags += '-sdk %s/sysroot ' % (android_toolchain_path)
         flags += '-tools-directory %s/bin' % (android_toolchain_path)
@@ -171,6 +177,42 @@ def cmake_options(self, args):
         options += '-DCMAKE_ANDROID_NDK:PATH=%s' % (args.android_ndk)
         return options
 
+    def ndk_toolchain_path(self, args):
+        return '%s/toolchains/llvm/prebuilt/%s' % (
+            args.android_ndk, StdlibDeploymentTarget.host_target().name)
+
+    def swiftpm_config(self, args, output_dir, swift_toolchain, resource_path):
+        config_file = '%s/swiftpm-android-%s.json' % (output_dir, args.android_arch)
+
+        if os.path.exists(config_file):
+            print("Using existing config at %s" % config_file)
+            return config_file
+
+        spm_json = '{\n'
+        spm_json += '  "version": 1,\n'
+        spm_json += '  "target": "%s-unknown-linux-android%s",\n' % (
+                    args.android_arch, args.android_api_level)
+        spm_json += '  "toolchain-bin-dir": "%s/bin",\n' % swift_toolchain
+        spm_json += '  "sdk": "%s/sysroot",\n' % self.ndk_toolchain_path(args)
+
+        spm_json += '  "extra-cc-flags": [ "-fPIC", "-I%s/usr/include" ],\n' % (
+                    args.cross_compile_deps_path)
+
+        spm_json += '  "extra-swiftc-flags": [\n'
+        spm_json += '    "-resource-dir", "%s",\n' % resource_path
+        spm_json += '    "-tools-directory", "%s/bin",\n' % (
+                    self.ndk_toolchain_path(args))
+        spm_json += '    "-Xcc", "-I%s/usr/include",\n' % args.cross_compile_deps_path
+        spm_json += '    "-L%s/usr/lib"\n' % args.cross_compile_deps_path
+        spm_json += '  ],\n'
+
+        spm_json += '  "extra-cpp-flags": [ "-lstdc++" ]\n'
+        spm_json += '}'
+
+        with open(config_file, 'w') as f:
+            f.write(spm_json)
+        return config_file
+
 
 class Target(object):
     """