Can build with toolchain other than swift-build’s

This is an essential part of the puzzle that is:

* swift-build requires Swift 3 to build and consume Package.swift
* Many packages exist that require Swift 2.2

The workaround here is to set `SWIFT_EXEC`, swift-build will use its toolchain for the parts that are intrinsically coupled to the toolchain, but composes a build-YAML that uses the toolchain with which `SWIFT_EXEC` is a part.

Since SWIFT_EXEC is an expected feature, we are adapting it here, but it may make more sense long-term to have a dedicated `--toolchain` flag.

Author: mxcl

Sources/Build/Command.compile(ClangModule).swift

@@ -15,7 +15,7 @@ import POSIX
 //FIXME: Incremental builds
 
 extension Command {
-    static func compile(clangModule module: ClangModule, externalModules: Set<Module>, configuration conf: Configuration, prefix: String) -> (Command, Command) {
+    static func compile(clangModule module: ClangModule, externalModules: Set<Module>, configuration conf: Configuration, prefix: String, CC: String) -> (Command, Command) {
 
         let wd = Path.join(prefix, "\(module.c99name).build")
         let mkdir = Command.createDirectory(wd)
@@ -61,7 +61,7 @@ extension Command {
             description: "Compiling \(module.name)",
             inputs: inputs,
             outputs: [productPath, module.targetName],
-            args: [Toolchain.clang] + args)
+            args: [CC] + args)
 
         let command = Command(node: module.targetName, tool: clang)
 

Sources/Build/Command.compile(SwiftModule).swift

@@ -12,7 +12,7 @@ import PackageType
 import Utility
 
 extension Command {
-    static func compile(swiftModule module: SwiftModule, configuration conf: Configuration, prefix: String, otherArgs: [String]) throws -> (Command, [Command]) {
+    static func compile(swiftModule module: SwiftModule, configuration conf: Configuration, prefix: String, otherArgs: [String], SWIFT_EXEC: String) throws -> (Command, [Command]) {
 
         let otherArgs = otherArgs + module.Xcc
 
@@ -24,15 +24,11 @@ extension Command {
         case .Debug:
             var args = ["-j8","-Onone","-g","-D","SWIFT_PACKAGE", "-enable-testing"]
 
-        #if os(OSX)
-            if let platformPath = Toolchain.platformPath {
-                let path = Path.join(platformPath, "Developer/Library/Frameworks")
-                args += ["-F", path]
-            } else {
-                throw Error.InvalidPlatformPath
-            }
-        #endif
-            let tool = SwiftcTool(module: module, prefix: prefix, otherArgs: args + otherArgs)
+          #if os(OSX)
+            args += ["-F", try platformFrameworksPath()]
+          #endif
+
+            let tool = SwiftcTool(module: module, prefix: prefix, otherArgs: args + otherArgs, executable: SWIFT_EXEC)
             let mkdirs = Set(tool.objects.map{ $0.parentDirectory }).map(Command.createDirectory)
             return (cmd(tool), mkdirs)
 
@@ -49,7 +45,7 @@ extension Command {
                 description: "Compiling \(module.name)",
                 inputs: inputs,
                 outputs: [productPath, module.targetName],
-                args: [Toolchain.swiftc, "-o", productPath] + args + module.sources.paths + otherArgs)
+                args: [SWIFT_EXEC, "-o", productPath] + args + module.sources.paths + otherArgs)
 
             return (cmd(tool), [])
         }

Sources/Build/Command.link().swift

@@ -17,22 +17,22 @@ import Utility
 
 
 extension Command {
-    static func link(product: Product, configuration conf: Configuration, prefix: String, otherArgs: [String]) throws -> Command {
+    static func link(product: Product, configuration conf: Configuration, prefix: String, otherArgs: [String], SWIFT_EXEC: String) throws -> Command {
 
         let objects: [String]
         switch conf {
         case .Release:
             objects = product.buildables.map{ Path.join(prefix, "\($0.c99name).o") }
         case .Debug:
-            objects = product.buildables.flatMap{ return SwiftcTool(module: $0, prefix: prefix, otherArgs: []).objects }
+            objects = product.buildables.flatMap{ return SwiftcTool(module: $0, prefix: prefix, otherArgs: [], executable: SWIFT_EXEC).objects }
         }
 
         let outpath = Path.join(prefix, product.outname)
 
         var args: [String]
         switch product.type {
         case .Library(.Dynamic), .Executable, .Test:
-            args = [Toolchain.swiftc] + otherArgs
+            args = [SWIFT_EXEC] + otherArgs
 
             if conf == .Debug {
                 args += ["-g"]
@@ -50,7 +50,7 @@ extension Command {
         let inputs = product.modules.flatMap { module -> [String] in
             switch conf {
             case .Debug:
-                let tool = SwiftcTool(module: module, prefix: prefix, otherArgs: [])
+                let tool = SwiftcTool(module: module, prefix: prefix, otherArgs: [], executable: SWIFT_EXEC)
                 // must return tool’s outputs and inputs as shell nodes don't calculate more than that
                 return tool.inputs + tool.outputs
             case .Release:
@@ -64,13 +64,7 @@ extension Command {
         case .Test:
           #if os(OSX)
             args += ["-Xlinker", "-bundle"]
-
-            if let platformPath = Toolchain.platformPath {
-                let path = Path.join(platformPath, "Developer/Library/Frameworks")
-                args += ["-F", path]
-            } else {
-                throw Error.InvalidPlatformPath
-            }
+            args += ["-F", try platformFrameworksPath()]
 
             // TODO should be llbuild rules∫
             if conf == .Debug {

Sources/Build/ToolProtocol.swift

@@ -58,13 +58,13 @@ struct SwiftcTool: ToolProtocol {
     let module: SwiftModule
     let prefix: String
     let otherArgs: [String]
+    let executable: String
 
     var inputs: [String] {
         return module.recursiveDependencies.map{ $0.targetName }
     }
 
     var outputs: [String]        { return [module.targetName] + objects }
-    var executable: String       { return Toolchain.swiftc }
     var moduleName: String       { return module.c99name }
     var moduleOutputPath: String { return Path.join(prefix, "\(module.c99name).swiftmodule") }
     var importPaths: String      { return prefix }

Sources/Build/describe().swift

@@ -18,7 +18,7 @@ import Utility
 /**
   - Returns: path to generated YAML for consumption by the llbuild based swift-build-tool
 */
-public func describe(prefix: String, _ conf: Configuration, _ modules: [Module], _ externalModules: Set<Module> , _ products: [Product], Xcc: [String], Xld: [String], Xswiftc: [String]) throws -> String {
+public func describe(prefix: String, _ conf: Configuration, _ modules: [Module], _ externalModules: Set<Module>, _ products: [Product], Xcc: [String], Xld: [String], Xswiftc: [String], toolchain: Toolchain) throws -> String {
 
     guard modules.count > 0 else {
         throw Error.NoModules
@@ -29,13 +29,16 @@ public func describe(prefix: String, _ conf: Configuration, _ modules: [Module],
     let prefix = try mkdir(prefix, conf.dirname)  //TODO llbuild this
     let swiftcArgs = Xcc + Xswiftc + verbosity.ccArgs
 
+    let SWIFT_EXEC = toolchain.SWIFT_EXEC
+    let CC = getenv("CC") ?? "clang"
+
     var commands = [Command]()
     var targets = Targets()
 
     for module in modules {
         switch module {
         case let module as SwiftModule:
-            let (compile, mkdirs) = try Command.compile(swiftModule: module, configuration: conf, prefix: prefix, otherArgs: swiftcArgs + platformArgs())
+            let (compile, mkdirs) = try Command.compile(swiftModule: module, configuration: conf, prefix: prefix, otherArgs: swiftcArgs + toolchain.platformArgs, SWIFT_EXEC: SWIFT_EXEC)
             commands.append(contentsOf: mkdirs + [compile])
             targets.append(compile, for: module)
 
@@ -50,7 +53,7 @@ public func describe(prefix: String, _ conf: Configuration, _ modules: [Module],
                 }
             }
 
-            let (compile, mkdir) = Command.compile(clangModule: module, externalModules: externalModules, configuration: conf, prefix: prefix)
+            let (compile, mkdir) = Command.compile(clangModule: module, externalModules: externalModules, configuration: conf, prefix: prefix, CC: CC)
             commands.append(compile)
             commands.append(mkdir)
             targets.main.cmds.append(compile)
@@ -64,7 +67,7 @@ public func describe(prefix: String, _ conf: Configuration, _ modules: [Module],
     }
 
     for product in products {
-        let command = try Command.link(product, configuration: conf, prefix: prefix, otherArgs: Xld + swiftcArgs + platformArgs())
+        let command = try Command.link(product, configuration: conf, prefix: prefix, otherArgs: Xld + swiftcArgs + toolchain.platformArgs, SWIFT_EXEC: SWIFT_EXEC)
         commands.append(command)
         targets.append(command, for: product)
     }

Sources/Build/misc.swift

@@ -9,21 +9,23 @@
 */
 
 import func POSIX.getenv
+import func POSIX.popen
 import PackageType
 import Utility
 
-func platformArgs() -> [String] {
-    var args = [String]()
-
-#if os(OSX)
-    args += ["-target", "x86_64-apple-macosx10.10"]
+public protocol Toolchain {
+    var platformArgs: [String] { get }
+    var sysroot: String?  { get }
+    var SWIFT_EXEC: String { get }
+    var clang: String { get }
+}
 
-    if let sysroot = Toolchain.sysroot {
-        args += ["-sdk", sysroot]
+func platformFrameworksPath() throws -> String {
+    guard let  popened = try? POSIX.popen(["xcrun", "--sdk", "macosx", "--show-sdk-platform-path"]),
+        let chuzzled = popened.chuzzle() else {
+            throw Error.InvalidPlatformPath
     }
-#endif
-
-    return args
+    return Path.join(chuzzled, "Developer/Library/Frameworks")
 }
 
 extension CModule {

Sources/Multitool/Error.swift

@@ -22,13 +22,22 @@ public enum CommandLineError: ErrorProtocol {
 
 public enum Error: ErrorProtocol {
     case NoManifestFound
+    case InvalidToolchain
+    case InvalidInstallation(String)
+    case InvalidSwiftExec(String)
 }
 
 extension Error: CustomStringConvertible {
     public var description: String {
         switch self {
         case .NoManifestFound:
             return "no \(Manifest.filename) file found"
+        case .InvalidToolchain:
+            return "invalid inferred toolchain"
+        case .InvalidInstallation(let prefix):
+            return "invalid or incomplete Swift toolchain:\n    \(prefix)"
+        case .InvalidSwiftExec(let value):
+            return "invalid SWIFT_EXEC value: \(value)"
         }
     }
 }

Sources/Multitool/Toolchain.swift

@@ -0,0 +1,27 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright 2015 - 2016 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import PackageType
+import Utility
+import POSIX
+
+#if Xcode
+
+    // when in Xcode we are built with same toolchain as we will run
+    // this is not a production ready mode
+
+    public let SWIFT_EXEC = getenv("SWIFT_EXEC")!.abspath()
+    public let llbuild = Path.join(getenv("SWIFT_EXEC")!, "../swift-build-tool").abspath()
+    public let libdir = Process.arguments.first!.parentDirectory
+#else
+    public let SWIFT_EXEC = Path.join(Process.arguments.first!, "../swiftc").abspath()
+    public let llbuild = Path.join(Process.arguments.first!, "../swift-build-tool").abspath()
+    public let libdir = Path.join(Process.arguments.first!, "../../lib/swift/pm").abspath()
+#endif

Sources/Multitool/build().swift

@@ -9,22 +9,11 @@
 */
 
 import func POSIX.getenv
+import PackageType
 import Utility
 
 public func build(YAMLPath YAMLPath: String, target: String) throws {
-    var args = [swift_build_tool(), "-f", YAMLPath, target]
+    var args = [llbuild, "-f", YAMLPath, target]
     if verbosity != .Concise { args.append("-v") }
     try system(args)
 }
-
-private func swift_build_tool() -> String {
-    if let tool = getenv("SWIFT_BUILD_TOOL") {  //FIXME remove and if people complain, make it a flag
-        return tool
-    } else if let path = try? Path.join(exepath, "..", "swift-build-tool").abspath() where path.isFile {
-        return path
-    } else {
-        return Toolchain.which("swift-build-tool")
-    }
-}
-
-private let exepath: String = try! Process.arguments.first!.abspath()

Sources/POSIX/system.swift

@@ -49,7 +49,12 @@ func posix_spawnp(path: String, args: [String], environment: [String: String] =
     defer { for arg in argv { free(arg) } }
 
     var environment = environment
-    for key in ["PATH", "SDKROOT", "TOOLCHAINS", "HOME", "SWIFT_EXEC"] {
+#if Xcode
+    let keys = ["SWIFT_EXEC", "HOME", "PATH"]
+#else
+    let keys = ["SWIFT_EXEC", "HOME", "PATH", "SDKROOT", "TOOLCHAINS"]
+#endif
+    for key in keys {
         if environment[key] == nil {
             environment[key] = POSIX.getenv(key)
         }

Sources/Utility/Path.swift

@@ -222,7 +222,7 @@ extension String {
 
     /**
      - Returns: true if the string is a file on the filesystem
-     - Note: if the entry is a symlink, but the symlink points to Array
+     - Note: if the entry is a symlink, but the symlink points to a
        file, then this function returns true. Use `isSymlink` if the
        distinction is important.
      */