Compile manifests instead of interpreting them

Author: hartbit

Sources/PackageLoading/ManifestLoader.swift

@@ -531,39 +531,39 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
             // Compute the path to runtime we need to load.
             let runtimePath = self.runtimePath(for: toolsVersion)
-            let interpreterFlags = self.interpreterFlags(for: toolsVersion)
+            let compilerFlags = self.compilerFlags(for: toolsVersion)
 
             // FIXME: Workaround for the module cache bug that's been haunting Swift CI
             // <rdar://problem/48443680>
             let moduleCachePath = ProcessEnv.vars["SWIFTPM_MODULECACHE_OVERRIDE"] ?? ProcessEnv.vars["SWIFTPM_TESTS_MODULECACHE"]
 
-            var cmd = [String]()
-          #if os(macOS)
-            // If enabled, use sandbox-exec on macOS. This provides some safety against
-            // arbitrary code execution when parsing manifest files. We only allow
-            // the permissions which are absolutely necessary for manifest parsing.
-            if isManifestSandboxEnabled {
-                let cacheDirs = [
-                    cacheDir,
-                    moduleCachePath.map{ AbsolutePath($0) }
-                ].compactMap{$0}
-                cmd += ["sandbox-exec", "-p", sandboxProfile(cacheDirs)]
-            }
-          #endif
+            var cmd: [String] = []
             cmd += [resources.swiftCompiler.pathString]
-            cmd += ["--driver-mode=swift"]
             cmd += verbosity.ccArgs
 
             // If we got the binDir that means we could be developing SwiftPM in Xcode
             // which produces a framework for dynamic package products.
-            let runtimeFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
-            if resources.binDir != nil, localFileSystem.exists(runtimeFrameworkPath)  {
-                cmd += ["-F", runtimeFrameworkPath.pathString, "-framework", "PackageDescription"]
+            let packageFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
+            if resources.binDir != nil, localFileSystem.exists(packageFrameworkPath)  {
+                cmd += [
+                    "-F", packageFrameworkPath.pathString,
+                    "-framework", "PackageDescription",
+                    "-Xlinker", "-rpath", "-Xlinker", packageFrameworkPath.pathString,
+                ]
             } else {
-                cmd += ["-L", runtimePath.pathString, "-lPackageDescription"]
+              #if os(macOS)
+                let rpath = resources.libDir.parentDirectory
+              #else
+                let rpath = runtimePath
+              #endif
+                cmd += [
+                    "-L", runtimePath.pathString,
+                    "-lPackageDescription",
+                    "-Xlinker", "-rpath", "-Xlinker", rpath.pathString
+                ]
             }
 
-            cmd += interpreterFlags
+            cmd += compilerFlags
             if let moduleCachePath = moduleCachePath {
                 cmd += ["-module-cache-path", moduleCachePath]
             }
@@ -579,36 +579,50 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
             cmd += [manifestPath.pathString]
 
-            // Create and open a temporary file to write json to.
             try withTemporaryFile { file in
+                // Set path to compiled manifest executable.
+                cmd += ["-o", file.path.pathString]
+
+                try Process.popen(arguments: cmd)
+
+                // Compile the manifest.
+                let compilerResult = try Process.popen(arguments: cmd)
+                let compilerOutput = try (compilerResult.utf8Output() + compilerResult.utf8stderrOutput()).spm_chuzzle()
+                manifestParseResult.compilerOutput = compilerOutput
+
+                // Return now if there was an error.
+                if compilerResult.exitStatus != .terminated(code: 0) {
+                    return
+                }
+
                 // Pass the fd in arguments.
-                cmd += ["-fileno", "\(file.fileHandle.fileDescriptor)"]
-
-                // Prefer swiftinterface if both swiftmodule and swiftinterface files are present.
-                //
-                // This will avoid failures during incremental builds when the
-                // slate swiftmodule file is still present from the previous
-                // install. We should be able to remove this after some
-                // transition period.
-                var env = ProcessEnv.vars
+                cmd = [file.path.pathString, "-fileno", "1"]
+
               #if os(macOS)
-                env["SWIFT_FORCE_MODULE_LOADING"] = "prefer-parseable"
+                // If enabled, use sandbox-exec on macOS. This provides some safety against
+                // arbitrary code execution when parsing manifest files. We only allow
+                // the permissions which are absolutely necessary for manifest parsing.
+                if isManifestSandboxEnabled {
+                    let cacheDirectories = [
+                        cacheDir,
+                        moduleCachePath.map({ AbsolutePath($0) })
+                    ].compactMap({ $0 })
+                    let profile = sandboxProfile(toolsVersion: toolsVersion, cacheDirectories: cacheDirectories)
+                    cmd += ["sandbox-exec", "-p", profile]
+                }
               #endif
 
                 // Run the command.
-                let result = try Process.popen(arguments: cmd, environment: env)
-                let output = try (result.utf8Output() + result.utf8stderrOutput()).spm_chuzzle()
-                manifestParseResult.compilerOutput = output
+                let runResult = try Process.popen(arguments: cmd)
+                let runOutput = try (runResult.utf8Output() + runResult.utf8stderrOutput()).spm_chuzzle()
 
                 // Return now if there was an error.
-                if result.exitStatus != .terminated(code: 0) {
+                if runResult.exitStatus != .terminated(code: 0) {
+                    manifestParseResult.errorOutput = runOutput
                     return
                 }
 
-                guard let json = try localFileSystem.readFileContents(file.path).validDescription else {
-                    throw StringError("the manifest has invalid encoding")
-                }
-                manifestParseResult.parsedManifest = json
+                manifestParseResult.parsedManifest = runOutput
             }
         }
 
@@ -661,7 +675,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
     private var _sdkRoot: AbsolutePath? = nil
 
     /// Returns the interpreter flags for a manifest.
-    public func interpreterFlags(
+    public func compilerFlags(
         for toolsVersion: ToolsVersion
     ) -> [String] {
         var cmd = [String]()
@@ -705,26 +719,31 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 }
 
 /// Returns the sandbox profile to be used when parsing manifest on macOS.
-private func sandboxProfile(_ cacheDirs: [AbsolutePath] = []) -> String {
+private func sandboxProfile(toolsVersion: ToolsVersion, cacheDirectories: [AbsolutePath] = []) -> String {
     let stream = BufferedOutputByteStream()
     stream <<< "(version 1)" <<< "\n"
     // Deny everything by default.
     stream <<< "(deny default)" <<< "\n"
     // Import the system sandbox profile.
     stream <<< "(import \"system.sb\")" <<< "\n"
-    // Allow reading all files.
-    stream <<< "(allow file-read*)" <<< "\n"
-    // These are required by the Swift compiler.
-    stream <<< "(allow process*)" <<< "\n"
-    stream <<< "(allow sysctl*)" <<< "\n"
-    // Allow writing in temporary locations.
-    stream <<< "(allow file-write*" <<< "\n"
-    for directory in Platform.darwinCacheDirectories() {
-        stream <<< "    (regex #\"^\(directory.pathString)/org\\.llvm\\.clang.*\")" <<< "\n"
-    }
-    for cacheDir in cacheDirs {
-        stream <<< "    (subpath \"\(cacheDir.pathString)\")" <<< "\n"
+
+    // The following accesses are only needed when interpreting the manifest (versus running a compiled version).
+    if toolsVersion < .vNext {
+        // Allow reading all files.
+        stream <<< "(allow file-read*)" <<< "\n"
+        // These are required by the Swift compiler.
+        stream <<< "(allow process*)" <<< "\n"
+        stream <<< "(allow sysctl*)" <<< "\n"
+        // Allow writing in temporary locations.
+        stream <<< "(allow file-write*" <<< "\n"
+        for directory in Platform.darwinCacheDirectories() {
+            stream <<< "    (regex #\"^\(directory.pathString)/org\\.llvm\\.clang.*\")" <<< "\n"
+        }
+        for directory in cacheDirectories {
+            stream <<< "    (subpath \"\(directory.pathString)\")" <<< "\n"
+        }
     }
+
     stream <<< ")" <<< "\n"
     return stream.bytes.description
 }

Sources/PackageModel/ToolsVersion.swift

@@ -21,6 +21,7 @@ public struct ToolsVersion: CustomStringConvertible, Comparable, Hashable, Codab
     public static let v4_2 = ToolsVersion(version: "4.2.0")
     public static let v5 = ToolsVersion(version: "5.0.0")
     public static let v5_2 = ToolsVersion(version: "5.2.0")
+    public static let vNext = ToolsVersion(version: "999.0.0")
 
     /// The current tools version in use.
     public static let currentToolsVersion = ToolsVersion(string:

Sources/Workspace/Workspace.swift

@@ -1137,15 +1137,15 @@ extension Workspace {
     }
 
     /// Returns manifest interpreter flags for a package.
-    public func interpreterFlags(for packagePath: AbsolutePath) -> [String] {
+    public func compilerFlags(for packagePath: AbsolutePath) -> [String] {
         // We ignore all failures here and return empty array.
         guard let manifestLoader = self.manifestLoader as? ManifestLoader,
               let toolsVersion = try? toolsVersionLoader.load(at: packagePath, fileSystem: fileSystem),
               currentToolsVersion >= toolsVersion,
               toolsVersion >= ToolsVersion.minimumRequired else {
             return []
         }
-        return manifestLoader.interpreterFlags(for: toolsVersion)
+        return manifestLoader.compilerFlags(for: toolsVersion)
     }
 
     /// Load the manifests for the current dependency tree.

Sources/Xcodeproj/pbxproj.swift

@@ -77,12 +77,12 @@ public func xcodeProject(
         let compilePhase = pdTarget.addSourcesBuildPhase()
         compilePhase.addBuildFile(fileRef: manifestFileRef)
 
-        var interpreterFlags = manifestLoader.interpreterFlags(for: package.manifest.toolsVersion)
-        if !interpreterFlags.isEmpty {
+        var compilerFlags = manifestLoader.compilerFlags(for: package.manifest.toolsVersion)
+        if !compilerFlags.isEmpty {
             // Patch the interpreter flags to use Xcode supported toolchain macro instead of the resolved path.
-            interpreterFlags[3] = "$(TOOLCHAIN_DIR)/usr/lib/swift/pm/\(package.manifest.toolsVersion.runtimeSubpath.pathString)"
+            compilerFlags[3] = "$(TOOLCHAIN_DIR)/usr/lib/swift/pm/\(package.manifest.toolsVersion.runtimeSubpath.pathString)"
         }
-        pdTarget.buildSettings.common.OTHER_SWIFT_FLAGS += interpreterFlags
+        pdTarget.buildSettings.common.OTHER_SWIFT_FLAGS += compilerFlags
         pdTarget.buildSettings.common.SWIFT_VERSION = package.manifest.toolsVersion.swiftLanguageVersion.xcodeBuildSettingValue
         pdTarget.buildSettings.common.LD = "/usr/bin/true"
     }

Tests/WorkspaceTests/WorkspaceTests.swift

@@ -146,7 +146,7 @@ final class WorkspaceTests: XCTestCase {
                         """
                 }
 
-                XCTAssertMatch((ws.interpreterFlags(for: foo)), [.equal("-swift-version"), .equal("4")])
+                XCTAssertMatch((ws.compilerFlags(for: foo)), [.equal("-swift-version"), .equal("4")])
             }
 
             do {
@@ -161,7 +161,7 @@ final class WorkspaceTests: XCTestCase {
                         """
                 }
 
-                XCTAssertEqual(ws.interpreterFlags(for: foo), [])
+                XCTAssertEqual(ws.compilerFlags(for: foo), [])
             }
         }
     }

Utilities/bootstrap

@@ -441,6 +441,14 @@ def build_swiftpm_with_cmake(args):
 
     build_with_cmake(args, cmake_flags, args.project_root, args.bootstrap_dir)
 
+    if platform.system() == 'Darwin':
+        for runtime in ["4", "4_2"]:
+            runtime_relative_path = "pm/%s/libPackageDescription.dylib" % runtime
+            runtime_path = os.path.join(args.bootstrap_dir, runtime_relative_path)
+            set_install_name_cmd = ["install_name_tool", "-id", "@rpath/%s" % runtime_relative_path, runtime_path]
+            note(' '.join(set_install_name_cmd))
+            subprocess.check_call(set_install_name_cmd, stderr=subprocess.PIPE)
+
     if args.llbuild_link_framework:
         swift_build = os.path.join(args.bootstrap_dir, "bin/swift-build")
         add_rpath_cmd = ["install_name_tool", "-add_rpath", args.llbuild_build_dir, swift_build]