Make the communication between the plugin host and the plugin slightly more sophisticated, in preparation for being able to send messages back and forth.

This approach uses `stdout` for JSON information from the plugin and `stderr` for freeform text from the plugin rather than a specialized zero byte denoting the start of the output JSON.

We also close `stdin` to avoid having the plugin block forever in case it tries to read from `stdin` (console input to the plugin isn't available).

No change in functionality except that trying to read from `stdin` from the plugin no longer blocks it.

Author: abertelrud

Sources/PackagePlugin/Plugin.swift

@@ -15,6 +15,9 @@
 */
 
 @_implementationOnly import Foundation
+#if os(Windows)
+@_implementationOnly import ucrt    // for stdio functions
+#endif
 
 // The way in which SwiftPM communicates with the plugin is an implementation
 // detail, but the way it currently works is that the plugin is compiled (in
@@ -25,11 +28,10 @@
 // likely change so that it is instead passed on `stdin` of the process that
 // runs the plugin, since that avoids any command line length limitations.
 //
-// Any generated commands and diagnostics are emitted on `stdout` after a zero
-// byte; this allows regular output, such as print statements for debugging,
-// to be emitted to SwiftPM verbatim. SwiftPM tries to interpret any stdout
-// contents after the last zero byte as a JSON encoded output struct in UTF-8
-// encoding; any failure to decode it is considered a protocol failure.
+// An output structure containing any generated commands and diagnostics is
+// passed back to SwiftPM on `stdout`. All freeform output from the plugin
+// is redirected to `stderr`, which SwiftPM shows to the user without inter-
+// preting it in any way.
 //
 // The exit code of the compiled plugin determines success or failure (though
 // failure to decode the output is also considered a failure to run the ex-
@@ -38,15 +40,34 @@
 extension Plugin {
 
     public static func main(_ arguments: [String]) throws {
+        
+        // Use the initial `stdout` for returning JSON, and redirect `stdout`
+        // to `stderr` for capturing freeform text.
+        let jsonOut = fdopen(dup(fileno(stdout)), "w")
+        dup2(fileno(stderr), fileno(stdout))
+        
+        // Close `stdin` to avoid blocking if the plugin tries to read input.
+        close(fileno(stdin))
+
+        // Private function for reporting internal errors and halting execution.
+        func internalError(_ message: String) -> Never {
+            Diagnostics.error("Internal Error: \(message)")
+            fputs("Internal Error: \(message)", stderr)
+            exit(1)
+        }
+        
         // Look for the input JSON as the last argument of the invocation.
         guard let inputData = ProcessInfo.processInfo.arguments.last?.data(using: .utf8) else {
-            fputs("Expected last argument to contain JSON input data in UTF-8 encoding, but didn't find it.", stderr)
-            Diagnostics.error("Expected last argument to contain JSON input data in UTF-8 encoding, but didn't find it.")
-            exit(1)
+            internalError("Expected last argument to contain JSON input data in UTF-8 encoding, but didn't find it.")
         }
-
+        
         // Deserialize the input JSON.
-        let input = try PluginInput(from: inputData)
+        let input: PluginInput
+        do {
+            input = try PluginInput(from: inputData)
+        } catch {
+            internalError("Couldn’t decode input JSON: \(error).")
+        }
 
         // Construct a PluginContext from the deserialized input.
         let context = PluginContext(
@@ -57,12 +78,13 @@ extension Plugin {
 
         // Instantiate the plugin. For now there are no parameters, but this is
         // where we would set them up, most likely as properties of the plugin
-        // instance (in a manner similar to SwiftArgumentParser).
+        // instance (in a manner similar to SwiftArgumentParser). This would
+        // use property wrappers to mark up properties in the plugin.
         let plugin = self.init()
 
         // Invoke the appropriate protocol method, based on the plugin action
         // that SwiftPM specified.
-        let commands: [Command]
+        let generatedCommands: [Command]
         switch input.pluginAction {
 
         case .createBuildToolCommands(let target):
@@ -73,7 +95,7 @@ extension Plugin {
             }
 
             // Ask the plugin to create build commands for the input target.
-            commands = try plugin.createBuildCommands(context: context, target: target)
+            generatedCommands = try plugin.createBuildCommands(context: context, target: target)
 
         case .performCommand(let targets, let arguments):
             // Check that the plugin implements the appropriate protocol for its
@@ -87,15 +109,21 @@ extension Plugin {
 
             // For command plugin there are currently no return commands (any
             // commands invoked by the plugin are invoked directly).
-            commands = []
+            generatedCommands = []
         }
 
-        // Construct the output structure to send to SwiftPM.
-        let output = try PluginOutput(commands: commands, diagnostics: Diagnostics.emittedDiagnostics)
+        // Construct the output structure to send back to SwiftPM.
+        let output: PluginOutput
+        do {
+            output = try PluginOutput(commands: generatedCommands, diagnostics: Diagnostics.emittedDiagnostics)
+        } catch {
+            internalError("Couldn’t encode output JSON: \(error).")
+        }
 
         // On stdout, write a zero byte followed by the JSON data — this is what libSwiftPM expects to see. Anything before the last zero byte is treated as freeform output from the plugin (such as debug output from `print` statements). Since `FileHandle.write()` doesn't obey buffering we first have to flush any existing output.
-        fputc(0, stdout)
-        fwrite([UInt8](output.outputData), 1, output.outputData.count, stdout)
+        if fwrite([UInt8](output.outputData), 1, output.outputData.count, jsonOut) != output.outputData.count {
+            internalError("Couldn’t write output JSON: \(strerror(errno).map{ String(cString: $0) } ?? String(describing: errno)).")
+        }
     }
 
     public static func main() throws {

Sources/Workspace/DefaultPluginScriptRunner.swift

@@ -182,54 +182,85 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
     }
 
     fileprivate func invoke(compiledExec: AbsolutePath, pluginArguments: [String], toolsVersion: ToolsVersion, writableDirectories: [AbsolutePath], input: Data) throws -> (outputJSON: Data, stdoutText: Data) {
-        // Construct the command line.
-        
-        // FIXME: Need to pass down the arguments and not ignore them.
-        
-        // FIXME: It would be more robust to pass it as `stdin` data, but we need TSC support for that.  When this is
-        // changed, PackagePlugin will need to change as well (but no plugins need to change).
-        var command = [compiledExec.pathString]
+        // Construct the command line.  We just pass along any arguments intended for the plugin.
+        var command = [compiledExec.pathString] + pluginArguments
         command += [String(decoding: input, as: UTF8.self)]
 
         // If enabled, run command in a sandbox.
         // This provides some safety against arbitrary code execution when invoking the plugin.
         // We only allow the permissions which are absolutely necessary.
         if self.enableSandbox {
-            command = Sandbox.apply(command: command, writableDirectories: writableDirectories)
+            command = Sandbox.apply(command: command, writableDirectories: writableDirectories + [self.cacheDir])
+        }
+
+        // Create and configure a Process.
+        let process = Process()
+        process.launchPath = command.first!
+        process.arguments = Array(command.dropFirst())
+        process.environment = ProcessInfo.processInfo.environment
+        process.currentDirectoryURL = self.cacheDir.asURL
+            
+        // Create a dispatch group for waiting until the process has terminated and the data has been read.
+        let waiters = DispatchGroup()
+        // Set up for capturing stdout data.
+        let stdoutPipe = Pipe()
+        var stdoutData = Data()
+        waiters.enter()
+        stdoutPipe.fileHandleForReading.readabilityHandler = { (fileHandle: FileHandle) -> Void in
+            let newData = fileHandle.availableData
+            if newData.isEmpty {
+                fileHandle.readabilityHandler = nil
+                waiters.leave()
+            }
+            else {
+                stdoutData.append(contentsOf: newData)
+            }
+        }
+        process.standardOutput = stdoutPipe
+        
+        // Set up for capturing stderr data.
+        waiters.enter()
+        let stderrPipe = Pipe()
+        var stderrData = Data()
+        stderrPipe.fileHandleForReading.readabilityHandler = { (fileHandle: FileHandle) -> Void in
+            let newData = fileHandle.availableData
+            if newData.isEmpty {
+                fileHandle.readabilityHandler = nil
+                waiters.leave()
+            }
+            else {
+                stderrData.append(contentsOf: newData)
+            }
+        }
+        process.standardError = stderrPipe
+        
+        // Set up a termination handler.
+        process.terminationHandler = { _ in
+            waiters.leave()
         }
 
-        // Invoke the plugin script as a subprocess.
-        let result: ProcessResult
+        // Start the process.
+        waiters.enter()
         do {
-            result = try Process.popen(arguments: command)
+            try process.run()
         } catch {
             throw DefaultPluginScriptRunnerError.subprocessDidNotStart("\(error)", command: command)
         }
 
-        // Collect the output. The `PackagePlugin` runtime library writes the output as a zero byte followed by
-        // the JSON-serialized PluginEvaluationResult. Since this appears after any free-form output from the
-        // script, it can be safely split out while maintaining the ability to see debug output without resorting
-        // to side-channel communication that might be not be very cross-platform (e.g. pipes, file handles, etc).
-        // We end up with an optional Data for the JSON, and two Datas for stdout and stderr respectively.
-        var stdoutPieces = (try? result.output.get().split(separator: 0, omittingEmptySubsequences: false)) ?? []
-        let jsonData = (stdoutPieces.count > 1) ? Data(stdoutPieces.removeLast()) : nil
-        let stdoutData = Data(stdoutPieces.joined())
-        let stderrData = (try? Data(result.stderrOutput.get())) ?? Data()
+        // Wait for the process to terminate and the readers to finish collecting all output.
+        waiters.wait()
 
-        // Throw an error if we the subprocess ended badly.
-        if result.exitStatus != .terminated(code: 0) {
-            let output = String(decoding: stdoutData + stderrData, as: UTF8.self)
-            throw DefaultPluginScriptRunnerError.subprocessFailed("\(result.exitStatus)", command: command, output: output)
-        }
+        
+        // Now `stdoutData` contains a JSON-encoded output structure, and `stderrData` contains any free text output from the plugin process.
+        let stderrText = String(decoding: stderrData, as: UTF8.self)
 
-        // Throw an error if we didn't get the JSON data.
-        guard let json = jsonData else {
-            let output = String(decoding: stdoutData + stderrData, as: UTF8.self)
-            throw DefaultPluginScriptRunnerError.missingPluginJSON("didn't receive JSON output data", command: command, output: output)
+        // Throw an error if we the subprocess ended badly.
+        if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+            throw DefaultPluginScriptRunnerError.subprocessFailed("\(process.terminationStatus)", command: command, output: stderrText)
         }
 
         // Otherwise return the JSON data and any output text.
-        return (outputJSON: json, stdoutText: stdoutData + stderrData)
+        return (outputJSON: stdoutData, stdoutText: stderrData)
     }
 }