Allow passing extra arguments to plugin compilation. (#5966)

For most case, compiling against the host toolchain does not need extra
arguments, but in some complicated setups, the host toolchain might need
extra arguments for search paths and such. In the case of the manifest,
the option of using `-Xmanifest` arguments to customize the manifest
compilation has existed for a while, but in the case of plugins, the
same option was not available and made consuming packages with plugins
very complicated.

The changes in this commit add `-Xbuild-tools-swiftc` to affect both manifest
and plugin compilation, allowing complicated setups to provide their values
and compile and use plugins. `-Xmanifest` is kept for backwards
compatibility, but it will only affect manifest compilation, as before,
and it will print a warning if used.

Includes modifications in one of the plugin tests to show that the
`-Xbuild-tools-swiftc` parameters are passed to the plugin compilation (I
could not find a similar test for `-Xmanifest`).

Author: drodriguez

CHANGELOG.md

@@ -3,6 +3,10 @@ Note: This is in reverse chronological order, so newer entries are added to the
 Swift Next
 -----------
 
+* [#5966]
+
+  Plugin compilation can be influenced by using `-Xbuild-tools-swiftc` arguments in the SwiftPM command line. This is similar to the existing mechanism for influencing the manifest compilation using `-Xmanifest` arguments. Manifest compilation will also be influenced by `-Xbuild-tools-swiftc`, but only if no other `-Xmanifest` arguments are provided. Using `-Xmanifest` will show a deprecation message. `-Xmanifest` will be removed in the future.
+
 * [#5728]
 
   In packages that specify resources using a future tools version, the generated resource bundle accessor will import `Foundation.Bundle` for its own implementation only. _Clients_ of such packages therefore no longer silently import `Foundation`, preventing inadvertent use of Foundation extensions to standard library APIs, which helps to avoid unexpected code size increases.
@@ -318,6 +322,7 @@ Swift 3.0
 [#5874]: https://github.com/apple/swift-package-manager/pull/5874
 [#5949]: https://github.com/apple/swift-package-manager/pull/5949
 [#5892]: https://github.com/apple/swift-package-manager/pull/5892
+[#5966]: https://github.com/apple/swift-package-manager/pull/5966
 [#6060]: https://github.com/apple/swift-package-manager/pull/6060
 [#6067]: https://github.com/apple/swift-package-manager/pull/6067
 [#6114]: https://github.com/apple/swift-package-manager/pull/6114

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Plugins/MySourceGenBuildToolPlugin/plugin.swift

@@ -2,7 +2,12 @@ import PackagePlugin
 
 @main
 struct MyPlugin: BuildToolPlugin {
-    
+    #if USE_CREATE
+    let verb = "Creating"
+    #else
+    let verb = "Generating"
+    #endif
+
     func createBuildCommands(context: PluginContext, target: Target) throws -> [Command] {
         print("Hello from the Build Tool Plugin!")
         guard let target = target as? SourceModuleTarget else { return [] }
@@ -12,7 +17,7 @@ struct MyPlugin: BuildToolPlugin {
             let outputPath = context.pluginWorkDirectory.appending(outputName)
             return .buildCommand(
                 displayName:
-                    "Generating \(outputName) from \($0.lastComponent)",
+                    "\(verb) \(outputName) from \($0.lastComponent)",
                 executable:
                     try context.tool(named: "MySourceGenBuildTool").path,
                 arguments: [

Sources/CoreCommands/Options.swift

@@ -336,15 +336,35 @@ public struct BuildOptions: ParsableArguments {
     )
     public var xcbuildFlags: [String] = []
 
+    @Option(
+        name: .customLong("Xbuild-tools-swiftc", withSingleDash: true),
+        parsing: .unconditionalSingleValue,
+        help: ArgumentHelp(
+            "Pass flag to Swift compiler invocations for build-time executables (manifest and plugins)",
+            visibility: .hidden
+        )
+    )
+    public var _buildToolsSwiftCFlags: [String] = []
+
     @Option(
         name: .customLong("Xmanifest", withSingleDash: true),
         parsing: .unconditionalSingleValue,
         help: ArgumentHelp(
-            "Pass flag to the manifest build invocation",
+            "Pass flag to the manifest build invocation. Deprecated: use '-Xbuild-tools-swiftc' instead",
             visibility: .hidden
         )
     )
-    var manifestFlags: [String] = []
+    public var _deprecated_manifestFlags: [String] = []
+
+    var manifestFlags: [String] {
+        self._deprecated_manifestFlags.isEmpty ?
+            self._buildToolsSwiftCFlags :
+            self._deprecated_manifestFlags
+    }
+
+    var pluginSwiftCFlags: [String] {
+        self._buildToolsSwiftCFlags
+    }
 
     public var buildFlags: BuildFlags {
         BuildFlags(

Sources/CoreCommands/SwiftTool.swift

@@ -423,6 +423,10 @@ public final class SwiftTool {
         if let _ = options.security.netrcFilePath, options.security.netrc == false {
             observabilityScope.emit(.mutuallyExclusiveArgumentsError(arguments: ["--disable-netrc", "--netrc-file"]))
         }
+
+        if !options.build._deprecated_manifestFlags.isEmpty {
+            observabilityScope.emit(warning: "'-Xmanifest' option is deprecated; use '-Xbuild-tools-swiftc' instead")
+        }
     }
 
     func waitForObservabilityEvents(timeout: DispatchTime) {
@@ -625,6 +629,7 @@ public final class SwiftTool {
             fileSystem: self.fileSystem,
             cacheDir: cacheDir,
             toolchain: self.getHostToolchain(),
+            extraPluginSwiftCFlags: self.options.build.pluginSwiftCFlags,
             enableSandbox: !self.shouldDisableSandbox,
             verboseOutput: self.logLevel <= .info
         )

Sources/Workspace/DefaultPluginScriptRunner.swift

@@ -24,16 +24,25 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
     private let fileSystem: FileSystem
     private let cacheDir: AbsolutePath
     private let toolchain: UserToolchain
+    private let extraPluginSwiftCFlags: [String]
     private let enableSandbox: Bool
     private let cancellator: Cancellator
     private let verboseOutput: Bool
 
     private let sdkRootCache = ThreadSafeBox<AbsolutePath>()
 
-    public init(fileSystem: FileSystem, cacheDir: AbsolutePath, toolchain: UserToolchain, enableSandbox: Bool = true, verboseOutput: Bool = false) {
+    public init(
+        fileSystem: FileSystem,
+        cacheDir: AbsolutePath,
+        toolchain: UserToolchain,
+        extraPluginSwiftCFlags: [String] = [],
+        enableSandbox: Bool = true,
+        verboseOutput: Bool = false
+    ) {
         self.fileSystem = fileSystem
         self.cacheDir = cacheDir
         self.toolchain = toolchain
+        self.extraPluginSwiftCFlags = extraPluginSwiftCFlags
         self.enableSandbox = enableSandbox
         self.cancellator = Cancellator(observabilityScope: .none)
         self.verboseOutput = verboseOutput
@@ -209,6 +218,9 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
         // Finally add the output path of the compiled executable.
         commandLine += ["-o", execFilePath.pathString]
 
+        // Add any extra flags passed for the host in the command line
+        commandLine += self.extraPluginSwiftCFlags
+
         if (verboseOutput) {
             commandLine.append("-v")
         }

Tests/FunctionalTests/PluginTests.swift

@@ -1009,4 +1009,25 @@ class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
     }
+
+    func testPluginCanBeAffectedByXBuildToolsParameters() throws {
+        // Only run the test if the environment in which we're running actually supports Swift concurrency (which the
+        // plugin APIs require).
+        try XCTSkipIf(
+            !UserToolchain.default.supportsSwiftConcurrency(),
+            "skipping because test environment doesn't support concurrency"
+        )
+
+        try fixture(name: "Miscellaneous/Plugins") { fixturePath in
+            let (stdout, _) = try executeSwiftBuild(
+                fixturePath.appending(component: "MySourceGenPlugin"),
+                configuration: .Debug,
+                extraArgs: ["--product", "MyLocalTool", "-Xbuild-tools-swiftc", "-DUSE_CREATING"]
+            )
+            XCTAssert(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Creating foo.swift from foo.dat"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
+        }
+    }
 }