Use prebuilts for macro tests. Add debug URL for swift-syntax. (#8229)

Re-enable use of prebuilts for tests that depend on macro targets. These
are guaranteed to be built for host as the macros are.

Add a hidden option to override the prebuilts download URL for
swift-syntax for testing. Also added support for this to be a file URL.

Add support for using the ssh GitHub URL for swift-syntax, allowing
multiple PackageReferences for prebuilts. Also elevated the
PackageIdentity to the root structures that need it.

Fixed the build prebuilts script to generate into a versioned directory
so we can support generating multiple versions at the same time.

Author: dschaefer2

Sources/CoreCommands/Options.swift

@@ -188,10 +188,19 @@ public struct CachingOptions: ParsableArguments {
     }
 
     /// Whether to use macro prebuilts or not
-    @Flag(name: .customLong("experimental-prebuilts"),
-          inversion: .prefixedEnableDisable,
-          help: "Whether to use prebuilt swift-syntax libraries for macros")
+    @Flag(
+        name: .customLong("experimental-prebuilts"),
+        inversion: .prefixedEnableDisable,
+        help: "Whether to use prebuilt swift-syntax libraries for macros"
+    )
     public var usePrebuilts: Bool = false
+
+    /// Hidden option to override the prebuilts download location for testing
+    @Option(
+        name: .customLong("experimental-prebuilts-download-url"),
+        help: .hidden
+    )
+    public var prebuiltsDownloadURL: String?
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -472,7 +472,8 @@ public final class SwiftCommandState {
                     .init(url: $0, supportsAvailability: true)
                 },
                 manifestImportRestrictions: .none,
-                usePrebuilts: options.caching.usePrebuilts
+                usePrebuilts: options.caching.usePrebuilts,
+                prebuiltsDownloadURL: options.caching.prebuiltsDownloadURL
             ),
             cancellator: self.cancellator,
             initializationWarningHandler: { self.observabilityScope.emit(warning: $0) },

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -671,9 +671,22 @@ private func createResolvedPackages(
                     }
                 }
 
-                if moduleBuilder.module.type == .macro, let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
-                    // using a prebuilt instead.
-                    continue
+                if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
+                    // See if we're using a prebuilt instead
+                    if moduleBuilder.module.type == .macro {
+                        continue
+                    } else if moduleBuilder.module.type == .test {
+                        // use prebuilt if this is a test that depends a macro target
+                        // these are guaranteed built for host
+                        if moduleBuilder.module.dependencies.contains(where: { dep in
+                            guard let module = dep.module else {
+                                return false
+                            }
+                            return module.type == .macro
+                        }) {
+                            continue
+                        }
+                    }
                 }
 
                 // Find the product in this package's dependency products.

Sources/PackageLoading/PackageBuilder.swift

@@ -270,8 +270,8 @@ public struct BinaryArtifact {
 
 /// A structure representing a prebuilt library to be used instead of a source dependency
 public struct PrebuiltLibrary {
-    /// The package reference.
-    public let packageRef: PackageReference
+    /// The package identity.
+    public let identity: PackageIdentity
 
     /// The name of the binary target the artifact corresponds to.
     public let libraryName: String
@@ -285,8 +285,8 @@ public struct PrebuiltLibrary {
     /// The C modules that need their includes directory added to the include path
     public let cModules: [String]
 
-    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
-        self.packageRef = packageRef
+    public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.identity = identity
         self.libraryName = libraryName
         self.path = path
         self.products = products
@@ -1242,9 +1242,9 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
-        // Add in flags for prebuilts if the target is a macro.
+        // Add in flags for prebuilts if the target is a macro or a macro test.
         // Currently we only support prebuilts for macros.
-        if target.type == .macro {
+        if target.type == .macro || target.isMacroTest(in: manifest) {
             let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
                 guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
                       let package = package,
@@ -1856,4 +1856,26 @@ extension TargetDescription {
     fileprivate var usesUnsafeFlags: Bool {
         settings.filter(\.kind.isUnsafeFlags).isEmpty == false
     }
+
+    fileprivate func isMacroTest(in manifest: Manifest) -> Bool {
+        guard self.type == .test else { return false }
+
+        return self.dependencies.contains(where: {
+            let name: String
+            switch $0 {
+            case .byName(name: let n, condition: _):
+                name = n
+            case .target(name: let n, condition: _):
+                name = n
+            default:
+                return false
+            }
+
+            guard let target = manifest.targetMap[name] else {
+                return false
+            }
+
+            return target.type == .macro
+        })
+    }
 }

Sources/Workspace/ManagedPrebuilt.swift

@@ -16,8 +16,8 @@ import PackageModel
 extension Workspace {
     /// A downloaded prebuilt managed by the workspace.
     public struct ManagedPrebuilt {
-        /// The package reference.
-        public let packageRef: PackageReference
+        /// The package identity
+        public let identity: PackageIdentity
 
         /// The name of the binary target the artifact corresponds to.
         public let libraryName: String
@@ -35,7 +35,7 @@ extension Workspace {
 
 extension Workspace.ManagedPrebuilt: CustomStringConvertible {
     public var description: String {
-        return "<ManagedArtifact: \(self.packageRef.identity).\(self.libraryName)>"
+        return "<ManagedArtifact: \(self.identity).\(self.libraryName)>"
     }
 }
 
@@ -56,7 +56,7 @@ extension Workspace {
         }
 
         init(_ artifacts: [ManagedPrebuilt]) throws {
-            let artifactsByPackagePath = Dictionary(grouping: artifacts, by: { $0.packageRef.identity })
+            let artifactsByPackagePath = Dictionary(grouping: artifacts, by: { $0.identity })
             self.artifactMap = try artifactsByPackagePath.mapValues{ artifacts in
                 try Dictionary(artifacts.map { ($0.libraryName, $0) }, uniquingKeysWith: { _, _ in
                     // should be unique
@@ -70,7 +70,7 @@ extension Workspace {
         }
 
         public func add(_ artifact: ManagedPrebuilt) {
-            self.artifactMap[artifact.packageRef.identity, default: [:]][artifact.libraryName] = artifact
+            self.artifactMap[artifact.identity, default: [:]][artifact.libraryName] = artifact
         }
 
         public func remove(packageIdentity: PackageIdentity, targetName: String) {

Sources/Workspace/Workspace+Configuration.swift

@@ -792,6 +792,9 @@ public struct WorkspaceConfiguration {
     /// Whether or not to use prebuilt swift-syntax for macros
     public var usePrebuilts: Bool
 
+    /// String URL to allow override of the prebuilts download location
+    public var prebuiltsDownloadURL: String?
+
     public init(
         skipDependenciesUpdates: Bool,
         prefetchBasedOnResolvedFile: Bool,
@@ -805,7 +808,8 @@ public struct WorkspaceConfiguration {
         sourceControlToRegistryDependencyTransformation: SourceControlToRegistryDependencyTransformation,
         defaultRegistry: Registry?,
         manifestImportRestrictions: (startingToolsVersion: ToolsVersion, allowedImports: [String])?,
-        usePrebuilts: Bool
+        usePrebuilts: Bool,
+        prebuiltsDownloadURL: String?
     ) {
         self.skipDependenciesUpdates = skipDependenciesUpdates
         self.prefetchBasedOnResolvedFile = prefetchBasedOnResolvedFile
@@ -820,6 +824,7 @@ public struct WorkspaceConfiguration {
         self.defaultRegistry = defaultRegistry
         self.manifestImportRestrictions = manifestImportRestrictions
         self.usePrebuilts = usePrebuilts
+        self.prebuiltsDownloadURL = prebuiltsDownloadURL
     }
 
     /// Default instance of WorkspaceConfiguration
@@ -837,7 +842,8 @@ public struct WorkspaceConfiguration {
             sourceControlToRegistryDependencyTransformation: .disabled,
             defaultRegistry: .none,
             manifestImportRestrictions: .none,
-            usePrebuilts: false
+            usePrebuilts: false,
+            prebuiltsDownloadURL: nil
         )
     }