Add support for extension products, allowing use of extensions across packages. (#3298)

This completes the PackageDescription interface changes proposed in the extensible build tools pitch.  Note that actual use of this is still guarded by a feature flag, since the proposal has not been accepted yet and there may be changes needed based on review.

Author: abertelrud

Fixtures/Miscellaneous/Extensions/MySourceGenClient/Package.swift

@@ -0,0 +1,25 @@
+// swift-tools-version: 999.0
+import PackageDescription
+
+let package = Package(
+    name: "MySourceGenClient",
+    dependencies: [
+        .package(path: "../MySourceGenExtension")
+    ],
+    targets: [
+        // A tool that uses an extension.
+        .executableTarget(
+            name: "MyTool",
+            dependencies: [
+                .product(name: "MySourceGenExt", package: "MySourceGenExtension")
+            ]
+        ),
+        // A unit that uses the extension.
+        .testTarget(
+            name: "MyTests",
+            dependencies: [
+                .product(name: "MySourceGenExt", package: "MySourceGenExtension")
+            ]
+        )
+    ]
+)

Fixtures/Miscellaneous/Extensions/MySourceGenClient/Sources/MyTool/Foo.dat

@@ -0,0 +1 @@
+Hello Extension Product!

Fixtures/Miscellaneous/Extensions/MySourceGenClient/Sources/MyTool/main.swift

@@ -0,0 +1 @@
+print("Generated string: '\(generatedString)'")

Fixtures/Miscellaneous/Extensions/MySourceGenClient/Tests/MyTests/MyTests.swift

@@ -0,0 +1,47 @@
+import XCTest
+import class Foundation.Bundle
+
+final class SwiftyProtobufTests: XCTestCase {
+    func testExample() throws {
+        // This is an example of a functional test case.
+        // Use XCTAssert and related functions to verify your tests produce the correct
+        // results.
+
+        // Some of the APIs that we use below are available in macOS 10.13 and above.
+        guard #available(macOS 10.13, *) else {
+            return
+        }
+
+        // Mac Catalyst won't have `Process`, but it is supported for executables.
+        #if !targetEnvironment(macCatalyst)
+
+        let fooBinary = productsDirectory.appendingPathComponent("MySourceGenTool")
+
+        let process = Process()
+        process.executableURL = fooBinary
+
+        let pipe = Pipe()
+        process.standardOutput = pipe
+
+        try process.run()
+        process.waitUntilExit()
+
+        let data = pipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(data: data, encoding: .utf8)
+
+        XCTAssertEqual(output, "Hello, world!\n")
+        #endif
+    }
+
+    /// Returns path to the built products directory.
+    var productsDirectory: URL {
+      #if os(macOS)
+        for bundle in Bundle.allBundles where bundle.bundlePath.hasSuffix(".xctest") {
+            return bundle.bundleURL.deletingLastPathComponent()
+        }
+        fatalError("couldn't find the products directory")
+      #else
+        return Bundle.main.bundleURL
+      #endif
+    }
+}

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Package.swift

@@ -5,18 +5,21 @@ let package = Package(
     name: "MySourceGenExtension",
     products: [
         // The product that vends MySourceGenExt to client packages.
-        // .extension(
-        //     name: "MySourceGenExt",
-        //     target: "MySourceGenExt"
-        // )
+        .extension(
+            name: "MySourceGenExt",
+            targets: ["MySourceGenExt"]
+        ),
+        .executable(
+            name: "MySourceGenTool",
+            targets: ["MySourceGenTool"]
+        )
     ],
     targets: [
         // A local tool that uses an extension.
         .executableTarget(
             name: "MyLocalTool",
             dependencies: [
                 "MySourceGenExt",
-                "MySourceGenTool"
             ]
         ),
         // The target that implements the extension and generates commands to invoke MySourceGenTool.

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/Foo.dat

@@ -1 +1 @@
-Hello Extension!
+Hello Extension Target!

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/main.swift

@@ -1 +1 @@
-print("Exec: \(data)")
+print("Generated string: '\(generatedString)'")

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenExt/extension.swift

@@ -6,7 +6,7 @@ for inputPath in targetBuildContext.otherFiles {
     let outputPath = targetBuildContext.outputDir.appending(outputName)
     commandConstructor.createCommand(
         displayName:
-            "MySourceGenTooling \(inputPath)",
+            "Generating \(outputName) from \(inputPath.filename)",
         executable:
             try targetBuildContext.lookupTool(named: "MySourceGenTool"),
         arguments: [

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenTool/main.swift

@@ -11,6 +11,6 @@ let outputFile = ProcessInfo.processInfo.arguments[2]
 
 let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
 let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
-let outputString = "public var data = \(dataAsHex.quotedForSourceCode)\n"
+let outputString = "public var generatedString = \(dataAsHex.quotedForSourceCode)\n"
 let outputData = outputString.data(using: .utf8)
 FileManager.default.createFile(atPath: outputFile, contents: outputData)

Sources/Build/BuildPlan.swift

@@ -1138,6 +1138,8 @@ public final class ProductBuildDescription {
                 }
             }
             args += ["-emit-executable"]
+        case .extension:
+            throw InternalError("unexpectedly asked to generate linker arguments for an extension product")
         }
 
         // Set rpath such that dynamic libraries are looked up
@@ -1153,7 +1155,7 @@ public final class ProductBuildDescription {
         // Embed the swift stdlib library path inside tests and executables on Darwin.
         if containsSwiftTargets {
           switch product.type {
-          case .library: break
+          case .library, .extension: break
           case .test, .executable:
               if buildParameters.triple.isDarwin() {
                   let stdlib = buildParameters.toolchain.macosSwiftStdlib
@@ -1424,8 +1426,8 @@ public class BuildPlan {
 
         var productMap: [ResolvedProduct: ProductBuildDescription] = [:]
         // Create product description for each product we have in the package graph except
-        // for automatic libraries because they don't produce any output.
-        for product in graph.allProducts where product.type != .library(.automatic) {
+        // for automatic libraries and extension because they don't produce any output.
+        for product in graph.allProducts where product.type != .library(.automatic) && product.type != .extension {
             productMap[product] = ProductBuildDescription(
                 product: product, buildParameters: buildParameters,
                 fs: fileSystem,
@@ -1588,10 +1590,10 @@ public class BuildPlan {
                 return target.dependencies.filter { $0.satisfies(self.buildEnvironment) }
 
             // For a product dependency, we only include its content only if we
-            // need to statically link it.
+            // need to statically link it or if it's an extension.
             case .product(let product, _):
                 switch product.type {
-                case .library(.automatic), .library(.static):
+                case .library(.automatic), .library(.static), .extension:
                     return product.targets.map { .target($0, conditions: []) }
                 case .library(.dynamic), .test, .executable:
                     return []

Sources/Build/ManifestBuilder.swift

@@ -554,8 +554,8 @@ extension LLBuildManifestBuilder {
                     // Establish a dependency on binary of the product.
                     inputs.append(file: planProduct.binary)
 
-                // For automatic and static libraries, add their targets as static input.
-                case .library(.automatic), .library(.static):
+                // For automatic and static libraries, and extensions, add their targets as static input.
+                case .library(.automatic), .library(.static), .extension:
                     for target in product.targets {
                         try addStaticTargetInputs(target)
                     }
@@ -674,7 +674,7 @@ extension LLBuildManifestBuilder {
                     let binary = planProduct.binary
                     inputs.append(file: binary)
 
-                case .library(.automatic), .library(.static):
+                case .library(.automatic), .library(.static), .extension:
                     for target in product.targets {
                         addStaticTargetInputs(target)
                     }
@@ -847,6 +847,8 @@ extension ResolvedProduct {
             throw InternalError("automatic library not supported")
         case .executable:
             return "\(name)-\(config).exe"
+        case .extension:
+            throw InternalError("unexpectedly asked for the llbuild target name of an extension product")
         }
     }
 

Sources/PackageCollections/Providers/JSONPackageCollectionProvider.swift

@@ -342,6 +342,8 @@ extension PackageModel.ProductType {
             self = .library(.init(from: libraryType))
         case .executable:
             self = .executable
+        case .extension:
+            self = .extension
         case .test:
             self = .test
         }

Sources/PackageCollectionsModel/PackageCollectionModel+v1.swift

@@ -324,6 +324,9 @@ extension PackageCollectionModel.V1 {
 
         /// An executable product.
         case executable
+        
+        /// An extension product.
+        case `extension`
 
         /// A test product.
         case test
@@ -332,7 +335,7 @@ extension PackageCollectionModel.V1 {
 
 extension PackageCollectionModel.V1.ProductType: Codable {
     private enum CodingKeys: String, CodingKey {
-        case library, executable, test
+        case library, executable, `extension`, test
     }
 
     public func encode(to encoder: Encoder) throws {
@@ -343,6 +346,8 @@ extension PackageCollectionModel.V1.ProductType: Codable {
             try unkeyedContainer.encode(a1)
         case .executable:
             try container.encodeNil(forKey: .executable)
+        case .extension:
+            try container.encodeNil(forKey: .extension)
         case .test:
             try container.encodeNil(forKey: .test)
         }
@@ -362,6 +367,8 @@ extension PackageCollectionModel.V1.ProductType: Codable {
             self = .test
         case .executable:
             self = .executable
+        case .extension:
+            self = .extension
         }
     }
 }

Sources/PackageDescription/Product.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2018 Apple Inc. and the Swift project authors
+ Copyright (c) 2018 - 2021 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See http://swift.org/LICENSE.txt for license information
@@ -128,6 +128,29 @@ public class Product: Encodable {
         }
     }
 
+    /// The extension product of a Swift package.
+    public final class Extension: Product {
+        private enum ExtensionCodingKeys: CodingKey {
+            case targets
+        }
+
+        /// The name of the extension target to vend as a product.
+        public let targets: [String]
+
+        init(name: String, targets: [String]) {
+            self.targets = targets
+            super.init(name: name)
+        }
+
+        public override func encode(to encoder: Encoder) throws {
+            try super.encode(to: encoder)
+            var productContainer = encoder.container(keyedBy: ProductCodingKeys.self)
+            try productContainer.encode("extension", forKey: .type)
+            var extensionContainer = encoder.container(keyedBy: ExtensionCodingKeys.self)
+            try extensionContainer.encode(targets, forKey: .targets)
+        }
+    }
+
     /// Creates a library product to allow clients that declare a dependency on this package
     /// to use the package's functionality.
     ///
@@ -161,6 +184,19 @@ public class Product: Encodable {
     ) -> Product {
         return Executable(name: name, targets: targets)
     }
+    
+    /// Creates an extension package product.
+    ///
+    /// - Parameters:
+    ///     - name: The name of the extension product.
+    ///     - targets: The extension targets to vend as a product.
+    @available(_PackageDescription, introduced: 999.0)
+    public static func `extension`(
+        name: String,
+        targets: [String]
+    ) -> Product {
+        return Extension(name: name, targets: targets)
+    }
 
     public func encode(to encoder: Encoder) throws {
         var container = encoder.container(keyedBy: ProductCodingKeys.self)

Sources/PackageLoading/Diagnostics.swift

@@ -34,7 +34,7 @@ extension Diagnostic.Message {
         switch product.type {
         case .library(.automatic):
             typeString = ""
-        case .executable, .test,
+        case .executable, .extension, .test,
              .library(.dynamic), .library(.static):
             typeString = " (\(product.type))"
         }
@@ -76,6 +76,14 @@ extension Diagnostic.Message {
         .error("executable product '\(product)' should not have more than one executable target")
     }
 
+    static func extensionProductWithNoTargets(product: String) -> Diagnostic.Message {
+        .error("extension product '\(product)' should have at least one extension target")
+    }
+
+    static func extensionProductWithNonExtensionTargets(product: String, otherTargets: [String]) -> Diagnostic.Message {
+        .error("extension product '\(product)' should have only extension targets (it has \(otherTargets.map{ "'\($0)'" }.joined(separator: ", ")))")
+    }
+
     static var noLibraryTargetsForREPL: Diagnostic.Message {
         .error("unable to synthesize a REPL product as there are no library targets in the package")
     }

Sources/PackageLoading/PackageBuilder.swift

@@ -1192,6 +1192,10 @@ public final class PackageBuilder {
                 guard self.validateExecutableProduct(product, with: targets) else {
                     continue
                 }
+            case .extension:
+                guard self.validateExtensionProduct(product, with: targets) else {
+                    continue
+                }
             }
 
             append(Product(name: product.name, type: product.type, targets: targets))
@@ -1205,7 +1209,7 @@ public final class PackageBuilder {
             // for them.
             let explicitProductsTargets = Set(self.manifest.products.flatMap{ product -> [String] in
                 switch product.type {
-                case .library, .test:
+                case .library, .extension, .test:
                     return []
                 case .executable:
                     return product.targets
@@ -1284,6 +1288,25 @@ public final class PackageBuilder {
 
         return true
     }
+
+    private func validateExtensionProduct(_ product: ProductDescription, with targets: [Target]) -> Bool {
+        let nonExtensionTargets = targets.filter{ $0.type != .extension }
+        guard nonExtensionTargets.isEmpty else {
+            diagnostics.emit(
+                .extensionProductWithNonExtensionTargets(product: product.name, otherTargets: nonExtensionTargets.map{ $0.name }),
+                location: diagnosticLocation()
+            )
+            return false
+        }
+        guard !targets.isEmpty else {
+            diagnostics.emit(
+                .extensionProductWithNoTargets(product: product.name),
+                location: diagnosticLocation()
+            )
+            return false
+        }
+        return true
+    }
 }
 
 /// We create this structure after scanning the filesystem for potential targets.

Sources/PackageLoading/PackageDescription4Loader.swift

@@ -269,6 +269,9 @@ extension PackageModel.ProductType {
             }
 
             self = .library(libraryType)
+            
+        case "extension":
+            self = .extension
 
         default:
             throw InternalError("unexpected product type: \(json)")

Sources/PackageModel/ManifestSourceGeneration.swift

@@ -156,6 +156,8 @@ fileprivate extension SourceCodeFragment {
             self.init(enum: "library", subnodes: params, multiline: true)
         case .executable:
             self.init(enum: "executable", subnodes: params, multiline: true)
+        case .extension:
+            self.init(enum: "extension", subnodes: params, multiline: true)
         case .test:
             self.init(enum: "test", subnodes: params, multiline: true)
         }