[6.0] Improvements to prebuilt/provided library handling (#7713)

Cherry-pick of
#7496 and
#7433, excluding
[16b4142](16b4142).

**Explanation**: The idea is to enable provided libraries to be a part
of a toolchain, which means that instead of completely eliding them from
a package graph, they need to be handled post-resolution in a special
way and reflected in a build plan for Swift targets (as compiler and
linker arguments). Unified handling of `config.json` and
`provided-libraries.json` resources and dropped use of `Bundle` type
(config should always be a part of SDK or a toolchain).
**Scope**: Localized, touches modules graph resolution and build
planning only in certain configurations.
**Risk**: Low. While the change has a somewhat broad scope, it's NFC for
most builds and has been incubated on `main` for 2 months with no known
issues.
**Testing**: New test cases added and existing ones updated.
**Issue**: rdar://125531670
**Reviewer**: @MaxDesiatov and @bnbarham

---------

Co-authored-by: Pavel Yaskevich <[email protected]>

Author: MaxDesiatov

Sources/Build/BuildDescription/ProductBuildDescription.swift

@@ -46,6 +46,9 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
     // Computed during build planning.
     var dylibs: [ProductBuildDescription] = []
 
+    /// The list of provided libraries that are going to be used by this product.
+    var providedLibraries: [String: AbsolutePath] = [:]
+
     /// Any additional flags to be added. These flags are expected to be computed during build planning.
     var additionalFlags: [String] = []
 
@@ -157,6 +160,8 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
             args += ["-F", self.buildParameters.buildPath.pathString]
         }
 
+        self.providedLibraries.forEach { args += ["-L", $1.pathString, "-l", $0] }
+
         args += ["-L", self.buildParameters.buildPath.pathString]
         args += try ["-o", binaryPath.pathString]
         args += ["-module-name", self.product.name.spm_mangledToC99ExtendedIdentifier()]

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -429,6 +429,8 @@ extension LLBuildManifestBuilder {
             if target.underlying is BinaryTarget { return }
             // Ignore Plugin Targets.
             if target.underlying is PluginTarget { return }
+            // Ignore Provided Libraries.
+            if target.underlying is ProvidedLibraryTarget { return }
 
             // Depend on the binary for executable targets.
             if target.type == .executable && !prepareForIndexing {

Sources/Build/BuildOperation.swift

@@ -275,7 +275,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     // TODO: Currently this function will only match frameworks.
     func detectUnexpressedDependencies(
-        availableLibraries: [LibraryMetadata],
+        availableLibraries: [ProvidedLibrary],
         targetDependencyMap: [String: [String]]?
     ) {
         // Ensure we only emit these once, regardless of how many builds are being done.
@@ -285,8 +285,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         Self.didEmitUnexpressedDependencies = true
 
         let availableFrameworks = Dictionary<String, PackageIdentity>(uniqueKeysWithValues: availableLibraries.compactMap {
-            if let identity = Set($0.identities.map(\.identity)).spm_only {
-                return ("\($0.productName!).framework", identity)
+            if let identity = Set($0.metadata.identities.map(\.identity)).spm_only {
+                return ("\($0.metadata.productName).framework", identity)
             } else {
                 return nil
             }

Sources/Build/BuildPlan/BuildPlan+Product.swift

@@ -116,6 +116,8 @@ extension BuildPlan {
         }
         buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
 
+        buildProduct.providedLibraries = dependencies.providedLibraries
+
         buildProduct.availableTools = dependencies.availableTools
     }
 
@@ -128,6 +130,7 @@ extension BuildPlan {
         staticTargets: [ResolvedModule],
         systemModules: [ResolvedModule],
         libraryBinaryPaths: Set<AbsolutePath>,
+        providedLibraries: [String: AbsolutePath],
         availableTools: [String: AbsolutePath]
     ) {
         /* Prior to tools-version 5.9, we used to erroneously recursively traverse executable/plugin dependencies and statically include their
@@ -205,6 +208,7 @@ extension BuildPlan {
         var staticTargets = [ResolvedModule]()
         var systemModules = [ResolvedModule]()
         var libraryBinaryPaths: Set<AbsolutePath> = []
+        var providedLibraries = [String: AbsolutePath]()
         var availableTools = [String: AbsolutePath]()
 
         for dependency in allTargets {
@@ -258,6 +262,8 @@ extension BuildPlan {
                     }
                 case .plugin:
                     continue
+                case .providedLibrary:
+                    providedLibraries[target.name] = target.underlying.path
                 }
 
             case .product(let product, _):
@@ -275,7 +281,7 @@ extension BuildPlan {
             }
         }
 
-        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, availableTools)
+        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, providedLibraries, availableTools)
     }
 
     /// Extracts the artifacts  from an artifactsArchive

Sources/Build/BuildPlan/BuildPlan+Swift.swift

@@ -14,6 +14,7 @@ import struct Basics.InternalError
 import class PackageModel.BinaryTarget
 import class PackageModel.ClangTarget
 import class PackageModel.SystemLibraryTarget
+import class PackageModel.ProvidedLibraryTarget
 
 extension BuildPlan {
     func plan(swiftTarget: SwiftTargetBuildDescription) throws {
@@ -48,6 +49,10 @@ extension BuildPlan {
                         swiftTarget.libraryBinaryPaths.insert(library.libraryPath)
                     }
                 }
+            case let target as ProvidedLibraryTarget:
+                swiftTarget.additionalFlags += [
+                    "-I", target.path.pathString
+                ]
             default:
                 break
             }

Sources/Build/BuildPlan/BuildPlan.swift

@@ -419,7 +419,7 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
                     toolsVersion: toolsVersion,
                     fileSystem: fileSystem
                 ))
-            case is SystemLibraryTarget, is BinaryTarget:
+            case is SystemLibraryTarget, is BinaryTarget, is ProvidedLibraryTarget:
                 break
             default:
                 throw InternalError("unhandled \(target.underlying)")

Sources/Commands/PackageCommands/EditCommands.swift

@@ -72,7 +72,6 @@ extension SwiftPackageCommand {
                 packageName: packageName,
                 forceRemove: shouldForceRemove,
                 root: swiftCommandState.getWorkspaceRoot(),
-                availableLibraries: swiftCommandState.getHostToolchain().providedLibraries,
                 observabilityScope: swiftCommandState.observabilityScope
             )
         }

Sources/Commands/PackageCommands/Update.swift

@@ -71,7 +71,7 @@ extension SwiftPackageCommand {
                 case .removed:
                     report += "\n"
                     report += "- \(package.identity) \(currentVersion)"
-                case .unchanged:
+                case .unchanged, .usesLibrary:
                     continue
                 }
             }

Sources/Commands/Snippets/Cards/TopCard.swift

@@ -151,6 +151,8 @@ fileprivate extension Target.Kind {
             return "snippets"
         case .macro:
             return "macros"
+        case .providedLibrary:
+            return "provided libraries"
         }
     }
 }

Sources/CoreCommands/SwiftCommandState.swift

@@ -622,7 +622,6 @@ public final class SwiftCommandState {
                 explicitProduct: explicitProduct,
                 forceResolvedVersions: options.resolver.forceResolvedVersions,
                 testEntryPointPath: testEntryPointPath,
-                availableLibraries: self.getHostToolchain().providedLibraries,
                 observabilityScope: self.observabilityScope
             )
 

Sources/PackageGraph/BoundVersion.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import struct PackageModel.ProvidedLibrary
 import struct TSCUtility.Version
 
 /// A bound version for a package within an assignment.
@@ -22,7 +23,7 @@ public enum BoundVersion: Equatable, Hashable {
     case excluded
 
     /// The version of the package to include.
-    case version(Version)
+    case version(Version, library: ProvidedLibrary? = nil)
 
     /// The package assignment is unversioned.
     case unversioned
@@ -36,7 +37,7 @@ extension BoundVersion: CustomStringConvertible {
         switch self {
         case .excluded:
             return "excluded"
-        case .version(let version):
+        case .version(let version, _):
             return version.description
         case .unversioned:
             return "unversioned"

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -34,7 +34,6 @@ extension ModulesGraph {
         customPlatformsRegistry: PlatformRegistry? = .none,
         customXCTestMinimumDeploymentTargets: [PackageModel.Platform: PlatformVersion]? = .none,
         testEntryPointPath: AbsolutePath? = nil,
-        availableLibraries: [LibraryMetadata],
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope
     ) throws -> ModulesGraph {
@@ -172,7 +171,6 @@ extension ModulesGraph {
             unsafeAllowedPackages: unsafeAllowedPackages,
             platformRegistry: customPlatformsRegistry ?? .default,
             platformVersionProvider: platformVersionProvider,
-            availableLibraries: availableLibraries,
             fileSystem: fileSystem,
             observabilityScope: observabilityScope
         )
@@ -272,7 +270,6 @@ private func createResolvedPackages(
     unsafeAllowedPackages: Set<PackageReference>,
     platformRegistry: PlatformRegistry,
     platformVersionProvider: PlatformVersionProvider,
-    availableLibraries: [LibraryMetadata],
     fileSystem: FileSystem,
     observabilityScope: ObservabilityScope
 ) throws -> IdentifiableSet<ResolvedPackage> {
@@ -551,32 +548,26 @@ private func createResolvedPackages(
                             t.name != productRef.name
                         }
 
-                        let identitiesAvailableInSDK = availableLibraries.flatMap { $0.identities.map { $0.identity } }
-                        // TODO: Do we have to care about "name" vs. identity here?
-                        if let name = productRef.package, identitiesAvailableInSDK.contains(PackageIdentity.plain(name)) {
-                            // Do not emit any diagnostic.
-                        } else {
-                            // Find a product name from the available product dependencies that is most similar to the required product name.
-                            let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
-                            var packageContainingBestMatchedProduct: String?
-                            if let bestMatchedProductName, productRef.name == bestMatchedProductName {
-                                let dependentPackages = packageBuilder.dependencies.map(\.package)
-                                for p in dependentPackages where p.targets.contains(where: { $0.name == bestMatchedProductName }) {
-                                    packageContainingBestMatchedProduct = p.identity.description
-                                    break
-                                }
+                        // Find a product name from the available product dependencies that is most similar to the required product name.
+                        let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
+                        var packageContainingBestMatchedProduct: String?
+                        if let bestMatchedProductName, productRef.name == bestMatchedProductName {
+                            let dependentPackages = packageBuilder.dependencies.map(\.package)
+                            for p in dependentPackages where p.targets.contains(where: { $0.name == bestMatchedProductName }) {
+                                packageContainingBestMatchedProduct = p.identity.description
+                                break
                             }
-                            let error = PackageGraphError.productDependencyNotFound(
-                                package: package.identity.description,
-                                targetName: targetBuilder.target.name,
-                                dependencyProductName: productRef.name,
-                                dependencyPackageName: productRef.package,
-                                dependencyProductInDecl: !declProductsAsDependency.isEmpty,
-                                similarProductName: bestMatchedProductName, 
-                                packageContainingSimilarProduct: packageContainingBestMatchedProduct
-                            )
-                            packageObservabilityScope.emit(error)
                         }
+                        let error = PackageGraphError.productDependencyNotFound(
+                            package: package.identity.description,
+                            targetName: targetBuilder.target.name,
+                            dependencyProductName: productRef.name,
+                            dependencyPackageName: productRef.package,
+                            dependencyProductInDecl: !declProductsAsDependency.isEmpty,
+                            similarProductName: bestMatchedProductName,
+                            packageContainingSimilarProduct: packageContainingBestMatchedProduct
+                        )
+                        packageObservabilityScope.emit(error)
                     }
                     continue
                 }

Sources/PackageGraph/Resolution/PubGrub/ContainerProvider.swift

@@ -97,6 +97,7 @@ final class ContainerProvider {
             ) { result in
                 let result = result.tryMap { container -> PubGrubPackageContainer in
                     let pubGrubContainer = PubGrubPackageContainer(underlying: container, pins: self.pins)
+
                     // only cache positive results
                     self.containersCache[package] = pubGrubContainer
                     return pubGrubContainer
@@ -107,12 +108,9 @@ final class ContainerProvider {
     }
 
     /// Starts prefetching the given containers.
-    func prefetch(containers identifiers: [PackageReference], availableLibraries: [LibraryMetadata]) {
-        let filteredIdentifiers = identifiers.filter {
-            $0.matchingPrebuiltLibrary(in: availableLibraries) == nil
-        }
+    func prefetch(containers identifiers: [PackageReference]) {
         // Process each container.
-        for identifier in filteredIdentifiers {
+        for identifier in identifiers {
             var needsFetching = false
             self.prefetches.memoize(identifier) {
                 let group = DispatchGroup()