SDK imports metadata (#75)

This provides new metadata during builds:
- Use the new `-sdk_imports` linker option when appropriate and provide a post-processed version of it as a resource in each bundle.
- For each codeless bundle, "stamp" its Info.plist with any clients, this allows finding the library associated with a particular codeless bundle.

Both behaviors are enabled by the `ENABLE_SDK_IMPORTS` build setting which defaults to the inverse of `ASSETCATALOG_COMPILER_SKIP_APP_STORE_DEPLOYMENT`.

Co-authored-by: Owen Voorhees <[email protected]>

Author: neonichu

Sources/SWBCore/PlannedTaskAction.swift

@@ -74,29 +74,32 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
     public var sdk: SDK?
     public var sdkVariant: SDKVariant?
     public var cleanupRequiredArchitectures: [String]
+    public var clientLibrariesForCodelessBundle: [String]
 
-    public init(scope: MacroEvaluationScope, productType: ProductTypeSpec?, platform: Platform?, sdk: SDK?, sdkVariant: SDKVariant?, cleanupRequiredArchitectures: [String]) {
+    public init(scope: MacroEvaluationScope, productType: ProductTypeSpec?, platform: Platform?, sdk: SDK?, sdkVariant: SDKVariant?, cleanupRequiredArchitectures: [String], clientLibrariesForCodelessBundle: [String] = []) {
         self.scope = scope
         self.productType = productType
         self.platform = platform
         self.sdk = sdk
         self.sdkVariant = sdkVariant
         self.cleanupRequiredArchitectures = cleanupRequiredArchitectures
+        self.clientLibrariesForCodelessBundle = clientLibrariesForCodelessBundle
     }
 
     public var targetBuildVersionPlatforms: Set<BuildVersion.Platform>? {
         targetBuildVersionPlatforms(in: scope)
     }
 
     public func serialize<T: Serializer>(to serializer: T) {
-        serializer.beginAggregate(7)
+        serializer.beginAggregate(8)
         serializer.serialize(scope)
         serializer.serialize(platform?.identifier)
         serializer.serialize(sdk?.canonicalName)
         serializer.serialize(sdkVariant?.name)
         serializer.serialize(productType?.identifier)
         serializer.serialize(productType?.domain)
         serializer.serialize(cleanupRequiredArchitectures)
+        serializer.serialize(clientLibrariesForCodelessBundle)
         serializer.endAggregate()
     }
 
@@ -105,7 +108,7 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
         // Get the platform registry to use to look up the platform from the deserializer's delegate.
         guard let delegate = deserializer.delegate as? (any InfoPlistProcessorTaskActionContextDeserializerDelegate) else { throw DeserializerError.invalidDelegate("delegate must be a BuildDescriptionDeserializerDelegate") }
 
-        try deserializer.beginAggregate(7)
+        try deserializer.beginAggregate(8)
         let scope: MacroEvaluationScope = try deserializer.deserialize()
 
         let platformIdentifier: String? = try deserializer.deserialize()
@@ -158,8 +161,9 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
         }()
 
         let cleanupRequiredArchitectures: [String] = try deserializer.deserialize()
+        let clientLibrariesForCodelessBundle: [String] = try deserializer.deserialize()
 
-        self = InfoPlistProcessorTaskActionContext(scope: scope, productType: productType, platform: platform, sdk: sdk, sdkVariant: sdkVariant, cleanupRequiredArchitectures: cleanupRequiredArchitectures)
+        self = InfoPlistProcessorTaskActionContext(scope: scope, productType: productType, platform: platform, sdk: sdk, sdkVariant: sdkVariant, cleanupRequiredArchitectures: cleanupRequiredArchitectures, clientLibrariesForCodelessBundle: clientLibrariesForCodelessBundle)
     }
 }
 
@@ -333,6 +337,7 @@ public protocol TaskActionCreationDelegate
     func createValidateDevelopmentAssetsTaskAction() -> any PlannedTaskAction
     func createSignatureCollectionTaskAction() -> any PlannedTaskAction
     func createClangModuleVerifierInputGeneratorTaskAction() -> any PlannedTaskAction
+    func createProcessSDKImportsTaskAction() -> any PlannedTaskAction
 }
 
 extension TaskActionCreationDelegate {

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -648,6 +648,7 @@ public final class BuiltinMacros {
     /// be migrated. This is merely a stop-gap measure until we can turn on the debug
     /// dylib for macOS completely.
     public static let ENABLE_PREVIEWS_DYLIB_OVERRIDE = BuiltinMacros.declareBooleanMacro("ENABLE_PREVIEWS_DYLIB_OVERRIDE")
+    public static let ENABLE_SDK_IMPORTS = BuiltinMacros.declareBooleanMacro("ENABLE_SDK_IMPORTS")
     public static let ENABLE_SIGNATURE_AGGREGATION = BuiltinMacros.declareBooleanMacro("ENABLE_SIGNATURE_AGGREGATION")
     public static let DISABLE_TASK_SANDBOXING = BuiltinMacros.declareBooleanMacro("DISABLE_TASK_SANDBOXING")
     public static let ENABLE_USER_SCRIPT_SANDBOXING = BuiltinMacros.declareBooleanMacro("ENABLE_USER_SCRIPT_SANDBOXING")
@@ -798,6 +799,7 @@ public final class BuiltinMacros {
     public static let LD_LTO_OBJECT_FILE = BuiltinMacros.declarePathMacro("LD_LTO_OBJECT_FILE")
     public static let LD_NO_PIE = BuiltinMacros.declareBooleanMacro("LD_NO_PIE")
     public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS")
+    public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE")
     public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS")
     public static let LEX = BuiltinMacros.declarePathMacro("LEX")
     public static let LEXFLAGS = BuiltinMacros.declareStringListMacro("LEXFLAGS")
@@ -1656,6 +1658,7 @@ public final class BuiltinMacros {
         ENABLE_DEBUG_DYLIB_OVERRIDE,
         ENFORCE_VALID_ARCHS,
         ENABLE_PREVIEWS_DYLIB_OVERRIDE,
+        ENABLE_SDK_IMPORTS,
         ENABLE_SIGNATURE_AGGREGATION,
         ENABLE_TESTABILITY,
         ENABLE_TESTING_SEARCH_PATHS,
@@ -1843,6 +1846,7 @@ public final class BuiltinMacros {
         LD_LTO_OBJECT_FILE,
         LD_NO_PIE,
         LD_RUNPATH_SEARCH_PATHS,
+        LD_SDK_IMPORTS_FILE,
         LD_WARN_UNUSED_DYLIBS,
         LEGACY_DEVELOPER_DIR,
         LEX,

Sources/SWBCore/Specs/CoreBuildSystem.xcspec

@@ -4462,6 +4462,13 @@ When this setting is enabled:
                 DisplayName = "Sticker Sharing Level";
                 Description = "When `GENERATE_INFOPLIST_FILE` is enabled, sets the value of the NSStickerSharingLevel key in the `Info.plist` file to the value of this build setting.";
             },
+
+            // Support for producing SDK imports metadata
+            {
+                Name = "ENABLE_SDK_IMPORTS";
+                Type = Boolean;
+                DefaultValue = "$(ASSETCATALOG_COMPILER_SKIP_APP_STORE_DEPLOYMENT:not)";
+            },
         );
     },
 )

Sources/SWBCore/Specs/RegisterSpecs.swift

@@ -123,6 +123,7 @@ public struct BuiltinSpecsExtension: SpecificationsExtension {
             CreateAssetPackManifestToolSpec.self,
             CreateBuildDirectorySpec.self,
             MergeInfoPlistSpec.self,
+            ProcessSDKImportsSpec.self,
             ProcessXCFrameworkLibrarySpec.self,
             RegisterExecutionPolicyExceptionToolSpec.self,
             SwiftHeaderToolSpec.self,

Sources/SWBCore/Specs/Tools/InfoPlistTool.swift

@@ -20,7 +20,7 @@ public final class InfoPlistToolSpec : GenericCommandLineToolSpec, SpecIdentifie
         fatalError("unexpected direct invocation")
     }
 
-    public func constructInfoPlistTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, generatedPkgInfoFile: Path? = nil, additionalContentFilePaths: [Path] = [], requiredArch: String? = nil, appPrivacyContentFiles: [Path] = []) async {
+    public func constructInfoPlistTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, generatedPkgInfoFile: Path? = nil, additionalContentFilePaths: [Path] = [], requiredArch: String? = nil, appPrivacyContentFiles: [Path] = [], clientLibrariesForCodelessBundle: [String] = []) async {
         let input = cbc.input
         let inputPath = input.absolutePath
         let outputPath = cbc.output
@@ -108,7 +108,7 @@ public final class InfoPlistToolSpec : GenericCommandLineToolSpec, SpecIdentifie
 
         commandLine += ["-o", outputPath.str]
 
-        let context = InfoPlistProcessorTaskActionContext(scope: cbc.scope, productType: cbc.producer.productType, platform: cbc.producer.platform, sdk: cbc.producer.sdk, sdkVariant: cbc.producer.sdkVariant, cleanupRequiredArchitectures: cleanupArchs.sorted())
+        let context = InfoPlistProcessorTaskActionContext(scope: cbc.scope, productType: cbc.producer.productType, platform: cbc.producer.platform, sdk: cbc.producer.sdk, sdkVariant: cbc.producer.sdkVariant, cleanupRequiredArchitectures: cleanupArchs.sorted(), clientLibrariesForCodelessBundle: clientLibrariesForCodelessBundle)
         let inputs = [inputPath] + effectiveAdditionalContentFilePaths + appPrivacyContentFiles
         let serializer = MsgPackSerializer()
         serializer.serialize(context)

Sources/SWBCore/Specs/Tools/LinkerTools.swift

@@ -556,6 +556,17 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
         // Generate the command line.
         var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup).map(\.asString)
 
+        // Add flags to emit SDK imports info.
+        let sdkImportsInfoFile = cbc.scope.evaluate(BuiltinMacros.LD_SDK_IMPORTS_FILE)
+        let supportsSDKImportsFeature = (try? optionContext?.toolVersion >= .init("1164")) == true
+        let usesLDClassic = commandLine.contains("-r") || commandLine.contains("-ld_classic") || cbc.scope.evaluate(BuiltinMacros.CURRENT_ARCH) == "armv7k"
+        if !usesLDClassic, supportsSDKImportsFeature, !sdkImportsInfoFile.isEmpty, cbc.scope.evaluate(BuiltinMacros.ENABLE_SDK_IMPORTS), cbc.producer.isApplePlatform {
+            commandLine.insert(contentsOf: ["-Xlinker", "-sdk_imports", "-Xlinker", sdkImportsInfoFile.str, "-Xlinker", "-sdk_imports_each_object"], at: commandLine.count - 2) // This preserves the assumption that the last argument is the linker output which a few tests make.
+            outputs.append(delegate.createNode(sdkImportsInfoFile))
+
+            await cbc.producer.processSDKImportsSpec.createTasks(CommandBuildContext(producer: cbc.producer, scope: cbc.scope, inputs: []), delegate, ldSDKImportsPath: sdkImportsInfoFile)
+        }
+
         // Select the driver to use based on the input file types, replacing the value computed by commandLineFromTemplate().
         let usedCXX = usedTools.values.contains(where: { $0.contains(where: { $0.languageDialect?.isPlusPlus ?? false }) })
         commandLine[0] = resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX)).str
@@ -957,7 +968,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
         }
 
         // Args without parameters (-Xlinker-prefixed, e.g. -Xlinker)
-        for arg in ["-bitcode_verify", "-export_dynamic"] {
+        for arg in ["-bitcode_verify", "-export_dynamic", "-sdk_imports_each_object"] {
             while let index = commandLine.firstIndex(of: arg) {
                 guard index > 0, commandLine[index - 1] == argPrefix else { break }
                 commandLine.removeSubrange(index - 1 ... index)
@@ -976,7 +987,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
         }
 
         // Args with a parameter (-Xlinker-prefixed, e.g. -Xlinker arg -Xlinker param)
-        for arg in ["-object_path_lto", "-add_ast_path", "-dependency_info", "-map", "-order_file", "-bitcode_symbol_map", "-final_output", "-allowable_client"] {
+        for arg in ["-object_path_lto", "-add_ast_path", "-dependency_info", "-map", "-order_file", "-bitcode_symbol_map", "-final_output", "-allowable_client", "-sdk_imports"] {
             while let index = commandLine.firstIndex(of: arg) {
                 guard index > 0,
                     index + 2 < commandLine.count,
@@ -1228,67 +1239,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
             return nil
         }
 
-        do {
-            do {
-                let commandLine = [toolPath.str, "-version_details"]
-                return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
-                    let gnuLD = [
-                        #/GNU ld version (?<version>[\d.]+)-.*/#,
-                        #/GNU ld \(GNU Binutils.*\) (?<version>[\d.]+)/#,
-                    ]
-                    if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
-                        return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
-                    }
-
-                    let goLD = [
-                        #/GNU gold version (?<version>[\d.]+)-.*/#,
-                        #/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
-                    ]
-                    if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
-                        return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
-                    }
-                    
-                    struct LDVersionDetails: Decodable {
-                        let version: Version
-                        let architectures: Set<String>
-                    }
-
-                    let details: LDVersionDetails
-                    do {
-                        details = try JSONDecoder().decode(LDVersionDetails.self, from: executionResult.stdout)
-                    } catch {
-                        throw CommandLineOutputJSONParsingError(commandLine: commandLine, data: executionResult.stdout)
-                    }
-                    
-                    return DiscoveredLdLinkerToolSpecInfo(linker: .ld64, toolPath: toolPath, toolVersion: details.version, architectures: details.architectures)
-                })
-            } catch let e as CommandLineOutputJSONParsingError {
-                let vCommandLine = [toolPath.str, "-v"]
-                return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, vCommandLine, { executionResult in
-                    let lld = [
-                        #/LLD (?<version>[\d.]+) .*/#,
-                    ]
-                    if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
-                        return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
-                    }
-
-                    let versionCommandLine = [toolPath.str, "--version"]
-                    return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, versionCommandLine, { executionResult in
-                        let lld = [
-                            #/LLD (?<version>[\d.]+) .*/#,
-                        ]
-                        if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
-                            return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
-                        }
-                    
-                        throw e
-                    })
-                })
-            }
-        } catch {
-            delegate.error(error)
-            return nil
-        }
+        return await discoveredLinkerToolsInfo(producer, delegate, at: toolPath)
     }
 }
 
@@ -1582,6 +1533,73 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
     }
 }
 
+/// Consults the global cache of discovered info for the linker at `toolPath` and returns it, creating it if necessary.
+///
+/// This is global and public because it is used by `SWBTaskExecution` and `CoreBasedTests`, which is the basis of many of our tests (so caching this info across tests is desirable).
+public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> (any DiscoveredCommandLineToolSpecInfo)? {
+    do {
+        do {
+            let commandLine = [toolPath.str, "-version_details"]
+            return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
+                let gnuLD = [
+                    #/GNU ld version (?<version>[\d.]+)-.*/#,
+                    #/GNU ld \(GNU Binutils.*\) (?<version>[\d.]+)/#,
+                ]
+                if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
+                    return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
+                }
+
+                let goLD = [
+                    #/GNU gold version (?<version>[\d.]+)-.*/#,
+                    #/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
+                ]
+                if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
+                    return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
+                }
+
+                struct LDVersionDetails: Decodable {
+                    let version: Version
+                    let architectures: Set<String>
+                }
+
+                let details: LDVersionDetails
+                do {
+                    details = try JSONDecoder().decode(LDVersionDetails.self, from: executionResult.stdout)
+                } catch {
+                    throw CommandLineOutputJSONParsingError(commandLine: commandLine, data: executionResult.stdout)
+                }
+
+                return DiscoveredLdLinkerToolSpecInfo(linker: .ld64, toolPath: toolPath, toolVersion: details.version, architectures: details.architectures)
+            })
+        } catch let e as CommandLineOutputJSONParsingError {
+            let vCommandLine = [toolPath.str, "-v"]
+            return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, vCommandLine, { executionResult in
+                let lld = [
+                    #/LLD (?<version>[\d.]+) .*/#,
+                ]
+                if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
+                    return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
+                }
+
+                let versionCommandLine = [toolPath.str, "--version"]
+                return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, versionCommandLine, { executionResult in
+                    let lld = [
+                        #/LLD (?<version>[\d.]+) .*/#,
+                    ]
+                    if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
+                        return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
+                    }
+
+                    throw e
+                })
+            })
+        }
+    } catch {
+        delegate.error(error)
+        return nil
+    }
+}
+
 /// Enumerates a linker command line, calling the provided `handle` closure for each logical argument, providing the argument name as `arg` and a `value` function which returns the value of that argument at `offset` positions distance from the `arg` value based on how many arguments it takes.
 ///
 /// The purpose of this function is to automatically abstract away handling of the `-Xlinker` and `-Wl` syntax used by compiler drivers to forward arguments to the linker which they themselves do not understand.