[6.0] Add prepare for index experimental build argument

.gitignore

@@ -18,3 +18,4 @@ Package.resolved
 .docc-build
 .vscode
 Utilities/InstalledSwiftPMConfiguration/config.json
+.devcontainer

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -561,6 +561,16 @@ public final class SwiftTargetBuildDescription {
             args += ["-emit-module-interface-path", self.parseableModuleInterfaceOutputPath.pathString]
         }
 
+        if self.buildParameters.prepareForIndexing {
+            args += [
+                "-Xfrontend", "-experimental-skip-all-function-bodies",
+                "-Xfrontend", "-experimental-lazy-typecheck",
+                "-Xfrontend", "-experimental-skip-non-exportable-decls",
+                "-Xfrontend", "-experimental-allow-module-with-compiler-errors",
+                "-Xfrontend", "-empty-abi-descriptor"
+            ]
+        }
+
         args += self.buildParameters.toolchain.extraFlags.swiftCompilerFlags
         // User arguments (from -Xswiftc) should follow generated arguments to allow user overrides
         args += self.buildParameters.flags.swiftCompilerFlags

Sources/Build/BuildManifest/LLBuildManifestBuilder+Clang.swift

@@ -110,4 +110,15 @@ extension LLBuildManifestBuilder {
             self.addNode(output, toTarget: .test)
         }
     }
+
+    /// Create a llbuild target for a Clang target preparation
+    func createClangPrepareCommand(
+        _ target: ClangTargetBuildDescription
+    ) throws {
+        // Create the node for the target so you can --target it.
+        // It is a no-op for index preparation.
+        let targetName = target.llbuildTargetName
+        let output: Node = .virtual(targetName)
+        self.manifest.addNode(output, toTarget: targetName)
+    }
 }

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -41,7 +41,7 @@ extension LLBuildManifestBuilder {
         let inputs = try self.computeSwiftCompileCmdInputs(target)
 
         // Outputs.
-        let objectNodes = try target.objects.map(Node.file)
+        let objectNodes = target.buildParameters.prepareForIndexing ? [] : try target.objects.map(Node.file)
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
@@ -394,7 +394,8 @@ extension LLBuildManifestBuilder {
             fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: target.buildParameters.configuration == .release,
-            outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.
+            outputFileMapPath: try target.writeOutputFileMap(), // FIXME: Eliminate side effect.
+            prepareForIndexing: target.buildParameters.prepareForIndexing
         )
     }
 
@@ -419,6 +420,8 @@ extension LLBuildManifestBuilder {
             self.manifest.addWriteEmbeddedResourcesCommand(resources: resourceFilesToEmbed, outputPath: resourcesEmbeddingSource)
         }
 
+        let prepareForIndexing = target.buildParameters.prepareForIndexing
+
         func addStaticTargetInputs(_ target: ResolvedModule) throws {
             // Ignore C Modules.
             if target.underlying is SystemLibraryTarget { return }
@@ -428,7 +431,7 @@ extension LLBuildManifestBuilder {
             if target.underlying is PluginTarget { return }
 
             // Depend on the binary for executable targets.
-            if target.type == .executable {
+            if target.type == .executable && !prepareForIndexing {
                 // FIXME: Optimize.
                 if let productDescription = try plan.productMap.values.first(where: {
                     try $0.product.type == .executable && $0.product.executableTarget.id == target.id
@@ -442,8 +445,16 @@ extension LLBuildManifestBuilder {
             case .swift(let target)?:
                 inputs.append(file: target.moduleOutputPath)
             case .clang(let target)?:
-                for object in try target.objects {
-                    inputs.append(file: object)
+                if prepareForIndexing {
+                    // In preparation, we're only building swiftmodules
+                    // propagate the dependency to the header files in this target
+                    for header in target.clangTarget.headers {
+                        inputs.append(file: header)
+                    }
+                } else {
+                    for object in try target.objects {
+                        inputs.append(file: object)
+                    }
                 }
             case nil:
                 throw InternalError("unexpected: target \(target) not in target map \(self.plan.targetMap)")

Sources/Build/BuildManifest/LLBuildManifestBuilder.swift

@@ -127,6 +127,44 @@ public class LLBuildManifestBuilder {
         return self.manifest
     }
 
+    package func generatePrepareManifest(at path: AbsolutePath) throws -> LLBuildManifest {
+        self.swiftGetVersionFiles.removeAll()
+
+        self.manifest.createTarget(TargetKind.main.targetName)
+        self.manifest.createTarget(TargetKind.test.targetName)
+        self.manifest.defaultTarget = TargetKind.main.targetName
+
+        addPackageStructureCommand()
+
+        for (_, description) in self.plan.targetMap {
+            switch description {
+            case .swift(let desc):
+                try self.createSwiftCompileCommand(desc)
+            case .clang(let desc):
+                if desc.target.buildTriple == .tools {
+                    // Need the clang targets for tools
+                    try self.createClangCompileCommand(desc)
+                } else {
+                    // Hook up the clang module target
+                    try self.createClangPrepareCommand(desc)
+                }
+            }
+        }
+
+        for (_, description) in self.plan.productMap {
+            // Need to generate macro products
+            switch description.product.type {
+            case .macro, .plugin:
+                try self.createProductCommand(description)
+            default:
+                break
+            }
+        }
+
+        try LLBuildManifestWriter.write(self.manifest, at: path, fileSystem: self.fileSystem)
+        return self.manifest
+    }
+
     func addNode(_ node: Node, toTarget targetKind: TargetKind) {
         self.manifest.addNode(node, toTarget: targetKind.targetName)
     }

Sources/Build/BuildOperation.swift

@@ -900,7 +900,9 @@ extension BuildDescription {
     ) throws -> (BuildDescription, LLBuildManifest) {
         // Generate the llbuild manifest.
         let llbuild = LLBuildManifestBuilder(plan, disableSandboxForPluginCommands: disableSandboxForPluginCommands, fileSystem: fileSystem, observabilityScope: observabilityScope)
-        let buildManifest = try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
+        let buildManifest = plan.destinationBuildParameters.prepareForIndexing
+            ? try llbuild.generatePrepareManifest(at: plan.destinationBuildParameters.llbuildManifest)
+            : try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
 
         let swiftCommands = llbuild.manifest.getCmdToolMap(kind: SwiftCompilerTool.self)
         let swiftFrontendCommands = llbuild.manifest.getCmdToolMap(kind: SwiftFrontendTool.self)

Sources/CoreCommands/Options.swift

@@ -438,6 +438,9 @@ public struct BuildOptions: ParsableArguments {
     @Flag(help: "Enable or disable indexing-while-building feature")
     public var indexStoreMode: StoreMode = .autoIndexStore
 
+    @Flag(name: .customLong("experimental-prepare-for-indexing"), help: .hidden)
+    var prepareForIndexing: Bool = false
+
     /// Whether to enable generation of `.swiftinterface`s alongside `.swiftmodule`s.
     @Flag(name: .customLong("enable-parseable-module-interfaces"))
     public var shouldEnableParseableModuleInterfaces: Bool = false

Sources/CoreCommands/SwiftCommandState.swift

@@ -728,7 +728,8 @@ public final class SwiftCommandState {
 
     private func _buildParams(
         toolchain: UserToolchain,
-        destination: BuildParameters.Destination
+        destination: BuildParameters.Destination,
+        prepareForIndexing: Bool? = nil
     ) throws -> BuildParameters {
         let triple = toolchain.targetTriple
 
@@ -753,6 +754,7 @@ public final class SwiftCommandState {
             sanitizers: options.build.enabledSanitizers,
             indexStoreMode: options.build.indexStoreMode.buildParameter,
             isXcodeBuildSystemEnabled: options.build.buildSystem == .xcode,
+            prepareForIndexing: prepareForIndexing ?? options.build.prepareForIndexing,
             debuggingParameters: .init(
                 debugInfoFormat: options.build.debugInfoFormat.buildParameter,
                 triple: triple,
@@ -801,7 +803,7 @@ public final class SwiftCommandState {
 
     private lazy var _toolsBuildParameters: Result<BuildParameters, Swift.Error> = {
         Result(catching: {
-            try _buildParams(toolchain: self.getHostToolchain(), destination: .host)
+            try _buildParams(toolchain: self.getHostToolchain(), destination: .host, prepareForIndexing: false)
         })
     }()
 

Sources/LLBuildManifest/LLBuildManifest.swift

@@ -408,7 +408,8 @@ public struct LLBuildManifest {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         assert(commands[name] == nil, "already had a command named '\(name)'")
         let tool = SwiftCompilerTool(
@@ -426,7 +427,8 @@ public struct LLBuildManifest {
             fileList: fileList,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
-            outputFileMapPath: outputFileMapPath
+            outputFileMapPath: outputFileMapPath,
+            prepareForIndexing: prepareForIndexing
         )
         commands[name] = Command(name: name, tool: tool)
     }

Sources/LLBuildManifest/Tools.swift

@@ -271,6 +271,7 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
+    public var prepareForIndexing: Bool
 
     init(
         inputs: [Node],
@@ -287,7 +288,8 @@ public struct SwiftCompilerTool: ToolProtocol {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         self.inputs = inputs
         self.outputs = outputs
@@ -304,6 +306,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
+        self.prepareForIndexing = prepareForIndexing
     }
 
     var description: String {
@@ -334,7 +337,10 @@ public struct SwiftCompilerTool: ToolProtocol {
         } else {
             arguments += ["-incremental"]
         }
-        arguments += ["-c", "@\(self.fileList.pathString)"]
+        if !prepareForIndexing {
+            arguments += ["-c"]
+        }
+        arguments += ["@\(self.fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Sources/SPMBuildCore/BuildParameters/BuildParameters+Debugging.swift

@@ -63,7 +63,7 @@ extension BuildParameters {
 
     /// The debugging strategy according to the current build parameters.
     public var debuggingStrategy: DebuggingStrategy? {
-        guard configuration == .debug else {
+        guard configuration == .debug, !prepareForIndexing else {
             return nil
         }
 

Sources/SPMBuildCore/BuildParameters/BuildParameters.swift

@@ -114,6 +114,9 @@ public struct BuildParameters: Encodable {
 
     public var shouldSkipBuilding: Bool
 
+    /// Do minimal build to prepare for indexing
+    public var prepareForIndexing: Bool
+
     /// Build parameters related to debugging.
     public var debuggingParameters: Debugging
 
@@ -144,6 +147,7 @@ public struct BuildParameters: Encodable {
         indexStoreMode: IndexStoreMode = .auto,
         isXcodeBuildSystemEnabled: Bool = false,
         shouldSkipBuilding: Bool = false,
+        prepareForIndexing: Bool = false,
         debuggingParameters: Debugging? = nil,
         driverParameters: Driver = .init(),
         linkingParameters: Linking = .init(),
@@ -199,6 +203,7 @@ public struct BuildParameters: Encodable {
         self.indexStoreMode = indexStoreMode
         self.isXcodeBuildSystemEnabled = isXcodeBuildSystemEnabled
         self.shouldSkipBuilding = shouldSkipBuilding
+        self.prepareForIndexing = prepareForIndexing
         self.driverParameters = driverParameters
         self.linkingParameters = linkingParameters
         self.outputParameters = outputParameters

Sources/SPMTestSupport/MockBuildTestHelper.swift

@@ -89,7 +89,8 @@ package func mockBuildParameters(
     useExplicitModuleBuild: Bool = false,
     linkerDeadStrip: Bool = true,
     linkTimeOptimizationMode: BuildParameters.LinkTimeOptimizationMode? = nil,
-    omitFramePointers: Bool? = nil
+    omitFramePointers: Bool? = nil,
+    prepareForIndexing: Bool = false
 ) -> BuildParameters {
     try! BuildParameters(
         destination: destination,
@@ -101,6 +102,7 @@ package func mockBuildParameters(
         pkgConfigDirectories: [],
         workers: 3,
         indexStoreMode: indexStoreMode,
+        prepareForIndexing: prepareForIndexing,
         debuggingParameters: .init(
             triple: triple,
             shouldEnableDebuggingEntitlement: config == .debug,

Tests/BuildTests/PrepareForIndexTests.swift

@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2015-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Build
+import Foundation
+import LLBuildManifest
+@_spi(SwiftPMInternal)
+import SPMTestSupport
+import TSCBasic
+import XCTest
+
+class PrepareForIndexTests: XCTestCase {
+    func testPrepare() throws {
+        let (graph, fs, scope) = try macrosPackageGraph()
+
+        let plan = try BuildPlan(
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: true),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: scope
+        )
+
+        let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
+        let manifest = try builder.generatePrepareManifest(at: "/manifest")
+
+        // Make sure we're building the swift modules
+        let outputs = manifest.commands.flatMap(\.value.tool.outputs).map(\.name)
+        XCTAssertTrue(outputs.contains(where: { $0.hasSuffix(".swiftmodule") }))
+
+        // Ensure swiftmodules built with correct arguments
+        let coreCommands = manifest.commands.values.filter {
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Core.build/Core.swiftmodule")
+            })
+        }
+        XCTAssertEqual(coreCommands.count, 1)
+        let coreSwiftc = try XCTUnwrap(coreCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertTrue(coreSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Ensure tools are built normally
+        let toolCommands = manifest.commands.values.filter {
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Modules-tool/SwiftSyntax.swiftmodule")
+            })
+        }
+        XCTAssertEqual(toolCommands.count, 1)
+        let toolSwiftc = try XCTUnwrap(toolCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertFalse(toolSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Make sure only object files for tools are built
+        XCTAssertTrue(
+            outputs.filter { $0.hasSuffix(".o") }.allSatisfy { $0.contains("-tool.build/") },
+            "outputs:\n\t\(outputs.filter { $0.hasSuffix(".o") }.joined(separator: "\n\t"))"
+        )
+    }
+
+    func testCModuleTarget() throws {
+        let (graph, fs, scope) = try trivialPackageGraph()
+
+        let plan = try BuildPlan(
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: true),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: scope
+        )
+        let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
+        let manifest = try builder.generatePrepareManifest(at: "/manifest")
+
+        // Ensure our C module is here.
+        let lib = try XCTUnwrap(graph.target(for: "lib", destination: .destination))
+        let name = lib.getLLBuildTargetName(buildParameters: plan.destinationBuildParameters)
+        XCTAssertTrue(manifest.targets.keys.contains(name))
+    }
+}