Add prepare for index experimental build argument (#7574)

This will be used by sourcekit-lsp to build the swiftmodule files it
needs for indexing.

Adds experimental-prepare-for-indexing flag to swift build. Creates a
llbuild manifest specific for the prepare build that skips generating
object files and linking of those files and calls swiftc to only create
the swiftmodule as quickly as it can.

 ### Motivation:

To support background indexing in sourcekit-lsp, it will request a
prepare for index build to build the swiftmodule files it needs to do
indexing. This build should be minimal to ensure indexing is fast so it
can respond to language server requests that require the index as soon
as possible.

 ### Modifications:

- Add an experimental-prepare-for-indexing flag to the BuildOptions and
pass it around to every that needs it.
- Build a custom llbuild manifest so that only the commands necessary
are added to the prepare build
- Add a target property that also ensures tool builds required for the
prepare build are performed as usual.
- In SwiftTargetBuildDescription, pass compile options that put the
swift compiler in prepare "mode".
- Ensure object files and binaries are only generated for tools when in
prepare mode.

 ### Result:

Implements prepare build mode without affecting regular build mode.

(cherry picked from commit 7bd34cc)

Author: dschaefer2

.gitignore

@@ -18,3 +18,4 @@ Package.resolved
 .docc-build
 .vscode
 Utilities/InstalledSwiftPMConfiguration/config.json
+.devcontainer

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -551,6 +551,17 @@ public final class SwiftTargetBuildDescription {
             args += ["-emit-module-interface-path", self.parseableModuleInterfaceOutputPath.pathString]
         }
 
+        if self.buildParameters.prepareForIndexing {
+            args += [
+                "-Xfrontend", "-enable-library-evolution",
+                "-Xfrontend", "-experimental-skip-all-function-bodies",
+                "-Xfrontend", "-experimental-lazy-typecheck",
+                "-Xfrontend", "-experimental-skip-non-exportable-decls",
+                "-Xfrontend", "-experimental-allow-module-with-compiler-errors",
+                "-Xfrontend", "-empty-abi-descriptor"
+            ]
+        }
+
         args += self.buildParameters.toolchain.extraFlags.swiftCompilerFlags
         // User arguments (from -Xswiftc) should follow generated arguments to allow user overrides
         args += self.buildParameters.flags.swiftCompilerFlags

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -41,7 +41,7 @@ extension LLBuildManifestBuilder {
         let inputs = try self.computeSwiftCompileCmdInputs(target)
 
         // Outputs.
-        let objectNodes = try target.objects.map(Node.file)
+        let objectNodes = target.buildParameters.prepareForIndexing ? [] : try target.objects.map(Node.file)
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
@@ -395,7 +395,8 @@ extension LLBuildManifestBuilder {
             fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
             wholeModuleOptimization: target.buildParameters.configuration == .release,
-            outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.
+            outputFileMapPath: try target.writeOutputFileMap(), // FIXME: Eliminate side effect.
+            prepareForIndexing: target.buildParameters.prepareForIndexing
         )
     }
 
@@ -420,6 +421,8 @@ extension LLBuildManifestBuilder {
             self.manifest.addWriteEmbeddedResourcesCommand(resources: resourceFilesToEmbed, outputPath: resourcesEmbeddingSource)
         }
 
+        let prepareForIndexing = target.buildParameters.prepareForIndexing
+
         func addStaticTargetInputs(_ target: ResolvedModule) throws {
             // Ignore C Modules.
             if target.underlying is SystemLibraryTarget { return }
@@ -429,7 +432,7 @@ extension LLBuildManifestBuilder {
             if target.underlying is PluginTarget { return }
 
             // Depend on the binary for executable targets.
-            if target.type == .executable {
+            if target.type == .executable && !prepareForIndexing {
                 // FIXME: Optimize.
                 let product = try plan.graph.allProducts.first {
                     try $0.type == .executable && $0.executableTarget.id == target.id
@@ -447,8 +450,16 @@ extension LLBuildManifestBuilder {
             case .swift(let target)?:
                 inputs.append(file: target.moduleOutputPath)
             case .clang(let target)?:
-                for object in try target.objects {
-                    inputs.append(file: object)
+                if prepareForIndexing {
+                    // In preparation, we're only building swiftmodules
+                    // propagate the dependency to the header files in this target
+                    for header in target.clangTarget.headers {
+                        inputs.append(file: header)
+                    }
+                } else {
+                    for object in try target.objects {
+                        inputs.append(file: object)
+                    }
                 }
             case nil:
                 throw InternalError("unexpected: target \(target) not in target map \(self.plan.targetMap)")

Sources/Build/BuildManifest/LLBuildManifestBuilder.swift

@@ -127,6 +127,41 @@ public class LLBuildManifestBuilder {
         return self.manifest
     }
 
+    package func generatePrepareManifest(at path: AbsolutePath) throws -> LLBuildManifest {
+        self.swiftGetVersionFiles.removeAll()
+
+        self.manifest.createTarget(TargetKind.main.targetName)
+        self.manifest.createTarget(TargetKind.test.targetName)
+        self.manifest.defaultTarget = TargetKind.main.targetName
+
+        addPackageStructureCommand()
+
+        for (_, description) in self.plan.targetMap {
+            switch description {
+            case .swift(let desc):
+                try self.createSwiftCompileCommand(desc)
+            case .clang(let desc):
+                // Need the clang targets for tools
+                if desc.target.buildTriple == .tools {
+                    try self.createClangCompileCommand(desc)
+                }
+            }
+        }
+
+        for (_, description) in self.plan.productMap {
+            // Need to generate macro products
+            switch description.product.type {
+            case .macro, .plugin:
+                try self.createProductCommand(description)
+            default:
+                break
+            }
+        }
+
+        try LLBuildManifestWriter.write(self.manifest, at: path, fileSystem: self.fileSystem)
+        return self.manifest
+    }
+
     func addNode(_ node: Node, toTarget targetKind: TargetKind) {
         self.manifest.addNode(node, toTarget: targetKind.targetName)
     }

Sources/Build/BuildOperation.swift

@@ -847,7 +847,9 @@ extension BuildDescription {
     ) throws -> (BuildDescription, LLBuildManifest) {
         // Generate the llbuild manifest.
         let llbuild = LLBuildManifestBuilder(plan, disableSandboxForPluginCommands: disableSandboxForPluginCommands, fileSystem: fileSystem, observabilityScope: observabilityScope)
-        let buildManifest = try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
+        let buildManifest = plan.destinationBuildParameters.prepareForIndexing
+            ? try llbuild.generatePrepareManifest(at: plan.destinationBuildParameters.llbuildManifest)
+            : try llbuild.generateManifest(at: plan.destinationBuildParameters.llbuildManifest)
 
         let swiftCommands = llbuild.manifest.getCmdToolMap(kind: SwiftCompilerTool.self)
         let swiftFrontendCommands = llbuild.manifest.getCmdToolMap(kind: SwiftFrontendTool.self)

Sources/CoreCommands/Options.swift

@@ -438,6 +438,9 @@ public struct BuildOptions: ParsableArguments {
     @Flag(help: "Enable or disable indexing-while-building feature")
     public var indexStoreMode: StoreMode = .autoIndexStore
 
+    @Flag(name: .customLong("experimental-prepare-for-indexing"), help: .hidden)
+    var prepareForIndexing: Bool = false
+
     /// Whether to enable generation of `.swiftinterface`s alongside `.swiftmodule`s.
     @Flag(name: .customLong("enable-parseable-module-interfaces"))
     public var shouldEnableParseableModuleInterfaces: Bool = false

Sources/CoreCommands/SwiftCommandState.swift

@@ -726,7 +726,10 @@ public final class SwiftCommandState {
     when building on macOS.
     """
 
-    private func _buildParams(toolchain: UserToolchain) throws -> BuildParameters {
+    private func _buildParams(
+        toolchain: UserToolchain,
+        prepareForIndexing: Bool? = nil
+    ) throws -> BuildParameters {
         let triple = toolchain.targetTriple
 
         let dataPath = self.scratchDirectory.appending(
@@ -749,6 +752,7 @@ public final class SwiftCommandState {
             sanitizers: options.build.enabledSanitizers,
             indexStoreMode: options.build.indexStoreMode.buildParameter,
             isXcodeBuildSystemEnabled: options.build.buildSystem == .xcode,
+            prepareForIndexing: prepareForIndexing ?? options.build.prepareForIndexing,
             debuggingParameters: .init(
                 debugInfoFormat: options.build.debugInfoFormat.buildParameter,
                 triple: triple,
@@ -797,7 +801,7 @@ public final class SwiftCommandState {
 
     private lazy var _toolsBuildParameters: Result<BuildParameters, Swift.Error> = {
         Result(catching: {
-            try _buildParams(toolchain: self.getHostToolchain())
+            try _buildParams(toolchain: self.getHostToolchain(), prepareForIndexing: false)
         })
     }()
 

Sources/LLBuildManifest/LLBuildManifest.swift

@@ -408,7 +408,8 @@ public struct LLBuildManifest {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         assert(commands[name] == nil, "already had a command named '\(name)'")
         let tool = SwiftCompilerTool(
@@ -426,7 +427,8 @@ public struct LLBuildManifest {
             fileList: fileList,
             isLibrary: isLibrary,
             wholeModuleOptimization: wholeModuleOptimization,
-            outputFileMapPath: outputFileMapPath
+            outputFileMapPath: outputFileMapPath,
+            prepareForIndexing: prepareForIndexing
         )
         commands[name] = Command(name: name, tool: tool)
     }

Sources/LLBuildManifest/Tools.swift

@@ -271,6 +271,7 @@ public struct SwiftCompilerTool: ToolProtocol {
     public var isLibrary: Bool
     public var wholeModuleOptimization: Bool
     public var outputFileMapPath: AbsolutePath
+    public var prepareForIndexing: Bool
 
     init(
         inputs: [Node],
@@ -287,7 +288,8 @@ public struct SwiftCompilerTool: ToolProtocol {
         fileList: AbsolutePath,
         isLibrary: Bool,
         wholeModuleOptimization: Bool,
-        outputFileMapPath: AbsolutePath
+        outputFileMapPath: AbsolutePath,
+        prepareForIndexing: Bool
     ) {
         self.inputs = inputs
         self.outputs = outputs
@@ -304,6 +306,7 @@ public struct SwiftCompilerTool: ToolProtocol {
         self.isLibrary = isLibrary
         self.wholeModuleOptimization = wholeModuleOptimization
         self.outputFileMapPath = outputFileMapPath
+        self.prepareForIndexing = prepareForIndexing
     }
 
     var description: String {
@@ -334,7 +337,10 @@ public struct SwiftCompilerTool: ToolProtocol {
         } else {
             arguments += ["-incremental"]
         }
-        arguments += ["-c", "@\(self.fileList.pathString)"]
+        if !prepareForIndexing {
+            arguments += ["-c"]
+        }
+        arguments += ["@\(self.fileList.pathString)"]
         arguments += ["-I", importPath.pathString]
         arguments += otherArguments
         return arguments

Sources/SPMBuildCore/BuildParameters/BuildParameters+Debugging.swift

@@ -63,7 +63,7 @@ extension BuildParameters {
 
     /// The debugging strategy according to the current build parameters.
     public var debuggingStrategy: DebuggingStrategy? {
-        guard configuration == .debug else {
+        guard configuration == .debug, !prepareForIndexing else {
             return nil
         }
 

Sources/SPMBuildCore/BuildParameters/BuildParameters.swift

@@ -102,6 +102,9 @@ public struct BuildParameters: Encodable {
 
     public var shouldSkipBuilding: Bool
 
+    /// Do minimal build to prepare for indexing
+    public var prepareForIndexing: Bool
+
     /// Build parameters related to debugging.
     public var debuggingParameters: Debugging
 
@@ -131,6 +134,7 @@ public struct BuildParameters: Encodable {
         indexStoreMode: IndexStoreMode = .auto,
         isXcodeBuildSystemEnabled: Bool = false,
         shouldSkipBuilding: Bool = false,
+        prepareForIndexing: Bool = false,
         debuggingParameters: Debugging? = nil,
         driverParameters: Driver = .init(),
         linkingParameters: Linking = .init(),
@@ -185,6 +189,7 @@ public struct BuildParameters: Encodable {
         self.indexStoreMode = indexStoreMode
         self.isXcodeBuildSystemEnabled = isXcodeBuildSystemEnabled
         self.shouldSkipBuilding = shouldSkipBuilding
+        self.prepareForIndexing = prepareForIndexing
         self.driverParameters = driverParameters
         self.linkingParameters = linkingParameters
         self.outputParameters = outputParameters

Sources/SPMTestSupport/MockBuildTestHelper.swift

@@ -83,7 +83,8 @@ public func mockBuildParameters(
     useExplicitModuleBuild: Bool = false,
     linkerDeadStrip: Bool = true,
     linkTimeOptimizationMode: BuildParameters.LinkTimeOptimizationMode? = nil,
-    omitFramePointers: Bool? = nil
+    omitFramePointers: Bool? = nil,
+    prepareForIndexing: Bool = false
 ) -> BuildParameters {
     try! BuildParameters(
         dataPath: buildPath,
@@ -94,6 +95,7 @@ public func mockBuildParameters(
         pkgConfigDirectories: [],
         workers: 3,
         indexStoreMode: indexStoreMode,
+        prepareForIndexing: prepareForIndexing,
         debuggingParameters: .init(
             triple: targetTriple,
             shouldEnableDebuggingEntitlement: config == .debug,

Tests/BuildTests/PrepareForIndexTests.swift

@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2015-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Build
+import Foundation
+import LLBuildManifest
+@_spi(SwiftPMInternal)
+import SPMTestSupport
+import TSCBasic
+import XCTest
+
+class PrepareForIndexTests: XCTestCase {
+    func testPrepare() throws {
+        let (graph, fs, scope) = try macrosPackageGraph()
+
+        let plan = try BuildPlan(
+            destinationBuildParameters: mockBuildParameters(destination: .target, prepareForIndexing: true),
+            toolsBuildParameters: mockBuildParameters(destination: .host, prepareForIndexing: false),
+            graph: graph,
+            fileSystem: fs,
+            observabilityScope: scope
+        )
+
+        let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
+        let manifest = try builder.generatePrepareManifest(at: "/manifest")
+
+        // Make sure we're building the swift modules
+        let outputs = manifest.commands.flatMap(\.value.tool.outputs).map(\.name)
+        XCTAssertTrue(outputs.contains(where: { $0.hasSuffix(".swiftmodule") }))
+
+        // Ensure swiftmodules built with correct arguments
+        let coreCommands = manifest.commands.values.filter {
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Core.build/Core.swiftmodule")
+            })
+        }
+        XCTAssertEqual(coreCommands.count, 1)
+        let coreSwiftc = try XCTUnwrap(coreCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertTrue(coreSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Ensure tools are built normally
+        let toolCommands = manifest.commands.values.filter {
+            $0.tool.outputs.contains(where: {
+                $0.name.hasSuffix("debug/Modules-tool/SwiftSyntax.swiftmodule")
+            })
+        }
+        XCTAssertEqual(toolCommands.count, 1)
+        let toolSwiftc = try XCTUnwrap(toolCommands.first?.tool as? SwiftCompilerTool)
+        XCTAssertFalse(toolSwiftc.otherArguments.contains("-experimental-skip-all-function-bodies"))
+
+        // Make sure only object files for tools are built
+        XCTAssertTrue(
+            outputs.filter { $0.hasSuffix(".o") }.allSatisfy { $0.contains("-tool.build/") },
+            "outputs:\n\t\(outputs.filter { $0.hasSuffix(".o") }.joined(separator: "\n\t"))"
+        )
+    }
+}