[Build] Add support for indexing-while-building

This add supports for indexing while building which will be useful for
LSP users. The indexing-while-building feature will be enabled by
default when building in debug mode but can be disabled by passing the
flag --disable-index-store. The feature is never enabled in release.

This feature is supported by Swift compiler on all platforms but clang
has some nuances. We can expect this feature to be available on Apple's
clang but not OSS or other distributions of clang. So, the feature is
selectively enabled for clang targets.

<rdar://problem/45689037> Add support for indexing-while-building

Author: aciidgh

Sources/Build/BuildPlan.swift

@@ -17,6 +17,16 @@ import func POSIX.getenv
 
 public struct BuildParameters {
 
+    /// Mode for the indexing-while-building feature.
+    public enum IndexStoreMode: Equatable {
+        /// Index store should be enabled.
+        case on
+        /// Index store should be disabled.
+        case off
+        /// Index store should be enabled in debug configuration.
+        case auto
+    }
+
     // FIXME: Error handling.
     //
     /// Path to the module cache directory to use for SwiftPM's own tests.
@@ -46,6 +56,12 @@ public struct BuildParameters {
         return dataPath.appending(component: configuration.dirname)
     }
 
+    /// The path to the index store directory.
+    public var indexStore: AbsolutePath {
+        assert(indexStoreMode != .off, "index store is disabled")
+        return buildPath.appending(components: "index", "store")
+    }
+
     /// The path to the code coverage directory.
     public var codeCovPath: AbsolutePath {
         return buildPath.appending(component: "codecov")
@@ -90,6 +106,9 @@ public struct BuildParameters {
     /// If should enable llbuild manifest caching.
     public let shouldEnableManifestCaching: Bool
 
+    /// The mode to use for indexing-while-building feature.
+    public let indexStoreMode: IndexStoreMode
+
     /// Whether to enable code coverage.
     public let enableCodeCoverage: Bool
 
@@ -119,7 +138,8 @@ public struct BuildParameters {
         shouldLinkStaticSwiftStdlib: Bool = false,
         shouldEnableManifestCaching: Bool = false,
         sanitizers: EnabledSanitizers = EnabledSanitizers(),
-        enableCodeCoverage: Bool = false
+        enableCodeCoverage: Bool = false,
+        indexStoreMode: IndexStoreMode = .auto
     ) {
         self.dataPath = dataPath
         self.configuration = configuration
@@ -131,6 +151,25 @@ public struct BuildParameters {
         self.shouldEnableManifestCaching = shouldEnableManifestCaching
         self.sanitizers = sanitizers
         self.enableCodeCoverage = enableCodeCoverage
+        self.indexStoreMode = indexStoreMode
+    }
+
+    /// Returns the compiler arguments for the index store, if enabled.
+    fileprivate var indexStoreArguments: [String] {
+        let addIndexStoreArguments: Bool
+        switch indexStoreMode {
+        case .on:
+            addIndexStoreArguments = true
+        case .off:
+            addIndexStoreArguments = false
+        case .auto:
+            addIndexStoreArguments = configuration == .debug 
+        }
+
+        if addIndexStoreArguments {
+            return ["-index-store-path", indexStore.asString]
+        }
+        return []
     }
 }
 
@@ -229,6 +268,16 @@ public final class ClangTargetBuildDescription {
         }
         args += ["-fblocks"]
 
+        // Enable index store, if appropriate.
+        //
+        // This feature is not widely available in OSS clang. So, we only enable
+        // index store for Apple's clang or if explicitly asked to. 
+        if Process.env.keys.contains("SWIFTPM_ENABLE_CLANG_INDEX_STORE") {
+            args += buildParameters.indexStoreArguments
+        } else if buildParameters.triple.isDarwin(), (try? buildParameters.toolchain._isClangCompilerVendorApple()) == true {
+            args += buildParameters.indexStoreArguments
+        }
+
         if !buildParameters.triple.isWindows() {
             // Using modules currently conflicts with the Windows SDKs.
             args += ["-fmodules", "-fmodule-name=" + target.c99name]
@@ -361,6 +410,7 @@ public final class SwiftTargetBuildDescription {
         case .release: break
         }
 
+        args += buildParameters.indexStoreArguments
         args += buildParameters.toolchain.extraSwiftCFlags
         args += optimizationArguments
         args += ["-j\(SwiftCompilerTool.numThreads)"]

Sources/Build/Toolchain.swift

@@ -18,6 +18,11 @@ public protocol Toolchain {
     /// Path of the `clang` compiler.
     func getClangCompiler() throws -> AbsolutePath
 
+    // FIXME: This is a temporary API until index store is widely available in
+    // the OSS clang compiler. This API should not used for any other purpose.
+    /// Returns true if clang compiler's vendor is Apple and nil if unknown.
+    func _isClangCompilerVendorApple() throws -> Bool?
+
     /// Additional flags to be passed to the C compiler.
     var extraCCFlags: [String] { get }
 
@@ -30,3 +35,9 @@ public protocol Toolchain {
     /// The dynamic library extension, for e.g. dylib, so.
     var dynamicLibraryExtension: String { get }
 }
+
+extension Toolchain {
+    public func _isClangCompilerVendorApple() throws -> Bool? {
+        return nil
+    }
+}

Sources/Commands/Options.swift

@@ -64,5 +64,8 @@ public class ToolOptions {
     /// Use Package.resolved file for resolving dependencies.
     public var forceResolvedVersions = false
 
+    /// The mode to use for indexing-while-building feature.
+    public var indexStoreMode: BuildParameters.IndexStoreMode = .auto
+
     public required init() {}
 }

Sources/Commands/SwiftTool.swift

@@ -356,6 +356,16 @@ public class SwiftTool<Options: ToolOptions> {
                 usage: "Force resolve to versions recorded in the Package.resolved file"),
             to: { $0.forceResolvedVersions = $1 })
 
+        binder.bind(
+            option: parser.add(option: "--enable-index-store", kind: Bool.self,
+                usage: "Enable indexing-while-building feature"),
+            to: { if $1 { $0.indexStoreMode = .on } })
+
+        binder.bind(
+            option: parser.add(option: "--disable-index-store", kind: Bool.self,
+                usage: "Disable indexing-while-building feature"),
+            to: { if $1 { $0.indexStoreMode = .off } })
+
         // Let subclasses bind arguments.
         type(of: self).defineArguments(parser: parser, binder: binder)
 
@@ -694,7 +704,8 @@ public class SwiftTool<Options: ToolOptions> {
                 flags: options.buildFlags,
                 shouldLinkStaticSwiftStdlib: options.shouldLinkStaticSwiftStdlib,
                 sanitizers: options.sanitizers,
-                enableCodeCoverage: options.shouldEnableCodeCoverage
+                enableCodeCoverage: options.shouldEnableCodeCoverage,
+                indexStoreMode: options.indexStoreMode
             )
         })
     }()

Sources/Workspace/UserToolchain.swift

@@ -160,10 +160,23 @@ public final class UserToolchain: Toolchain {
         }
         let toolPath = try AbsolutePath(validating: foundPath)
 
+        // If we found clang using xcrun, assume the vendor is Apple.
+        // FIXME: This might not be the best way to determine this.
+        #if os(macOS)
+            __isClangCompilerVendorApple = true
+        #endif
+
         _clangCompiler = toolPath
         return toolPath
     }
     private var _clangCompiler: AbsolutePath?
+    private var __isClangCompilerVendorApple: Bool?
+
+    public func _isClangCompilerVendorApple() throws -> Bool? {
+        // The boolean gets computed as a side-effect of lookup for clang compiler.
+        _ = try getClangCompiler()
+        return __isClangCompilerVendorApple
+    }
 
     /// Returns the path to llvm-cov tool.
     public func getLLVMCov() throws -> AbsolutePath {

Tests/BuildTests/BuildPlanTests.swift

@@ -34,6 +34,14 @@ private struct MockToolchain: Toolchain {
     func getClangCompiler() throws -> AbsolutePath {
         return AbsolutePath("/fake/path/to/clang")
     }
+
+    func _isClangCompilerVendorApple() throws -> Bool? {
+      #if os(macOS)
+        return true
+      #else
+        return false
+      #endif
+    }
 }
 
 final class BuildPlanTests: XCTestCase {
@@ -47,15 +55,18 @@ final class BuildPlanTests: XCTestCase {
         buildPath: AbsolutePath = AbsolutePath("/path/to/build"),
         config: Build.Configuration = .debug,
         shouldLinkStaticSwiftStdlib: Bool = false,
-        destinationTriple: Triple = Triple.hostTriple
+        destinationTriple: Triple = Triple.hostTriple,
+        indexStoreMode: BuildParameters.IndexStoreMode = .off
     ) -> BuildParameters {
         return BuildParameters(
             dataPath: buildPath,
             configuration: config,
             toolchain: MockToolchain(),
             destinationTriple: destinationTriple,
             flags: BuildFlags(),
-            shouldLinkStaticSwiftStdlib: shouldLinkStaticSwiftStdlib)
+            shouldLinkStaticSwiftStdlib: shouldLinkStaticSwiftStdlib,
+            indexStoreMode: indexStoreMode
+        )
     }
 
     func testBasicSwiftPackage() throws {
@@ -991,6 +1002,49 @@ final class BuildPlanTests: XCTestCase {
             "@/path/to/build/debug/exe.product/Objects.LinkFileList",
             ])
     }
+
+    func testIndexStore() throws {
+        let fs = InMemoryFileSystem(emptyFiles:
+            "/Pkg/Sources/exe/main.swift",
+            "/Pkg/Sources/lib/lib.c",
+            "/Pkg/Sources/lib/include/lib.h"
+        )
+
+        let diagnostics = DiagnosticsEngine()
+        let graph = loadPackageGraph(root: "/Pkg", fs: fs, diagnostics: diagnostics,
+            manifests: [
+                Manifest.createV4Manifest(
+                    name: "Pkg",
+                    path: "/Pkg",
+                    url: "/Pkg",
+                    targets: [
+                        TargetDescription(name: "exe", dependencies: ["lib"]),
+                        TargetDescription(name: "lib", dependencies: []),
+                    ]),
+            ]
+        )
+        XCTAssertNoDiagnostics(diagnostics)
+
+        func check(for mode: BuildParameters.IndexStoreMode, config: Build.Configuration) throws {
+            let result = BuildPlanResult(plan: try BuildPlan(buildParameters: mockBuildParameters(config: config, indexStoreMode: mode), graph: graph, diagnostics: diagnostics, fileSystem: fs))
+
+            let lib = try result.target(for: "lib").clangTarget()
+            let path = StringPattern.equal(result.plan.buildParameters.indexStore.asString)
+
+        #if os(macOS)
+            XCTAssertMatch(lib.basicArguments(), [.anySequence, "-index-store-path", path, .anySequence])
+        #else
+            XCTAssertNoMatch(lib.basicArguments(), [.anySequence, "-index-store-path", path, .anySequence])
+        #endif
+
+            let exe = try result.target(for: "exe").swiftTarget().compileArguments()
+            XCTAssertMatch(exe, [.anySequence, "-index-store-path", path, .anySequence])
+        }
+
+        try check(for: .auto, config: .debug)
+        try check(for: .on, config: .debug)
+        try check(for: .on, config: .release)
+    }
 }
 
 // MARK:- Test Helpers

Tests/BuildTests/XCTestManifests.swift

@@ -16,6 +16,7 @@ extension BuildPlanTests {
         ("testCppModule", testCppModule),
         ("testDynamicProducts", testDynamicProducts),
         ("testExecAsDependency", testExecAsDependency),
+        ("testIndexStore", testIndexStore),
         ("testNonReachableProductsAndTargets", testNonReachableProductsAndTargets),
         ("testPkgConfigGenericDiagnostic", testPkgConfigGenericDiagnostic),
         ("testPkgConfigHintDiagnostic", testPkgConfigHintDiagnostic),