Add basic QueryEngine scaffolding w/ Encodable hashing (#7347)

### Motivation:

Caching of computations in SwiftPM is ad-hoc and mostly relies on
in-memory `ThreadSafeKeyValueStore`, which utilizes locking and makes it
harder to add `Sendable` conformances on types that have these stores as
properties. To allow consistent and persistent caching when SwiftPM
processes are relaunched, we can use a SQLite-backed async-first caching
engine.

### Modifications:

This change ports [most of the current `GeneratorEngine`
implementation](https://github.com/apple/swift-sdk-generator/blob/main/Sources/GeneratorEngine/Engine.swift)
from the Swift SDK Generator repository to the SwiftPM code base as
`QueryEngine`. Since SwiftNIO is not supported on Windows, references to
`AsyncHTTPClient` have been removed. Additionally, we can't use macros
in the SwiftPM code base either, thus the `Query` protocol has to
conform to `Encodable` instead of using macro-generated conformances. We
don't have a consistent hashing implementation for `Encodable` yet, and
a temporary stub for it is marked with `fatalError` for now.

### Result:

NFC, new `QueryEngine` module is not used anywhere yet.

Author: MaxDesiatov

Package.swift

@@ -12,26 +12,34 @@
 //
 //===----------------------------------------------------------------------===//
 
-import PackageDescription
 import class Foundation.ProcessInfo
+import PackageDescription
 
 // When building the toolchain on the CI for ELF platforms, remove the CI's
 // stdlib absolute runpath and add ELF's $ORIGIN relative paths before installing.
-let swiftpmLinkSettings : [LinkerSetting]
-let packageLibraryLinkSettings : [LinkerSetting]
+let swiftpmLinkSettings: [LinkerSetting]
+let packageLibraryLinkSettings: [LinkerSetting]
 if let resourceDirPath = ProcessInfo.processInfo.environment["SWIFTCI_INSTALL_RPATH_OS"] {
-  swiftpmLinkSettings = [ .unsafeFlags(["-no-toolchain-stdlib-rpath", "-Xlinker", "-rpath", "-Xlinker", "$ORIGIN/../lib/swift/\(resourceDirPath)"]) ]
-  packageLibraryLinkSettings = [ .unsafeFlags(["-no-toolchain-stdlib-rpath", "-Xlinker", "-rpath", "-Xlinker", "$ORIGIN/../../\(resourceDirPath)"]) ]
+    swiftpmLinkSettings = [.unsafeFlags([
+        "-no-toolchain-stdlib-rpath",
+        "-Xlinker", "-rpath",
+        "-Xlinker", "$ORIGIN/../lib/swift/\(resourceDirPath)",
+    ])]
+    packageLibraryLinkSettings = [.unsafeFlags([
+        "-no-toolchain-stdlib-rpath",
+        "-Xlinker", "-rpath",
+        "-Xlinker", "$ORIGIN/../../\(resourceDirPath)",
+    ])]
 } else {
-  swiftpmLinkSettings = []
-  packageLibraryLinkSettings = []
+    swiftpmLinkSettings = []
+    packageLibraryLinkSettings = []
 }
 
 /** SwiftPMDataModel is the subset of SwiftPM product that includes just its data model.
-This allows some clients (such as IDEs) that use SwiftPM's data model but not its build system
-to not have to depend on SwiftDriver, SwiftLLBuild, etc. We should probably have better names here,
-though that could break some clients.
-*/
+ This allows some clients (such as IDEs) that use SwiftPM's data model but not its build system
+ to not have to depend on SwiftDriver, SwiftLLBuild, etc. We should probably have better names here,
+ though that could break some clients.
+ */
 let swiftPMDataModelProduct = (
     name: "SwiftPMDataModel",
     targets: [
@@ -51,7 +59,7 @@ let swiftPMDataModelProduct = (
  command line tools, while `libSwiftPMDataModel` includes only the data model.
 
  NOTE: This API is *unstable* and may change at any time.
-*/
+ */
 let swiftPMProduct = (
     name: "SwiftPM",
     targets: swiftPMDataModelProduct.targets + [
@@ -69,8 +77,8 @@ let systemSQLitePkgConfig: String? = "sqlite3"
 #endif
 
 /** An array of products which have two versions listed: one dynamically linked, the other with the
-automatic linking type with `-auto` suffix appended to product's name.
-*/
+ automatic linking type with `-auto` suffix appended to product's name.
+ */
 let autoProducts = [swiftPMProduct, swiftPMDataModelProduct]
 
 
@@ -90,11 +98,11 @@ let package = Package(
     name: "SwiftPM",
     platforms: [
         .macOS(.v13),
-        .iOS(.v16)
+        .iOS(.v16),
     ],
     products:
-        autoProducts.flatMap {
-          [
+    autoProducts.flatMap {
+        [
             .library(
                 name: $0.name,
                 type: .dynamic,
@@ -103,9 +111,9 @@ let package = Package(
             .library(
                 name: "\($0.name)-auto",
                 targets: $0.targets
-            )
-          ]
-        } + [
+            ),
+        ]
+    } + [
         .library(
             name: "XCBuildSupport",
             targets: ["XCBuildSupport"]
@@ -166,7 +174,7 @@ let package = Package(
             name: "SourceKitLSPAPI",
             dependencies: [
                 "Build",
-                "SPMBuildCore"
+                "SPMBuildCore",
             ],
             exclude: ["CMakeLists.txt"],
             swiftSettings: [.enableExperimentalFeature("AccessLevelOnImport")]
@@ -213,7 +221,7 @@ let package = Package(
             name: "SourceControl",
             dependencies: [
                 "Basics",
-                "PackageModel"
+                "PackageModel",
             ],
             exclude: ["CMakeLists.txt"]
         ),
@@ -255,7 +263,7 @@ let package = Package(
             dependencies: [
                 "Basics",
                 "PackageLoading",
-                "PackageModel"
+                "PackageModel",
             ],
             exclude: ["CMakeLists.txt", "README.md"]
         ),
@@ -267,7 +275,7 @@ let package = Package(
             name: "PackageCollectionsModel",
             dependencies: [],
             exclude: [
-                "Formats/v1.md"
+                "Formats/v1.md",
             ]
         ),
 
@@ -301,7 +309,7 @@ let package = Package(
             ],
             exclude: ["CMakeLists.txt"]
         ),
-        
+
         .target(
             name: "PackageSigning",
             dependencies: [
@@ -320,7 +328,7 @@ let package = Package(
             name: "SPMBuildCore",
             dependencies: [
                 "Basics",
-                "PackageGraph"
+                "PackageGraph",
             ],
             exclude: ["CMakeLists.txt"]
         ),
@@ -460,6 +468,14 @@ let package = Package(
             ]
         ),
 
+        .target(
+            name: "QueryEngine",
+            dependencies: [
+                "Basics",
+                .product(name: "Crypto", package: "swift-crypto"),
+            ]
+        ),
+
         .executableTarget(
             /** The main executable provided by SwiftPM */
             name: "swift-package",
@@ -562,7 +578,8 @@ let package = Package(
         .target(
             /** Test for thread-santizer. */
             name: "tsan_utils",
-            dependencies: []),
+            dependencies: []
+        ),
 
         // MARK: SwiftPM tests
 
@@ -664,7 +681,7 @@ let package = Package(
             name: "XCBuildSupportTests",
             dependencies: ["XCBuildSupport", "SPMTestSupport"],
             exclude: ["Inputs/Foo.pc"]
-        )
+        ),
     ],
     swiftLanguageVersions: [.v5]
 )
@@ -682,15 +699,16 @@ package.targets.append(contentsOf: [
     ),
 ])
 
-// rdar://101868275 "error: cannot find 'XCTAssertEqual' in scope" can affect almost any functional test, so we flat out disable them all until we know what is going on
+// rdar://101868275 "error: cannot find 'XCTAssertEqual' in scope" can affect almost any functional test, so we flat out
+// disable them all until we know what is going on
 if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] == nil {
     package.targets.append(contentsOf: [
         .testTarget(
             name: "FunctionalTests",
             dependencies: [
                 "swift-package-manager",
                 "PackageModel",
-                "SPMTestSupport"
+                "SPMTestSupport",
             ]
         ),
 

Sources/Basics/Concurrency/ThrowingDefer.swift

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// Runs a cleanup closure (`deferred`) after a given `work` closure,
+/// making sure `deferred` is run also when `work` throws an error.
+/// - Parameters:
+///   - work: The work that should be performed. Will always be executed.
+///   - deferred: The cleanup that needs to be done in any case.
+/// - Throws: Any error thrown by `deferred` or `work` (in that order).
+/// - Returns: The result of `work`.
+/// - Note: If `work` **and** `deferred` throw an error,
+///         the one thrown by `deferred` is thrown from this function.
+/// - SeeAlso: ``withAsyncThrowing(do:defer:)``
+public func withThrowing<T>(
+    do work: () throws -> T,
+    defer deferred: () throws -> Void
+) throws -> T {
+    do {
+        let result = try work()
+        try deferred()
+        return result
+    } catch {
+        try deferred()
+        throw error
+    }
+}
+
+/// Runs an async cleanup closure (`deferred`) after a given async `work` closure,
+/// making sure `deferred` is run also when `work` throws an error.
+/// - Parameters:
+///   - work: The work that should be performed. Will always be executed.
+///   - deferred: The cleanup that needs to be done in any case.
+/// - Throws: Any error thrown by `deferred` or `work` (in that order).
+/// - Returns: The result of `work`.
+/// - Note: If `work` **and** `deferred` throw an error,
+///         the one thrown by `deferred` is thrown from this function.
+/// - SeeAlso: ``withThrowing(do:defer:)``
+public func withAsyncThrowing<T: Sendable>(
+    do work: @Sendable () async throws -> T,
+    defer deferred: @Sendable () async throws -> Void
+) async throws -> T {
+    do {
+        let result = try await work()
+        try await deferred()
+        return result
+    } catch {
+        try await deferred()
+        throw error
+    }
+}

Sources/Basics/SQLiteBackedCache.swift

@@ -84,8 +84,8 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
-    public func put(
-        key: Key,
+    private func put(
+        rawKey key: SQLite.SQLiteValue,
         value: Value,
         replace: Bool = false,
         observabilityScope: ObservabilityScope? = nil
@@ -95,7 +95,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
             try self.executeStatement(query) { statement in
                 let data = try self.jsonEncoder.encode(value)
                 let bindings: [SQLite.SQLiteValue] = [
-                    .string(key),
+                    key,
                     .blob(data),
                 ]
                 try statement.bind(bindings)
@@ -107,17 +107,39 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
             }
             observabilityScope?
                 .emit(
-                    warning: "truncating \(self.tableName) cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes"
+                    warning: """
+                    truncating \(self.tableName) cache database since it reached max size of \(
+                        self.configuration.maxSizeInBytes ?? 0
+                    ) bytes
+                    """
                 )
             try self.executeStatement("DELETE FROM \(self.tableName);") { statement in
                 try statement.step()
             }
-            try self.put(key: key, value: value, replace: replace, observabilityScope: observabilityScope)
+            try self.put(rawKey: key, value: value, replace: replace, observabilityScope: observabilityScope)
         } catch {
             throw error
         }
     }
 
+    public func put(
+        blobKey key: some Sequence<UInt8>,
+        value: Value,
+        replace: Bool = false,
+        observabilityScope: ObservabilityScope? = nil
+    ) throws {
+        try self.put(rawKey: .blob(Data(key)), value: value, observabilityScope: observabilityScope)
+    }
+
+    public func put(
+        key: Key,
+        value: Value,
+        replace: Bool = false,
+        observabilityScope: ObservabilityScope? = nil
+    ) throws {
+        try self.put(rawKey: .string(key), value: value, replace: replace, observabilityScope: observabilityScope)
+    }
+
     public func get(key: Key) throws -> Value? {
         let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;"
         return try self.executeStatement(query) { statement -> Value? in
@@ -129,6 +151,17 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
         }
     }
 
+    public func get(blobKey key: some Sequence<UInt8>) throws -> Value? {
+        let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;"
+        return try self.executeStatement(query) { statement -> Value? in
+            try statement.bind([.blob(Data(key))])
+            let data = try statement.step()?.blob(at: 0)
+            return try data.flatMap {
+                try self.jsonDecoder.decode(Value.self, from: $0)
+            }
+        }
+    }
+
     public func remove(key: Key) throws {
         let query = "DELETE FROM \(self.tableName) WHERE key = ?;"
         try self.executeStatement(query) { statement in
@@ -143,7 +176,7 @@ public final class SQLiteBackedCache<Value: Codable>: Closable {
             let result: Result<T, Error>
             let statement = try db.prepare(query: query)
             do {
-                result = .success(try body(statement))
+                result = try .success(body(statement))
             } catch {
                 result = .failure(error)
             }

Sources/QueryEngine/FileSystem/AsyncFileSystem.swift

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import protocol _Concurrency.Actor
+import protocol Crypto.HashFunction
+import struct SystemPackage.Errno
+import struct SystemPackage.FilePath
+
+public protocol AsyncFileSystem: Actor {
+    func withOpenReadableFile<T>(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T
+    func withOpenWritableFile<T>(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T
+}
+
+enum FileSystemError: Error {
+    case fileDoesNotExist(FilePath)
+    case bufferLimitExceeded(FilePath)
+    case systemError(FilePath, Errno)
+}
+
+extension Error {
+    func attach(path: FilePath) -> any Error {
+        if let error = self as? Errno {
+            FileSystemError.systemError(path, error)
+        } else {
+            self
+        }
+    }
+}