Basics: migrate TSCUtility.SQLite to Basics (#5764)

This moves the SQLite API wrapper to SPM from tools-support-core.  There
is currently a single user of this API, which is SPM.  By migrating the
API to SPM we can deprecate the TSC interfaces to prevent new adoption.
It also reduces the dependencies on `TSCUtility` for SPM which works
towards the longer term goal of shedding the tools-support-core
dependency.

Author: compnerd

CMakeLists.txt

@@ -59,6 +59,7 @@ endif()
 
 find_package(dispatch QUIET)
 find_package(Foundation QUIET)
+find_package(SQLite3 REQUIRED)
 
 add_subdirectory(Sources)
 add_subdirectory(cmake/modules)

Package.swift

@@ -170,9 +170,12 @@ let package = Package(
 
         // MARK: SwiftPM specific support libraries
 
+        .systemLibrary(name: "SPMSQLite3", pkgConfig: "sqlite3"),
+
         .target(
             name: "Basics",
             dependencies: [
+                "SPMSQLite3",
                 .product(name: "OrderedCollections", package: "swift-collections"),
                 .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"),
                 .product(name: "SystemPackage", package: "swift-system"),

Sources/Basics/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(Basics
   Netrc.swift
   NSLock+Extensions.swift
   Observability.swift
+  SQLite.swift
   Sandbox.swift
   String+Extensions.swift
   Triple+Extensions.swift
@@ -38,7 +39,8 @@ target_link_libraries(Basics PUBLIC
   TSCBasic
   TSCUtility)
 target_link_libraries(Basics PRIVATE
-   TSCclibc)
+  SPMSQLite3
+  TSCclibc)
 # NOTE(compnerd) workaround for CMake not setting up include flags yet
 set_target_properties(Basics PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/Basics/SQLite.swift

@@ -0,0 +1,322 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Foundation
+
+import TSCBasic
+
+@_implementationOnly import SPMSQLite3
+
+/// A minimal SQLite wrapper.
+public struct SQLite {
+    /// The location of the database.
+    public let location: Location
+
+    /// The configuration for the database.
+    public let configuration: Configuration
+
+    /// Pointer to the database.
+    let db: OpaquePointer
+
+    /// Create or open the database at the given path.
+    ///
+    /// The database is opened in serialized mode.
+    public init(location: Location, configuration: Configuration = Configuration()) throws {
+        self.location = location
+        self.configuration = configuration
+
+        var handle: OpaquePointer?
+        try Self.checkError ({
+            sqlite3_open_v2(
+                location.pathString,
+                &handle,
+                SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX,
+                nil
+            )
+        },
+        description: "Unable to open database at \(self.location)")
+
+        guard let db = handle else {
+            throw StringError("Unable to open database at \(self.location)")
+        }
+        self.db = db
+        try Self.checkError({ sqlite3_extended_result_codes(db, 1) }, description: "Unable to configure database")
+        try Self.checkError({ sqlite3_busy_timeout(db, self.configuration.busyTimeoutMilliseconds) }, description: "Unable to configure database busy timeout")
+        if let maxPageCount = self.configuration.maxPageCount {
+            try self.exec(query: "PRAGMA max_page_count=\(maxPageCount);")
+        }
+    }
+
+    @available(*, deprecated, message: "use init(location:configuration) instead")
+    public init(dbPath: AbsolutePath) throws {
+        try self.init(location: .path(dbPath))
+    }
+
+    /// Prepare the given query.
+    public func prepare(query: String) throws -> PreparedStatement {
+        try PreparedStatement(db: self.db, query: query)
+    }
+
+    /// Directly execute the given query.
+    ///
+    /// Note: Use withCString for string arguments.
+    public func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws {
+        let query = withVaList(args) { ptr in
+            sqlite3_vmprintf(queryString, ptr)
+        }
+
+        let wcb = callback.map { CallbackWrapper($0) }
+        let callbackCtx = wcb.map { Unmanaged.passUnretained($0).toOpaque() }
+
+        var err: UnsafeMutablePointer<Int8>?
+        try Self.checkError { sqlite3_exec(db, query, sqlite_callback, callbackCtx, &err) }
+
+        sqlite3_free(query)
+
+        if let err = err {
+            let errorString = String(cString: err)
+            sqlite3_free(err)
+            throw StringError(errorString)
+        }
+    }
+
+    public func close() throws {
+        try Self.checkError { sqlite3_close(db) }
+    }
+
+    public typealias SQLiteExecCallback = ([Column]) -> Void
+
+    public struct Configuration {
+        public var busyTimeoutMilliseconds: Int32
+        public var maxSizeInBytes: Int?
+
+        // https://www.sqlite.org/pgszchng2016.html
+        private let defaultPageSizeInBytes = 1024
+
+        public init() {
+            self.busyTimeoutMilliseconds = 5000
+            self.maxSizeInBytes = .none
+        }
+
+        // FIXME: deprecated 12/2020, remove once clients migrated over
+        @available(*, deprecated, message: "use busyTimeout instead")
+        public var busyTimeoutSeconds: Int32 {
+            get {
+                self._busyTimeoutSeconds
+            } set {
+                self._busyTimeoutSeconds = newValue
+            }
+        }
+
+        // so tests dont warn
+        internal var _busyTimeoutSeconds: Int32 {
+            get {
+                return Int32(truncatingIfNeeded: Int(Double(self.busyTimeoutMilliseconds) / 1000))
+            } set {
+                self.busyTimeoutMilliseconds = newValue * 1000
+            }
+        }
+
+        public var maxSizeInMegabytes: Int? {
+            get {
+                self.maxSizeInBytes.map { $0 / (1024 * 1024) }
+            }
+            set {
+                self.maxSizeInBytes = newValue.map { $0 * 1024 * 1024 }
+            }
+        }
+
+        public var maxPageCount: Int? {
+            self.maxSizeInBytes.map { $0 / self.defaultPageSizeInBytes }
+        }
+    }
+
+    public enum Location {
+        case path(AbsolutePath)
+        case memory
+        case temporary
+
+        var pathString: String {
+            switch self {
+            case .path(let path):
+                return path.pathString
+            case .memory:
+                return ":memory:"
+            case .temporary:
+                return ""
+            }
+        }
+    }
+
+    /// Represents an sqlite value.
+    public enum SQLiteValue {
+        case null
+        case string(String)
+        case int(Int)
+        case blob(Data)
+    }
+
+    /// Represents a row returned by called step() on a prepared statement.
+    public struct Row {
+        /// The pointer to the prepared statment.
+        let stmt: OpaquePointer
+
+        /// Get integer at the given column index.
+        public func int(at index: Int32) -> Int {
+            Int(sqlite3_column_int64(self.stmt, index))
+        }
+
+        /// Get blob data at the given column index.
+        public func blob(at index: Int32) -> Data {
+            let bytes = sqlite3_column_blob(stmt, index)!
+            let count = sqlite3_column_bytes(stmt, index)
+            return Data(bytes: bytes, count: Int(count))
+        }
+
+        /// Get string at the given column index.
+        public func string(at index: Int32) -> String {
+            return String(cString: sqlite3_column_text(self.stmt, index))
+        }
+    }
+
+    public struct Column {
+        public var name: String
+        public var value: String
+    }
+
+    /// Represents a prepared statement.
+    public struct PreparedStatement {
+        typealias sqlite3_destructor_type = (@convention(c) (UnsafeMutableRawPointer?) -> Void)
+        static let SQLITE_STATIC = unsafeBitCast(0, to: sqlite3_destructor_type.self)
+        static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self)
+
+        /// The pointer to the prepared statment.
+        let stmt: OpaquePointer
+
+        public init(db: OpaquePointer, query: String) throws {
+            var stmt: OpaquePointer?
+            try SQLite.checkError { sqlite3_prepare_v2(db, query, -1, &stmt, nil) }
+            self.stmt = stmt!
+        }
+
+        /// Evaluate the prepared statement.
+        @discardableResult
+        public func step() throws -> Row? {
+            let result = sqlite3_step(stmt)
+
+            switch result {
+            case SQLITE_DONE:
+                return nil
+            case SQLITE_ROW:
+                return Row(stmt: self.stmt)
+            default:
+                throw StringError(String(cString: sqlite3_errstr(result)))
+            }
+        }
+
+        /// Bind the given arguments to the statement.
+        public func bind(_ arguments: [SQLiteValue]) throws {
+            for (idx, argument) in arguments.enumerated() {
+                let idx = Int32(idx) + 1
+                switch argument {
+                case .null:
+                    try checkError { sqlite3_bind_null(stmt, idx) }
+                case .int(let int):
+                    try checkError { sqlite3_bind_int64(stmt, idx, Int64(int)) }
+                case .string(let str):
+                    try checkError { sqlite3_bind_text(stmt, idx, str, -1, Self.SQLITE_TRANSIENT) }
+                case .blob(let blob):
+                    try checkError {
+                        blob.withUnsafeBytes { ptr in
+                            sqlite3_bind_blob(
+                                stmt,
+                                idx,
+                                ptr.baseAddress,
+                                Int32(blob.count),
+                                Self.SQLITE_TRANSIENT
+                            )
+                        }
+                    }
+                }
+            }
+        }
+
+        /// Reset the prepared statement.
+        public func reset() throws {
+            try SQLite.checkError { sqlite3_reset(stmt) }
+        }
+
+        /// Clear bindings from the prepared statment.
+        public func clearBindings() throws {
+            try SQLite.checkError { sqlite3_clear_bindings(stmt) }
+        }
+
+        /// Finalize the statement and free up resources.
+        public func finalize() throws {
+            try SQLite.checkError { sqlite3_finalize(stmt) }
+        }
+    }
+
+    fileprivate class CallbackWrapper {
+        var callback: SQLiteExecCallback
+        init(_ callback: @escaping SQLiteExecCallback) {
+            self.callback = callback
+        }
+    }
+
+    private static func checkError(_ fn: () -> Int32, description prefix: String? = .none) throws {
+        let result = fn()
+        if result != SQLITE_OK {
+            var description = String(cString: sqlite3_errstr(result))
+            switch description.lowercased() {
+            case "database or disk is full":
+                throw Errors.databaseFull
+            default:
+                if let prefix = prefix {
+                    description = "\(prefix): \(description)"
+                }
+                throw StringError(description)
+            }
+        }
+    }
+
+    public enum Errors: Error {
+        case databaseFull
+    }
+}
+
+private func sqlite_callback(
+    _ ctx: UnsafeMutableRawPointer?,
+    _ numColumns: Int32,
+    _ columns: UnsafeMutablePointer<UnsafeMutablePointer<Int8>?>?,
+    _ columnNames: UnsafeMutablePointer<UnsafeMutablePointer<Int8>?>?
+) -> Int32 {
+    guard let ctx = ctx else { return 0 }
+    guard let columnNames = columnNames, let columns = columns else { return 0 }
+    let numColumns = Int(numColumns)
+    var result: [SQLite.Column] = []
+
+    for idx in 0 ..< numColumns {
+        var name = ""
+        if let ptr = columnNames.advanced(by: idx).pointee {
+            name = String(cString: ptr)
+        }
+        var value = ""
+        if let ptr = columns.advanced(by: idx).pointee {
+            value = String(cString: ptr)
+        }
+        result.append(SQLite.Column(name: name, value: value))
+    }
+
+    let wcb = Unmanaged<SQLite.CallbackWrapper>.fromOpaque(ctx).takeUnretainedValue()
+    wcb.callback(result)
+
+    return 0
+}

Sources/Basics/SQLiteBackedCache.swift

@@ -13,7 +13,6 @@
 import Foundation
 
 import TSCBasic
-import struct TSCUtility.SQLite
 
 /// SQLite backed persistent cache.
 public final class SQLiteBackedCache<Value: Codable>: Closable {

Sources/CMakeLists.txt

@@ -6,6 +6,7 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
+add_subdirectory(SPMSQLite3)
 add_subdirectory(Basics)
 add_subdirectory(Build)
 add_subdirectory(Commands)

Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift

@@ -20,8 +20,6 @@ import struct Foundation.URL
 import PackageModel
 import TSCBasic
 
-import struct TSCUtility.SQLite
-
 final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable {
     private static let packageCollectionsTableName = "package_collections"
     private static let packagesFTSName = "fts_packages"