Package fingerprint storage

Package.swift

@@ -258,6 +258,15 @@ let package = Package(
             ],
             exclude: ["CMakeLists.txt"]
         ),
+
+        .target(
+            name: "PackageFingerprint",
+            dependencies: [
+                "Basics",
+                "PackageModel",
+            ],
+            exclude: ["CMakeLists.txt"]
+        ),
 
         // MARK: Package Manager Functionality
 
@@ -484,6 +493,10 @@ let package = Package(
             name: "PackageCollectionsTests",
             dependencies: ["PackageCollections", "SPMTestSupport"]
         ),
+        .testTarget(
+            name: "PackageFingerprintTests",
+            dependencies: ["PackageFingerprint", "SPMTestSupport"]
+        ),
         .testTarget(
             name: "PackageRegistryTests",
             dependencies: ["SPMTestSupport", "PackageRegistry"]

Sources/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(PackageCollectionsModel)
 add_subdirectory(PackageCollectionsSigning)
 add_subdirectory(PackageCollectionsSigningLibc)
 add_subdirectory(PackageDescription)
+add_subdirectory(PackageFingerprint)
 add_subdirectory(PackageGraph)
 add_subdirectory(PackageLoading)
 add_subdirectory(PackageModel)

Sources/PackageFingerprint/CMakeLists.txt

@@ -0,0 +1,28 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+add_library(PackageFingerprint
+  FilePackageFingerprintStorage.swift
+  Model.swift
+  PackageFingerprintStorage.swift)
+target_link_libraries(PackageFingerprint PUBLIC
+  Basics
+  PackageModel
+  TSCBasic
+  TSCUtility)
+# NOTE(compnerd) workaround for CMake not setting up include flags yet
+set_target_properties(PackageFingerprint PROPERTIES
+  INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
+
+if(USE_CMAKE_INSTALL)
+install(TARGETS PackageFingerprint
+  ARCHIVE DESTINATION lib
+  LIBRARY DESTINATION lib
+  RUNTIME DESTINATION bin)
+endif()
+set_property(GLOBAL APPEND PROPERTY SwiftPM_EXPORTS PackageFingerprint)

Sources/PackageFingerprint/FilePackageFingerprintStorage.swift

@@ -0,0 +1,190 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Basics
+import Dispatch
+import Foundation
+import PackageModel
+import TSCBasic
+import TSCUtility
+
+public struct FilePackageFingerprintStorage: PackageFingerprintStorage {
+    let fileSystem: FileSystem
+    let directoryPath: AbsolutePath
+
+    private let encoder: JSONEncoder
+    private let decoder: JSONDecoder
+
+    init(fileSystem: FileSystem, directoryPath: AbsolutePath) {
+        self.fileSystem = fileSystem
+        self.directoryPath = directoryPath
+
+        self.encoder = JSONEncoder.makeWithDefaults()
+        self.decoder = JSONDecoder.makeWithDefaults()
+    }
+
+    public func get(package: PackageIdentity,
+                    version: Version,
+                    observabilityScope: ObservabilityScope,
+                    callbackQueue: DispatchQueue,
+                    callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void)
+    {
+        let callback = self.makeAsync(callback, on: callbackQueue)
+
+        do {
+            let packageFingerprints = try self.withLock {
+                try self.loadFromDisk(package: package)
+            }
+
+            guard let fingerprints = packageFingerprints[version] else {
+                throw PackageFingerprintStorageError.notFound
+            }
+
+            callback(.success(fingerprints))
+        } catch {
+            callback(.failure(error))
+        }
+    }
+
+    public func put(package: PackageIdentity,
+                    version: Version,
+                    fingerprint: Fingerprint,
+                    observabilityScope: ObservabilityScope,
+                    callbackQueue: DispatchQueue,
+                    callback: @escaping (Result<Void, Error>) -> Void)
+    {
+        let callback = self.makeAsync(callback, on: callbackQueue)
+
+        do {
+            try self.withLock {
+                var packageFingerprints = try self.loadFromDisk(package: package)
+
+                if let existing = packageFingerprints[version]?[fingerprint.origin.kind] {
+                    // Error if we try to write a different fingerprint
+                    guard fingerprint == existing else {
+                        throw PackageFingerprintStorageError.conflict(given: fingerprint, existing: existing)
+                    }
+                    // Don't need to do anything if fingerprints are the same
+                    return
+                }
+
+                var fingerprints = packageFingerprints.removeValue(forKey: version) ?? [:]
+                fingerprints[fingerprint.origin.kind] = fingerprint
+                packageFingerprints[version] = fingerprints
+
+                try self.saveToDisk(package: package, fingerprints: packageFingerprints)
+            }
+            callback(.success(()))
+        } catch {
+            callback(.failure(error))
+        }
+    }
+
+    private func loadFromDisk(package: PackageIdentity) throws -> PackageFingerprints {
+        let path = self.directoryPath.appending(component: package.fingerprintFilename)
+
+        guard self.fileSystem.exists(path) else {
+            return .init()
+        }
+
+        let buffer = try fileSystem.readFileContents(path).contents
+        guard buffer.count > 0 else {
+            return .init()
+        }
+
+        let container = try self.decoder.decode(StorageModel.Container.self, from: Data(buffer))
+        return try container.packageFingerprints()
+    }
+
+    private func saveToDisk(package: PackageIdentity, fingerprints: PackageFingerprints) throws {
+        if !self.fileSystem.exists(self.directoryPath) {
+            try self.fileSystem.createDirectory(self.directoryPath, recursive: true)
+        }
+
+        let container = StorageModel.Container(fingerprints)
+        let buffer = try encoder.encode(container)
+
+        let path = self.directoryPath.appending(component: package.fingerprintFilename)
+        try self.fileSystem.writeFileContents(path, bytes: ByteString(buffer))
+    }
+
+    private func withLock<T>(_ body: () throws -> T) throws -> T {
+        if !self.fileSystem.exists(self.directoryPath) {
+            try self.fileSystem.createDirectory(self.directoryPath, recursive: true)
+        }
+        return try self.fileSystem.withLock(on: self.directoryPath, type: .exclusive, body)
+    }
+
+    private func makeAsync<T>(_ closure: @escaping (Result<T, Error>) -> Void, on queue: DispatchQueue) -> (Result<T, Error>) -> Void {
+        { result in queue.async { closure(result) } }
+    }
+}
+
+private enum StorageModel {
+    struct Container: Codable {
+        let versionFingerprints: [String: [String: StoredFingerprint]]
+
+        init(_ versionFingerprints: PackageFingerprints) {
+            self.versionFingerprints = Dictionary(uniqueKeysWithValues: versionFingerprints.map { version, fingerprints in
+                let fingerprintByKind: [String: StoredFingerprint] = Dictionary(uniqueKeysWithValues: fingerprints.map { kind, fingerprint in
+                    let origin: String
+                    switch fingerprint.origin {
+                    case .sourceControl(let url):
+                        origin = url.absoluteString
+                    case .registry(let url):
+                        origin = url.absoluteString
+                    }
+                    return (kind.rawValue, StoredFingerprint(origin: origin, fingerprint: fingerprint.value))
+                })
+                return (version.description, fingerprintByKind)
+            })
+        }
+
+        func packageFingerprints() throws -> PackageFingerprints {
+            try Dictionary(uniqueKeysWithValues: self.versionFingerprints.map { version, fingerprints in
+                let fingerprintByKind: [Fingerprint.Kind: Fingerprint] = try Dictionary(uniqueKeysWithValues: fingerprints.map { kind, fingerprint in
+                    guard let kind = Fingerprint.Kind(rawValue: kind) else {
+                        throw SerializationError.unknownKind(kind)
+                    }
+                    guard let originURL = Foundation.URL(string: fingerprint.origin) else {
+                        throw SerializationError.invalidURL(fingerprint.origin)
+                    }
+
+                    let origin: Fingerprint.Origin
+                    switch kind {
+                    case .sourceControl:
+                        origin = .sourceControl(originURL)
+                    case .registry:
+                        origin = .registry(originURL)
+                    }
+
+                    return (kind, Fingerprint(origin: origin, value: fingerprint.fingerprint))
+                })
+                return (Version(stringLiteral: version), fingerprintByKind)
+            })
+        }
+    }
+
+    struct StoredFingerprint: Codable {
+        let origin: String
+        let fingerprint: String
+    }
+}
+
+extension PackageIdentity {
+    var fingerprintFilename: String {
+        "\(self.description).json"
+    }
+}
+
+private enum SerializationError: Error {
+    case unknownKind(String)
+    case invalidURL(String)
+}

Sources/PackageFingerprint/Model.swift

@@ -0,0 +1,58 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import struct Foundation.URL
+import TSCUtility
+
+public struct Fingerprint: Equatable {
+    public let origin: Origin
+    public let value: String
+}
+
+public extension Fingerprint {
+    enum Kind: String, Hashable {
+        case sourceControl
+        case registry
+    }
+
+    enum Origin: Equatable, CustomStringConvertible {
+        case sourceControl(Foundation.URL)
+        case registry(Foundation.URL)
+
+        var kind: Fingerprint.Kind {
+            switch self {
+            case .sourceControl:
+                return .sourceControl
+            case .registry:
+                return .registry
+            }
+        }
+
+        var url: Foundation.URL? {
+            switch self {
+            case .sourceControl(let url):
+                return url
+            case .registry(let url):
+                return url
+            }
+        }
+
+        public var description: String {
+            switch self {
+            case .sourceControl(let url):
+                return "sourceControl(\(url))"
+            case .registry(let url):
+                return "registry(\(url))"
+            }
+        }
+    }
+}
+
+public typealias PackageFingerprints = [Version: [Fingerprint.Kind: Fingerprint]]

Sources/PackageFingerprint/PackageFingerprintStorage.swift

@@ -0,0 +1,34 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Basics
+import Dispatch
+import PackageModel
+import TSCUtility
+
+public protocol PackageFingerprintStorage {
+    func get(package: PackageIdentity,
+             version: Version,
+             observabilityScope: ObservabilityScope,
+             callbackQueue: DispatchQueue,
+             callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void)
+
+    func put(package: PackageIdentity,
+             version: Version,
+             fingerprint: Fingerprint,
+             observabilityScope: ObservabilityScope,
+             callbackQueue: DispatchQueue,
+             callback: @escaping (Result<Void, Error>) -> Void)
+}
+
+public enum PackageFingerprintStorageError: Error, Equatable {
+    case conflict(given: Fingerprint, existing: Fingerprint)
+    case notFound
+}