Refactor generation of SHA256 checksums (#3116)

* Add sha256Checksum property to ByteString in extension
* Adopt sha256Checksum property in RepositorySpecifier.fileSystemIdentifier
* Adopt sha256Checksum property in PIF.sign(_:)
* Adopt sha256Checksum property in ManifestLoader
* Use CryptoKitSHA256 as workspace checksum algorithm when available

Author: mattt

Sources/Basics/ByteString+Extensions.swift

@@ -0,0 +1,27 @@
+/*
+ This source file is part of the Swift.org open source project
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import TSCBasic
+
+extension ByteString {
+    /// A lowercase, hexadecimal representation of the SHA256 hash
+    /// generated for the byte string's contents.
+    ///
+    /// This property uses the CryptoKit implementation of
+    /// Secure Hashing Algorithm 2 (SHA-2) hashing with a 256-bit digest, when available,
+    /// falling back on a native implementation in Swift provided by TSCBasic.
+    public var sha256Checksum: String {
+        #if canImport(CryptoKit)
+        if #available(macOS 10.15, *) {
+            return CryptoKitSHA256().hash(self).hexadecimalRepresentation
+        }
+        #endif
+
+        return SHA256().hash(self).hexadecimalRepresentation
+    }
+}

Sources/Basics/CMakeLists.txt

@@ -7,6 +7,7 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(Basics
+  ByteString+Extensions.swift
   ConcurrencyHelpers.swift
   Dictionary+Extensions.swift
   DispatchTimeInterval+Extensions.swift

Sources/PackageLoading/ManifestLoader.swift

@@ -664,6 +664,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         let toolsVersion: ToolsVersion
         let env: [String: String]
         let swiftpmVersion: String
+        let sha256Checksum: String
 
         init (packageIdentity: PackageIdentity,
               manifestPath: AbsolutePath,
@@ -672,37 +673,38 @@ public final class ManifestLoader: ManifestLoaderProtocol {
               swiftpmVersion: String,
               fileSystem: FileSystem
         ) throws {
+            let manifestContents = try fileSystem.readFileContents(manifestPath).contents
+            let sha256Checksum = try Self.computeSHA256Checksum(packageIdentity: packageIdentity, manifestContents: manifestContents, toolsVersion: toolsVersion, env: env, swiftpmVersion: swiftpmVersion)
+
             self.packageIdentity = packageIdentity
             self.manifestPath = manifestPath
-            self.manifestContents = try fileSystem.readFileContents(manifestPath).contents
+            self.manifestContents = manifestContents
             self.toolsVersion = toolsVersion
             self.env = env
             self.swiftpmVersion = swiftpmVersion
+            self.sha256Checksum = sha256Checksum
         }
 
-        // TODO: is there a way to avoid the dual hashing?
         func hash(into hasher: inout Hasher) {
-            hasher.combine(self.packageIdentity)
-            hasher.combine(self.manifestContents)
-            hasher.combine(self.toolsVersion.description)
-            for key in self.env.keys.sorted(by: >) {
-                hasher.combine(key)
-                hasher.combine(env[key]!) // forced unwrap safe
-            }
-            hasher.combine(self.swiftpmVersion)
+            hasher.combine(self.sha256Checksum)
         }
 
-        // TODO: is there a way to avoid the dual hashing?
-        func computeHash() throws -> ByteString {
+        private static func computeSHA256Checksum(
+            packageIdentity: PackageIdentity,
+            manifestContents: [UInt8],
+            toolsVersion: ToolsVersion,
+            env: [String: String],
+            swiftpmVersion: String
+        ) throws -> String {
             let stream = BufferedOutputByteStream()
-            stream <<< self.packageIdentity
-            stream <<< self.manifestContents
-            stream <<< self.toolsVersion.description
-            for key in self.env.keys.sorted(by: >) {
-                stream <<< key <<< env[key]!  // forced unwrap safe
+            stream <<< packageIdentity
+            stream <<< manifestContents
+            stream <<< toolsVersion.description
+            for key in env.keys.sorted(by: >) {
+                stream <<< key <<< env[key]! // forced unwrap safe
             }
-            stream <<< self.swiftpmVersion
-            return SHA256().hash(stream.bytes)
+            stream <<< swiftpmVersion
+            return stream.bytes.sha256Checksum
         }
     }
 
@@ -849,7 +851,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
                 if compilerResult.exitStatus != .terminated(code: 0) {
                     return
                 }
-                
+
                 // Pass an open file descriptor of a file to which the JSON representation of the manifest will be written.
                 let jsonOutputFile = tmpDir.appending(component: "\(packageIdentity)-output.json")
                 guard let jsonOutputFileDesc = fopen(jsonOutputFile.pathString, "w") else {
@@ -1135,12 +1137,11 @@ private final class SQLiteManifestCache: Closable {
     }
 
     func put(key: ManifestLoader.ManifestCacheKey, manifest: ManifestLoader.ManifestParseResult) throws {
-        let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"        
+        let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"
         try self.executeStatement(query) { statement -> Void in
-            let keyHash = try key.computeHash()
             let data = try self.jsonEncoder.encode(manifest)
             let bindings: [SQLite.SQLiteValue] = [
-                .string(keyHash.hexadecimalRepresentation),
+                .string(key.sha256Checksum),
                 .blob(data),
             ]
             try statement.bind(bindings)
@@ -1151,8 +1152,7 @@ private final class SQLiteManifestCache: Closable {
     func get(key: ManifestLoader.ManifestCacheKey) throws -> ManifestLoader.ManifestParseResult? {
         let query = "SELECT value FROM MANIFEST_CACHE WHERE key == ? LIMIT 1;"
         return try self.executeStatement(query) { statement ->  ManifestLoader.ManifestParseResult? in
-            let keyHash = try key.computeHash()
-            try statement.bind([.string(keyHash.hexadecimalRepresentation)])
+            try statement.bind([.string(key.sha256Checksum)])
             let data = try statement.step()?.blob(at: 0)
             return try data.flatMap {
                 try self.jsonDecoder.decode(ManifestLoader.ManifestParseResult.self, from: $0)

Sources/SourceControl/Repository.swift

@@ -26,10 +26,9 @@ public struct RepositorySpecifier: Hashable, Codable {
     /// unique for each repository.
     public var fileSystemIdentifier: String {
         // Use first 8 chars of a stable hash.
-        let hash = SHA256().hash(url).hexadecimalRepresentation
-        let suffix = hash.dropLast(hash.count - 8)
+        let suffix = ByteString(encodingAsUTF8: url).sha256Checksum.prefix(8)
 
-        return basename + "-" + suffix
+        return "\(basename)-\(suffix)"
     }
 
     /// Returns the cleaned basename for the specifier.

Sources/Workspace/Workspace.swift

@@ -303,6 +303,13 @@ public class Workspace {
         self.downloader = downloader
         self.netrcFilePath = netrcFilePath
         self.archiver = archiver
+
+        var checksumAlgorithm = checksumAlgorithm
+        #if canImport(CryptoKit)
+        if checksumAlgorithm is SHA256, #available(macOS 10.15, *) {
+            checksumAlgorithm = CryptoKitSHA256()
+        }
+        #endif
         self.checksumAlgorithm = checksumAlgorithm
         self.isResolverPrefetchingEnabled = isResolverPrefetchingEnabled
         self.skipUpdate = skipUpdate

Sources/XCBuildSupport/PIF.swift

@@ -1205,7 +1205,7 @@ extension PIF {
         func sign<T: PIFSignableObject & Encodable>(_ obj: T) throws {
             let signatureContent = try encoder.encode(obj)
             let bytes = ByteString(signatureContent)
-            obj.signature = SHA256().hash(bytes).hexadecimalRepresentation
+            obj.signature = bytes.sha256Checksum
         }
 
         let projects = workspace.projects

Tests/BasicsTests/ByteStringExtensionsTests.swift

@@ -0,0 +1,23 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Basics
+import TSCBasic
+import XCTest
+
+final class ByteStringExtensionsTests: XCTestCase {
+    func testSHA256Checksum() {
+        let byteString = ByteString(encodingAsUTF8: "abc")
+        XCTAssertEqual(byteString.contents, [0x61, 0x62, 0x63])
+
+        // See https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf
+        XCTAssertEqual(byteString.sha256Checksum, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+    }
+}