Merge branch 'main' into revert-3677-revert-collections-again

Author: tomerd

CHANGELOG.md

@@ -2,6 +2,10 @@ Note: This is in reverse chronological order, so newer entries are added to the
 
 Swift 5.6
 -----------
+* [SE-0332]
+
+  Package plugins of the type `command` can now be declared in packages that specify a tools version of 5.6 or later, and can be invoked using the `swift package` subcommand.
+
 * [#3649]
 
   Semantic version dependencies can now be resolved against Git tag names that contain only major and minor version identifiers.  A tag with the form `X.Y` will be treated as `X.Y.0`. This improves compatibility with existing repositories.
@@ -22,6 +26,28 @@ Swift 5.6
 
   Adding a dependency requirement can now be done with the convenience initializer `.package(url: String, exact: Version)`.
 
+* [#3641]
+
+  Dependency requirement enum calling convention is deprecated in favour of labeled argument:    
+    * `.package(url: String, .branch(String))` -> `.package(url: String, branch: String)`
+    * `.package(url: String, .revision(String))` -> `.package(url: String, revision: String)`    
+    * `.package(url: String, .exact(Version))` -> `.package(url: String, exact: Version)` 
+
+* [#3717]
+
+  Introduce a second version of `Package.resolved` file format which more accurately captures package identity.
+
+* [#3890]
+
+  To increase the security of packages, SwiftPM performs trust on first use (TOFU) validation. The fingerprint of a package is now being recorded when the package is first downloaded from a Git repository or package registry. Subsequent downloads must have fingerpints matching previous recorded values, otherwise it would result in build warnings or failures depending on settings.   
+
+* [#3670], [#3901], [#3942]
+
+  Location of configuration files (including mirror file) have changed to accomodate new features that require more robust configuration directories structure, such as SE-0292:  
+    * `<project>/.swiftpm/config` (mirrors file) was moved to `<project>/.swiftpm/configuration/mirrors.json`. SwiftPM 5.6 will automatically copy the file from the old location to the new one and emit a warning to prompt the user to delete the file from the old location.
+    * `~/.swiftpm/config/collections.json` (collections file) was moved to `~/.swiftpm/configuration/collections.json`. SwiftPM 5.6 will automatically copy the file from the old location to the new one and emit a warning to prompt the user to delete the file from the old location.
+
+
 Swift 5.5
 -----------
 * [#3410]
@@ -44,6 +70,7 @@ Swift 5.5
 
   Test targets can now link against executable targets as if they were libraries, so that they can test any data structures or algorithms in them.  All the code in the executable except for the main entry point itself is available to the unit test.  Separate executables are still linked, and can be tested as a subprocess in the same way as before.  This feature is available to tests defined in packages that have a tools version of `5.5` or newer. 
 
+
 Swift 5.4
 -----------
 * [#2937]
@@ -67,7 +94,6 @@ Swift 5.4
     The package manager now throws an error if a manifest file contains invalid UTF-8 byte sequences.
 
 
-
 Swift 4.2
 ---------
 
@@ -108,6 +134,7 @@ Swift 4.2
 * [#1489]
   A simpler progress bar is now generated for "dumb" terminals.
 
+
 Swift 4.1
 ---------
 
@@ -130,6 +157,7 @@ Swift 4.0
 * `--specifier` option for `swift test` is now deprecated.
   Use `--filter` instead which supports regex.
 
+
 Swift 3.0
 ---------
 
@@ -153,11 +181,12 @@ Swift 3.0
 
 * The `Package` initializer now requires the `name:` parameter.
 
-[SE-0129]: https://github.com/apple/swift-evolution/blob/master/proposals/0129-package-manager-test-naming-conventions.md
-[SE-0135]: https://github.com/apple/swift-evolution/blob/master/proposals/0135-package-manager-support-for-differentiating-packages-by-swift-version.md
-[SE-0201]: https://github.com/apple/swift-evolution/blob/master/proposals/0201-package-manager-local-dependencies.md
-[SE-0208]: https://github.com/apple/swift-evolution/blob/master/proposals/0208-package-manager-system-library-targets.md
-[SE-0209]: https://github.com/apple/swift-evolution/blob/master/proposals/0209-package-manager-swift-lang-version-update.md
+[SE-0129]: https://github.com/apple/swift-evolution/blob/main/proposals/0129-package-manager-test-naming-conventions.md
+[SE-0135]: https://github.com/apple/swift-evolution/blob/main/proposals/0135-package-manager-support-for-differentiating-packages-by-swift-version.md
+[SE-0201]: https://github.com/apple/swift-evolution/blob/main/proposals/0201-package-manager-local-dependencies.md
+[SE-0208]: https://github.com/apple/swift-evolution/blob/main/proposals/0208-package-manager-system-library-targets.md
+[SE-0209]: https://github.com/apple/swift-evolution/blob/main/proposals/0209-package-manager-swift-lang-version-update.md
+[SE-0332]: https://github.com/apple/swift-evolution/blob/main/proposals/0332-swiftpm-command-plugins.md
 
 [SR-5918]: https://bugs.swift.org/browse/SR-5918
 [SR-6978]: https://bugs.swift.org/browse/SR-6978
@@ -175,3 +204,8 @@ Swift 3.0
 [#3486]: https://github.com/apple/swift-package-manager/pull/3486
 [#3641]: https://github.com/apple/swift-package-manager/pull/3641
 [#3649]: https://github.com/apple/swift-package-manager/pull/3649
+[#3670]: https://github.com/apple/swift-package-manager/pull/3670
+[#3717]: https://github.com/apple/swift-package-manager/pull/3717
+[#3890]: https://github.com/apple/swift-package-manager/pull/3890
+[#3901]: https://github.com/apple/swift-package-manager/pull/3901
+[#3942]: https://github.com/apple/swift-package-manager/pull/3942

Fixtures/Miscellaneous/TestDiscovery/Async/Tests/AsyncTests/SwiftTests.swift

@@ -7,6 +7,9 @@ class AsyncTests: XCTestCase {
     func testAsync() async {
     }
 
+    func testAsyncThrows() async throws {
+    }
+
     @MainActor func testMainActor() async {
         XCTAssertTrue(Thread.isMainThread)
     }

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Package.swift

@@ -0,0 +1,10 @@
+// swift-tools-version:4.2
+import PackageDescription
+
+let package = Package(
+    name: "Simple",
+    targets: [
+        .target(name: "Simple"),
+        .testTarget(name: "SimpleTests", dependencies: ["Simple"]),
+    ]
+)

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Sources/Simple/Simple.swift

@@ -0,0 +1,6 @@
+struct Simple {
+    func hello() {}
+
+    @available(*, deprecated, message: "use hello instead")
+    func deprecatedHello() {}
+}

Fixtures/Miscellaneous/TestDiscovery/Deprecation/Tests/SimpleTests/SwiftTests.swift

@@ -0,0 +1,13 @@
+import XCTest
+@testable import Simple
+
+class SimpleTests: XCTestCase {
+    func testHello() {
+        Simple().hello()
+    }
+
+    @available(*, deprecated, message: "testing deprecated API")
+    func testDeprecatedHello() {
+        Simple().deprecatedHello()
+    }
+}

Fixtures/Miscellaneous/TestDiscovery/Simple/Tests/SimpleTests/SwiftTests.swift

@@ -9,7 +9,10 @@ class SimpleTests: XCTestCase {
     func test_Example2() {
     }
 
-    func testExample3(arg: String) {
+    func testThrowing() throws {
+    }
+
+    func testWithArgs(arg: String) {
     }
 
     func nontest() {

Sources/Basics/ByteString+Extensions.swift

@@ -16,12 +16,6 @@ extension ByteString {
     /// Secure Hashing Algorithm 2 (SHA-2) hashing with a 256-bit digest, when available,
     /// falling back on a native implementation in Swift provided by TSCBasic.
     public var sha256Checksum: String {
-        #if canImport(CryptoKit)
-        if #available(macOS 10.15, *) {
-            return CryptoKitSHA256().hash(self).hexadecimalRepresentation
-        }
-        #endif
-
         return SHA256().hash(self).hexadecimalRepresentation
     }
 }

Sources/Basics/DispatchTimeInterval+Extensions.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Copyright (c) 2020-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See http://swift.org/LICENSE.txt for license information
@@ -12,7 +12,7 @@ import Dispatch
 import struct Foundation.TimeInterval
 
 extension DispatchTimeInterval {
-    func timeInterval() -> TimeInterval? {
+    public func timeInterval() -> TimeInterval? {
         switch self {
         case .seconds(let value):
             return Double(value)
@@ -27,7 +27,7 @@ extension DispatchTimeInterval {
         }
     }
 
-    func milliseconds() -> Int? {
+    public func milliseconds() -> Int? {
         switch self {
         case .seconds(let value):
             return value.multipliedReportingOverflow(by: 1000).partialValue
@@ -42,7 +42,7 @@ extension DispatchTimeInterval {
         }
     }
 
-    func seconds() -> Int? {
+    public func seconds() -> Int? {
         switch self {
         case .seconds(let value):
             return value

Sources/Basics/Sandbox.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See http://swift.org/LICENSE.txt for license information
@@ -13,23 +13,38 @@ import TSCBasic
 import TSCUtility
 
 public enum Sandbox {
+
+    /// Applies a sandbox invocation to the given command line (if the platform supports it),
+    /// and returns the modified command line. On platforms that don't support sandboxing, the
+    /// command line is returned unmodified.
+    ///
+    /// - Parameters:
+    ///   - command: The command line to sandbox (including executable as first argument)
+    ///   - strictness: The basic strictness level of the standbox.
+    ///   - writableDirectories: Paths under which writing should be allowed, even if they would otherwise be read-only based on the strictness or paths in `readOnlyDirectories`.
+    ///   - readOnlyDirectories: Paths under which writing should be denied, even if they would have otherwise been allowed by the rules implied by the strictness level.
     public static func apply(
         command: [String],
+        strictness: Strictness = .default,
         writableDirectories: [AbsolutePath] = [],
-        strictness: Strictness = .default
+        readOnlyDirectories: [AbsolutePath] = []
     ) -> [String] {
         #if os(macOS)
-        let profile = macOSSandboxProfile(writableDirectories: writableDirectories, strictness: strictness)
+        let profile = macOSSandboxProfile(strictness: strictness, writableDirectories: writableDirectories, readOnlyDirectories: readOnlyDirectories)
         return ["/usr/bin/sandbox-exec", "-p", profile] + command
         #else
         // rdar://40235432, rdar://75636874 tracks implementing sandboxes for other platforms.
         return command
         #endif
     }
 
+    /// Basic strictness level of a sandbox applied to a command line.
     public enum Strictness: Equatable {
+        /// Blocks network access and all file system modifications.
         case `default`
+        /// More lenient restrictions than the default, for compatibility with SwiftPM manifests using a tools version older than 5.3.
         case manifest_pre_53 // backwards compatibility for manifests
+        /// Like `default`, but also makes temporary-files directories (such as `/tmp`) on the platform writable.
         case writableTemporaryDirectory
     }
 }
@@ -38,8 +53,9 @@ public enum Sandbox {
 
 #if os(macOS)
 fileprivate func macOSSandboxProfile(
+    strictness: Sandbox.Strictness,
     writableDirectories: [AbsolutePath],
-    strictness: Sandbox.Strictness
+    readOnlyDirectories: [AbsolutePath]
 ) -> String {
     var contents = "(version 1)\n"
 
@@ -63,9 +79,8 @@ fileprivate func macOSSandboxProfile(
     }
 
     // Allow writing only to certain directories.
-    var writableDirectoriesExpression = writableDirectories.map {
-        "(subpath \(resolveSymlinks($0).quotedAsSubpathForSandboxProfile))"
-    }
+    var writableDirectoriesExpression: [String] = []
+
     // The following accesses are only needed when interpreting the manifest (versus running a compiled version).
     if strictness == .manifest_pre_53 {
         writableDirectoriesExpression += Platform.threadSafeDarwinCacheDirectories.get().map {
@@ -80,6 +95,7 @@ fileprivate func macOSSandboxProfile(
         }
     }
 
+    // Emit rules for paths under which writing is allowed. Some of these expressions may be regular expressions and others literal subpaths.
     if writableDirectoriesExpression.count > 0 {
         contents += "(allow file-write*\n"
         for expression in writableDirectoriesExpression {
@@ -88,6 +104,24 @@ fileprivate func macOSSandboxProfile(
         contents += ")\n"
     }
 
+    // Emit rules for paths under which writing should be disallowed, even if they would be covered by a previous rule to allow writing to them. A classic case is a package which is located under the temporary directory, which should be read-only even though the temporary directory as a whole is writable.
+    if readOnlyDirectories.count > 0 {
+        contents += "(deny file-write*\n"
+        for path in readOnlyDirectories {
+            contents += "    (subpath \(resolveSymlinks(path).quotedAsSubpathForSandboxProfile))\n"
+        }
+        contents += ")\n"
+    }
+
+    // Emit rules for paths under which writing is allowed, even if they are descendants directories that are otherwise read-only.
+    if writableDirectories.count > 0 {
+        contents += "(allow file-write*\n"
+        for path in writableDirectories {
+            contents += "    (subpath \(resolveSymlinks(path).quotedAsSubpathForSandboxProfile))\n"
+        }
+        contents += ")\n"
+    }
+
     return contents
 }
 

Sources/Basics/VirtualFileSystem.swift

@@ -177,6 +177,8 @@ public class VirtualFileSystem: FileSystem {
 
     public var cachesDirectory: AbsolutePath? = nil
 
+    public var tempDirectory = AbsolutePath("/")
+
     public func createSymbolicLink(_ path: AbsolutePath, pointingAt destination: AbsolutePath, relative: Bool) throws {
         throw Errors.readOnlyFileSystem
     }

Sources/Build/BuildOperation.swift

@@ -209,6 +209,36 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         let prebuildCommandResults = try graph.reachableTargets.reduce(into: [:], { partial, target in
             partial[target] = try buildToolPluginInvocationResults[target].map { try self.runPrebuildCommands(for: $0) }
         })
+
+        // Emit warnings about any unhandled files in authored packages. We do this after applying build tool plugins, once we know what files they handled.
+        for package in graph.rootPackages where package.manifest.toolsVersion >= .v5_3 {
+            for target in package.targets {
+                // Get the set of unhandled files in targets.
+                var unhandledFiles = Set(target.underlyingTarget.others)
+                if unhandledFiles.isEmpty { continue }
+                
+                // Subtract out any that were inputs to any commands generated by plugins.
+                if let result = buildToolPluginInvocationResults[target] {
+                    let handledFiles = result.flatMap{ $0.buildCommands.flatMap{ $0.inputFiles } }
+                    unhandledFiles.subtract(handledFiles)
+                }
+                if unhandledFiles.isEmpty { continue }
+                
+                // Emit a diagnostic if any remain. This is kept the same as the previous message for now, but this could be improved.
+                let diagnosticsEmitter = self.observabilityScope.makeDiagnosticsEmitter {
+                    var metadata = ObservabilityMetadata()
+                    metadata.packageIdentity = package.identity
+                    metadata.packageKind = package.manifest.packageKind
+                    metadata.targetName = target.name
+                    return metadata
+                }
+                var warning = "found \(unhandledFiles.count) file(s) which are unhandled; explicitly declare them as resources or exclude from the target\n"
+                for file in unhandledFiles {
+                    warning += "    " + file.pathString + "\n"
+                }
+                diagnosticsEmitter.emit(warning: warning)
+            }
+        }
 
         // Create the build plan based, on the graph and any information from plugins.
         let plan = try BuildPlan(
@@ -314,7 +344,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 // TODO: We need to also use any working directory, but that support isn't yet available on all platforms at a lower level.
                 var commandLine = [command.configuration.executable.pathString] + command.configuration.arguments
                 if !self.disableSandboxForPluginCommands {
-                    commandLine = Sandbox.apply(command: commandLine, writableDirectories: [pluginResult.pluginOutputDirectory], strictness: .writableTemporaryDirectory)
+                    commandLine = Sandbox.apply(command: commandLine, strictness: .writableTemporaryDirectory, writableDirectories: [pluginResult.pluginOutputDirectory])
                 }
                 let processResult = try Process.popen(arguments: commandLine, environment: command.configuration.environment)
                 let output = try processResult.utf8Output() + processResult.utf8stderrOutput()