throw an error for Swift tools version ≤ 5.3 when anything but a single U+0020 is used as the spacing between "//" and "swift-tools-version"

Up to and through Swift 5.3, the format of the swift tools version specification must be prefixed exactly with "// swift-tools-version:", where there is 1 and only 1 space character (U+0020) between "//" and  "swift-tools-version". Even though future versions (> 5.3) will support any combination of horizontal whitespace characters for the spacing, if the specified Swift tools version ≤ 5.3, then the specification must use "// swift-tools-version:" to be backward compatible with lower Swift versions.

A new capture group is added to the regex pattern, for the continuous sequence of whitespace characters between "//" and "swift-tools-version". The capture group for the version specifier remains unchanged, but becomes the 2nd capture group and the 3rd matching range. If both the whitespace sequence and the version specifier are present, and if the version specifier is valid, then compare the version with 5.3. If Swift tools version ≤ 5.3, then throw an error if the whitespace sequence contains anything but a single U+0020. The error informs the user the whitespace characters used, and informs the user that only a single U+0020 is valid.

The test function `testBasics()` is renamed to `testValidVersions` to better reflect what it tests. All valid versions that use anything but a single U+0020 as spacing have their version specifier raised to above 5.3. A new test function `testBackwardCompatibilityError()` is added to verify that if Swift tools version ≤ 5.3, anything but U+0020 is rejected as the spacing between "//" and  "swift-tools-version". Version specifications are moved from `testNonMatching()` to a new test function `testDefault()`, because Swift tools version defaults to 3.1 if the specification reaches a newline character before any pre-defined misspelling.

Author: WowbaggersLiquidLunch

Sources/PackageLoading/ToolsVersionLoader.swift

@@ -102,6 +102,9 @@ public class ToolsVersionLoader: ToolsVersionLoaderProtocol {
         case inaccessibleManifest(path: AbsolutePath, reason: String)
         /// Malformed tools version specifier
         case malformedToolsVersion(specifier: String, currentToolsVersion: ToolsVersion)
+        // TODO: Make the case more general, to better adapt to future changes.
+        /// The spacing between "//" and  "swift-tools-version" either is empty or uses whitespace characters unsupported by Swift ≤ 5.3.
+        case invalidSpacingAfterSlashes(charactersUsed: String, specifiedVersion: ToolsVersion)
 
         public var description: String {
             switch self {
@@ -111,6 +114,17 @@ public class ToolsVersionLoader: ToolsVersionLoaderProtocol {
                 return "the package manifest at '\(manifestFile)' cannot be accessed (\(reason))"
             case .malformedToolsVersion(let versionSpecifier, let currentToolsVersion):
                 return "the tools version '\(versionSpecifier)' is not valid; consider using '// swift-tools-version:\(currentToolsVersion.major).\(currentToolsVersion.minor)' to specify the current tools version"
+            case let .invalidSpacingAfterSlashes(charactersUsed, specifiedVersion):
+                // Tell the user what characters are currently used (invalidly) in the specification in place of U+0020.
+                let unicodeCodePointsOfCharactersUsed: [UInt32] = charactersUsed.flatMap(\.unicodeScalars).map(\.value)
+                let unicodeCodePointsOfCharactersUsedPrefixedByUPlus: [String] = unicodeCodePointsOfCharactersUsed.map { codePoint in
+                    var codePointString = String(codePoint, radix: 16).uppercased()
+                    if codePointString.count < 4 {
+                        codePointString = String(repeating: "0", count: 4 - codePointString.count) + codePointString
+                    }
+                    return "U+\(codePointString)"
+                }
+                return "\(charactersUsed.isEmpty ? "zero spacing" : "horizontal whitespace sequence [\(unicodeCodePointsOfCharactersUsedPrefixedByUPlus.joined(separator: ", "))]") between \"//\" and \"swift-tools-version\" is supported by only Swift > 5.3; consider using a single space (U+0020) for Swift \(specifiedVersion)"
             }
         }
     }
@@ -132,9 +146,16 @@ public class ToolsVersionLoader: ToolsVersionLoaderProtocol {
         do { contents = try fileSystem.readFileContents(file) } catch {
             throw Error.inaccessibleManifest(path: file, reason: String(describing: error))
         }
-
+        
+        /// The constituent parts of the swift tools version specification found in the comment.
+        let deconstructedToolsVersionSpecification = ToolsVersionLoader.split(contents)
+        
         // Get the version specifier string from tools version file.
-        guard let versionSpecifier = ToolsVersionLoader.split(contents).versionSpecifier else {
+        guard
+            let spacingAfterSlashes = deconstructedToolsVersionSpecification.spacingAfterSlashes,
+            let versionSpecifier = deconstructedToolsVersionSpecification.versionSpecifier
+        else {
+            // TODO: Make the diagnsosis more granular by having the regex capture more groups.
             // Try to diagnose if there is a misspelling of the swift-tools-version comment.
             let splitted = contents.contents.split(
                 separator: UInt8(ascii: "\n"),
@@ -153,11 +174,23 @@ public class ToolsVersionLoader: ToolsVersionLoaderProtocol {
         guard let version = ToolsVersion(string: versionSpecifier) else {
             throw Error.malformedToolsVersion(specifier: versionSpecifier, currentToolsVersion: currentToolsVersion)
         }
+        
+        // Ensure that for Swift ≤ 5.3, a single U+0020 is used as spacing between "//" and "swift-tools-version".
+        // Use `ToolsVersion(version:)` instead of `ToolsVeresion(string:)` here to avoid forced unwrappiing.
+        guard spacingAfterSlashes == " " || version > ToolsVersion(version: "5.3.0") else {
+            throw Error.invalidSpacingAfterSlashes(charactersUsed: spacingAfterSlashes, specifiedVersion: version)
+        }
+        
         return version
     }
 
-    /// Splits the bytes to the version specifier (if present) and rest of the contents.
-    public static func split(_ bytes: ByteString) -> (versionSpecifier: String?, rest: [UInt8]) {
+    /// Splits the bytes to constituent parts of the swift tools version specification and rest of the contents.
+    ///
+    /// The constituent parts include the spacing between "//" and "swift-tools-version", and the version specifier, if either is present.
+    ///
+    /// - Parameter bytes: The raw bytes of the content of `Package.swift`.
+    /// - Returns: The spacing between "//" and "swift-tools-version" (if present, or `nil`), the version specifier (if present, or `nil`), and of raw bytes of the rest of the content of `Package.swift`.
+    public static func split(_ bytes: ByteString) -> (spacingAfterSlashes: String?, versionSpecifier: String?, rest: [UInt8]) {
         let splitted = bytes.contents.split(
             separator: UInt8(ascii: "\n"),
             maxSplits: 1,
@@ -166,19 +199,32 @@ public class ToolsVersionLoader: ToolsVersionLoaderProtocol {
         guard let firstLine = ByteString(splitted[0]).validDescription,
               let match = ToolsVersionLoader.regex.firstMatch(
                   in: firstLine, options: [], range: NSRange(location: 0, length: firstLine.count)),
-              match.numberOfRanges >= 2 else {
-            return (nil, bytes.contents)
+              // The 3 ranges are:
+              //   1. The entire matched string.
+              //   2. Capture group 1: the comment spacing.
+              //   3. Capture group 2: The version specifier.
+              // Since the version specifier is in the last range, if the number of ranges is less than 3, then no version specifier is captured by the regex.
+              // FIXME: Should this be `== 3` instead?
+              match.numberOfRanges >= 3 else {
+            return (nil, nil, bytes.contents)
         }
-        let versionSpecifier = NSString(string: firstLine).substring(with: match.range(at: 1))
+        let spacingAfterSlashes = NSString(string: firstLine).substring(with: match.range(at: 1))
+        let versionSpecifier = NSString(string: firstLine).substring(with: match.range(at: 2))
         // FIXME: We can probably optimize here and return array slice.
-        return (versionSpecifier, splitted.count == 1 ? [] : Array(splitted[1]))
+        return (spacingAfterSlashes, versionSpecifier, splitted.count == 1 ? [] : Array(splitted[1]))
     }
 
     /// The regex to match swift tools version specification.
+    ///
+    /// The specification must have the following format:
     /// - It should start with `//` followed by any amount of _horizontal_ whitespace characters.
     /// - Following that it should contain the case insensitive string `swift-tools-version:`.
     /// - The text between the above string and `;` or string end becomes the tools version specifier.
+    ///
+    /// There are 2 capture groups in the regex pattern:
+    /// 1. The continuous sequence of whitespace characters between "//" and "swift-tools-version".
+    /// 2. The version specifier.
     static let regex = try! NSRegularExpression(
-        pattern: "^//\\h*?swift-tools-version:(.*?)(?:;.*|$)",
+        pattern: "^//(\\h*?)swift-tools-version:(.*?)(?:;.*|$)",
         options: [.caseInsensitive])
 }