Disallow whitespace between @, attribute name and (

Add a Swift version parameter to parser. When parsing in Swift 6 mode, disallow whitespace between `@`, the attribute’s name and `(`. In Swift 5 warn.

Use this opportunity to refactor how we diagnose extraneous whitespace.

rdar://121668395

Author: ahoppen

Sources/SwiftParser/Attributes.swift

@@ -176,7 +176,14 @@ extension Parser {
     argumentMode: AttributeArgumentMode,
     parseArguments: (inout Parser) -> RawAttributeSyntax.Arguments
   ) -> RawAttributeListSyntax.Element {
-    let (unexpectedBeforeAtSign, atSign) = self.expect(.atSign)
+    var (unexpectedBeforeAtSign, atSign) = self.expect(.atSign)
+    if atSign.trailingTriviaByteLength > 0 || self.currentToken.leadingTriviaByteLength > 0 {
+      let diagnostic = TokenDiagnostic(
+        self.swiftVersion < .v6 ? .extraneousTrailingWhitespaceWarning : .extraneousTrailingWhitespaceError,
+        byteOffset: atSign.leadingTriviaByteLength + atSign.tokenText.count
+      )
+      atSign = atSign.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
+    }
     let attributeName = self.parseType()
     let shouldParseArgument: Bool
     switch argumentMode {
@@ -190,7 +197,14 @@ extension Parser {
       shouldParseArgument = false
     }
     if shouldParseArgument {
-      let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
+      var (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
+      if attributeName.raw.trailingTriviaByteLength > 0 || leftParen.leadingTriviaByteLength > 0 {
+        let diagnostic = TokenDiagnostic(
+          self.swiftVersion < .v6 ? .extraneousLeadingWhitespaceWarning : .extraneousLeadingWhitespaceError,
+          byteOffset: 0
+        )
+        leftParen = leftParen.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
+      }
       let argument = parseArguments(&self)
       let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
       return .attribute(

Sources/SwiftParser/CMakeLists.txt

@@ -30,8 +30,9 @@ add_swift_syntax_library(SwiftParser
   Recovery.swift
   Specifiers.swift
   Statements.swift
-  StringLiterals.swift
   StringLiteralRepresentedLiteralValue.swift
+  StringLiterals.swift
+  SwiftVersion.swift
   SyntaxUtils.swift
   TokenConsumer.swift
   TokenPrecedence.swift

Sources/SwiftParser/Declarations.swift

@@ -1930,19 +1930,18 @@ extension Parser {
   ) -> RawMacroExpansionDeclSyntax {
 
     var (unexpectedBeforePound, pound) = self.eat(handle)
-    if pound.trailingTriviaByteLength != 0 {
-      // `#` and the macro name must not be separated by a newline.
-      unexpectedBeforePound = RawUnexpectedNodesSyntax(combining: unexpectedBeforePound, pound, arena: self.arena)
-      pound = RawTokenSyntax(missing: .pound, text: "#", leadingTriviaPieces: pound.leadingTriviaPieces, arena: self.arena)
+    if pound.trailingTriviaByteLength > 0 || currentToken.leadingTriviaByteLength > 0 {
+      // If there are whitespaces after '#' diagnose.
+      let diagnostic = TokenDiagnostic(
+        .extraneousTrailingWhitespaceError,
+        byteOffset: pound.leadingTriviaByteLength + pound.tokenText.count
+      )
+      pound = pound.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
     }
-    var unexpectedBeforeMacro: RawUnexpectedNodesSyntax?
-    var macro: RawTokenSyntax
+    let unexpectedBeforeMacro: RawUnexpectedNodesSyntax?
+    let macro: RawTokenSyntax
     if !self.atStartOfLine {
       (unexpectedBeforeMacro, macro) = self.expectIdentifier(allowKeywordsAsIdentifier: true)
-      if macro.leadingTriviaByteLength != 0 {
-        unexpectedBeforeMacro = RawUnexpectedNodesSyntax(combining: unexpectedBeforeMacro, macro, arena: self.arena)
-        pound = self.missingToken(.identifier, text: macro.tokenText)
-      }
     } else {
       unexpectedBeforeMacro = nil
       macro = self.missingToken(.identifier)

Sources/SwiftParser/Expressions.swift

@@ -1271,20 +1271,18 @@ extension Parser {
     flavor: ExprFlavor
   ) -> RawMacroExpansionExprSyntax {
     var (unexpectedBeforePound, pound) = self.expect(.pound)
-    if pound.trailingTriviaByteLength != 0 {
+    if pound.trailingTriviaByteLength > 0 || currentToken.leadingTriviaByteLength > 0 {
       // If there are whitespaces after '#' diagnose.
-      unexpectedBeforePound = RawUnexpectedNodesSyntax(combining: unexpectedBeforePound, pound, arena: self.arena)
-      pound = self.missingToken(.pound)
+      let diagnostic = TokenDiagnostic(
+        .extraneousTrailingWhitespaceError,
+        byteOffset: pound.leadingTriviaByteLength + pound.tokenText.count
+      )
+      pound = pound.tokenView.withTokenDiagnostic(tokenDiagnostic: diagnostic, arena: self.arena)
     }
-    var unexpectedBeforeMacroName: RawUnexpectedNodesSyntax?
-    var macroName: RawTokenSyntax
+    let unexpectedBeforeMacroName: RawUnexpectedNodesSyntax?
+    let macroName: RawTokenSyntax
     if !self.atStartOfLine {
       (unexpectedBeforeMacroName, macroName) = self.expectIdentifier(allowKeywordsAsIdentifier: true)
-      if macroName.leadingTriviaByteLength != 0 {
-        // If there're whitespaces after '#' diagnose.
-        unexpectedBeforeMacroName = RawUnexpectedNodesSyntax(combining: unexpectedBeforeMacroName, macroName, arena: self.arena)
-        pound = self.missingToken(.identifier, text: macroName.tokenText)
-      }
     } else {
       unexpectedBeforeMacroName = nil
       macroName = self.missingToken(.identifier)

Sources/SwiftParser/Lexer/Lexeme.swift

@@ -102,7 +102,8 @@ extension Lexer {
       SyntaxText(baseAddress: start, count: byteLength)
     }
 
-    var textRange: Range<SyntaxText.Index> {
+    @_spi(Testing)
+    public var textRange: Range<SyntaxText.Index> {
       leadingTriviaByteLength..<leadingTriviaByteLength + textByteLength
     }
 

Sources/SwiftParser/Lookahead.swift

@@ -27,24 +27,30 @@ extension Parser {
     /// i.e. how far it looked ahead.
     var tokensConsumed: Int = 0
 
+    /// The Swift version as which this source file should be parsed.
+    let swiftVersion: SwiftVersion
+
     /// The experimental features that have been enabled in the underlying
     /// parser.
     let experimentalFeatures: ExperimentalFeatures
 
     private init(
       lexemes: Lexer.LexemeSequence,
       currentToken: Lexer.Lexeme,
+      swiftVersion: SwiftVersion,
       experimentalFeatures: ExperimentalFeatures
     ) {
       self.lexemes = lexemes
       self.currentToken = currentToken
+      self.swiftVersion = swiftVersion
       self.experimentalFeatures = experimentalFeatures
     }
 
     fileprivate init(cloning other: Parser) {
       self.init(
         lexemes: other.lexemes,
         currentToken: other.currentToken,
+        swiftVersion: other.swiftVersion,
         experimentalFeatures: other.experimentalFeatures
       )
     }
@@ -55,6 +61,7 @@ extension Parser {
       return Lookahead(
         lexemes: self.lexemes,
         currentToken: self.currentToken,
+        swiftVersion: self.swiftVersion,
         experimentalFeatures: self.experimentalFeatures
       )
     }

Sources/SwiftParser/ParseSourceFile.swift

@@ -26,19 +26,21 @@ extension Parser {
   @_spi(ExperimentalLanguageFeatures)
   public static func parse(
     source: UnsafeBufferPointer<UInt8>,
+    swiftVersion: SwiftVersion? = nil,
     experimentalFeatures: ExperimentalFeatures
   ) -> SourceFileSyntax {
-    var parser = Parser(source, experimentalFeatures: experimentalFeatures)
+    var parser = Parser(source, swiftVersion: swiftVersion, experimentalFeatures: experimentalFeatures)
     return SourceFileSyntax.parse(from: &parser)
   }
 
   /// Parse the source code in the given buffer as Swift source file. See
   /// `Parser.init` for more details.
   public static func parse(
     source: UnsafeBufferPointer<UInt8>,
-    maximumNestingLevel: Int? = nil
+    maximumNestingLevel: Int? = nil,
+    swiftVersion: SwiftVersion? = nil
   ) -> SourceFileSyntax {
-    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
+    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, swiftVersion: swiftVersion)
     return SourceFileSyntax.parse(from: &parser)
   }
 

Sources/SwiftParser/Parser.swift

@@ -117,6 +117,9 @@ public struct Parser {
   /// Parser should own a ``LookaheadTracker`` so that we can share one `furthestOffset` in a parse.
   let lookaheadTrackerOwner: LookaheadTrackerOwner
 
+  /// The Swift version as which this source file should be parsed.
+  let swiftVersion: SwiftVersion
+
   /// The experimental features that have been enabled.
   let experimentalFeatures: ExperimentalFeatures
 
@@ -187,26 +190,28 @@ public struct Parser {
   ///            arena is created automatically, and `input` copied into the
   ///            arena. If non-`nil`, `input` must be within its registered
   ///            source buffer or allocator.
+  ///  - swiftVersion: The version of Swift using which the file should be parsed.
+  ///                  Defaults to the latest version.
   ///  - experimentalFeatures: The experimental features enabled for the parser.
   private init(
     buffer input: UnsafeBufferPointer<UInt8>,
     maximumNestingLevel: Int?,
     parseTransition: IncrementalParseTransition?,
     arena: ParsingSyntaxArena?,
+    swiftVersion: SwiftVersion?,
     experimentalFeatures: ExperimentalFeatures
   ) {
     var input = input
     if let arena {
       self.arena = arena
       precondition(arena.contains(text: SyntaxText(baseAddress: input.baseAddress, count: input.count)))
     } else {
-      self.arena = ParsingSyntaxArena(
-        parseTriviaFunction: TriviaParser.parseTrivia(_:position:)
-      )
+      self.arena = ParsingSyntaxArena(parseTriviaFunction: TriviaParser.parseTrivia)
       input = self.arena.internSourceBuffer(input)
     }
 
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
+    self.swiftVersion = swiftVersion ?? .v6
     self.experimentalFeatures = experimentalFeatures
     self.lookaheadTrackerOwner = LookaheadTrackerOwner()
 
@@ -224,6 +229,7 @@ public struct Parser {
     string input: String,
     maximumNestingLevel: Int?,
     parseTransition: IncrementalParseTransition?,
+    swiftVersion: SwiftVersion?,
     experimentalFeatures: ExperimentalFeatures
   ) {
     var input = input
@@ -234,6 +240,7 @@ public struct Parser {
         maximumNestingLevel: maximumNestingLevel,
         parseTransition: parseTransition,
         arena: nil,
+        swiftVersion: swiftVersion,
         experimentalFeatures: experimentalFeatures
       )
     }
@@ -243,13 +250,15 @@ public struct Parser {
   public init(
     _ input: String,
     maximumNestingLevel: Int? = nil,
-    parseTransition: IncrementalParseTransition? = nil
+    parseTransition: IncrementalParseTransition? = nil,
+    swiftVersion: SwiftVersion? = nil
   ) {
     // Chain to the private String initializer.
     self.init(
       string: input,
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: parseTransition,
+      swiftVersion: swiftVersion,
       experimentalFeatures: []
     )
   }
@@ -277,14 +286,16 @@ public struct Parser {
     _ input: UnsafeBufferPointer<UInt8>,
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
-    arena: ParsingSyntaxArena? = nil
+    arena: ParsingSyntaxArena? = nil,
+    swiftVersion: SwiftVersion? = nil
   ) {
     // Chain to the private buffer initializer.
     self.init(
       buffer: input,
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: parseTransition,
       arena: arena,
+      swiftVersion: swiftVersion,
       experimentalFeatures: []
     )
   }
@@ -296,13 +307,15 @@ public struct Parser {
     _ input: String,
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
+    swiftVersion: SwiftVersion? = nil,
     experimentalFeatures: ExperimentalFeatures
   ) {
     // Chain to the private String initializer.
     self.init(
       string: input,
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: parseTransition,
+      swiftVersion: swiftVersion,
       experimentalFeatures: experimentalFeatures
     )
   }
@@ -315,6 +328,7 @@ public struct Parser {
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition? = nil,
     arena: ParsingSyntaxArena? = nil,
+    swiftVersion: SwiftVersion? = nil,
     experimentalFeatures: ExperimentalFeatures
   ) {
     // Chain to the private buffer initializer.
@@ -323,6 +337,7 @@ public struct Parser {
       maximumNestingLevel: maximumNestingLevel,
       parseTransition: parseTransition,
       arena: arena,
+      swiftVersion: swiftVersion,
       experimentalFeatures: experimentalFeatures
     )
   }

Sources/SwiftParser/StringLiterals.swift

@@ -64,10 +64,11 @@ fileprivate class StringLiteralExpressionIndentationChecker {
       // error is fixed
       return nil
     }
-    return token.tokenView.withTokenDiagnostic(
+    let tokenWithDiagnostic = token.tokenView.withTokenDiagnostic(
       tokenDiagnostic: TokenDiagnostic(.insufficientIndentationInMultilineStringLiteral, byteOffset: 0),
       arena: arena
     )
+    return RawSyntax(tokenWithDiagnostic)
   }
 
   private func visitLayoutNode(node: RawSyntax) -> RawSyntax? {

Sources/SwiftParser/SwiftVersion.swift

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+extension Parser {
+  /// A Swift language version.
+  public enum SwiftVersion: Comparable {
+    case v4
+    case v5
+    case v6
+  }
+}

Sources/SwiftParser/TokenConsumer.swift

@@ -18,6 +18,8 @@ protocol TokenConsumer {
   /// The current token syntax being examined by the consumer
   var currentToken: Lexer.Lexeme { get }
 
+  var swiftVersion: Parser.SwiftVersion { get }
+
   /// The experimental features that have been enabled.
   var experimentalFeatures: Parser.ExperimentalFeatures { get }