Merge pull request #734 from DougGregor/misc-parser-fixes

Author: DougGregor

Sources/SwiftParser/Attributes.swift

@@ -183,7 +183,7 @@ extension Parser {
 
 
     let atSign = self.eat(.atSign)
-    let ident = self.consumeIdentifier()
+    let ident = self.consumeIdentifierOrRethrows()
     let leftParen: RawTokenSyntax?
     let arg: RawSyntax?
     let unexpectedBeforeRightParen: RawUnexpectedNodesSyntax?
@@ -458,7 +458,7 @@ extension Parser {
             name: name,
             colon: nil,
             arena: self.arena))
-          continue
+          break
         }
 
         let (unexpectedBeforeColon, colon) = self.expect(.colon)

Sources/SwiftParser/Expressions.swift

@@ -721,6 +721,9 @@ extension Parser {
     case .__file__Keyword:
       let tok = self.eat(.__file__Keyword)
       return RawExprSyntax(RawPoundFileExprSyntax(poundFile: tok, arena: self.arena))
+    case .poundFileIDKeyword:
+      let tok = self.eat(.poundFileIDKeyword)
+      return RawExprSyntax(RawPoundFileIDExprSyntax(poundFileID: tok, arena: self.arena))
     case .poundFileKeyword:
       let tok = self.eat(.poundFileKeyword)
       return RawExprSyntax(RawPoundFileExprSyntax(poundFile: tok, arena: self.arena))

Sources/SwiftParser/Lexer.swift

@@ -1439,7 +1439,7 @@ extension Lexer.Cursor {
         return .integerLiteral
       }
 
-      self.advance(while: { $0.isDigit || $0 == Unicode.Scalar("_") })
+      self.advance(while: { $0.isHexDigit || $0 == Unicode.Scalar("_") })
 
       if !self.isAtEndOfFile, self.peek() != UInt8(ascii: "p") && self.peek() != UInt8(ascii: "P") {
         if !Unicode.Scalar(PtrOnDot!.peek(at: 1)).isDigit {
@@ -2007,6 +2007,8 @@ extension Lexer.Cursor {
   ) -> RawTokenKind? {
     var Tmp = TokStart
     var poundCount = 0
+    var parenCount = 0
+
     while Tmp.advance(matching: UInt8(ascii: "#")) != nil {
       poundCount += 1
     }
@@ -2072,6 +2074,15 @@ extension Lexer.Cursor {
 //        // delimiter.
 //        throw DelimiterLexError(.unprintableASCII, resumeAt: cursor.successor())
 
+      case UInt8(ascii: "("):
+        parenCount += 1
+
+      case UInt8(ascii: ")"):
+        if parenCount == 0 {
+          return nil
+        }
+        parenCount -= 1
+
       default:
         continue DELIMITLOOP
       }

Sources/SwiftParser/Lookahead.swift

@@ -146,7 +146,7 @@ extension Parser.Lookahead {
 
     repeat {
       self.eat(.atSign)
-      self.consumeIdentifier()
+      self.consumeIdentifierOrRethrows()
       if self.consume(if: .leftParen) != nil {
         while !self.at(.eof), !self.at(.rightParen), !self.at(.poundEndifKeyword) {
           self.skipSingle()

Sources/SwiftParser/Names.swift

@@ -330,6 +330,19 @@ extension TokenConsumer {
     }
   }
 
+  mutating func consumeIdentifierOrRethrows() -> Token {
+    switch self.currentToken.tokenKind {
+    case .selfKeyword,
+        .capitalSelfKeyword,
+        .anyKeyword,
+        .identifier,
+        .rethrowsKeyword:
+      return self.consumeAnyToken()
+    default:
+      return self.missingToken(.identifier)
+    }
+  }
+
   mutating func consumeInteger() -> Token {
     switch self.currentToken.tokenKind {
     case .integerLiteral:

Sources/SwiftParser/Types.swift

@@ -807,41 +807,57 @@ extension Parser {
 
     var attributeProgress = LoopProgressCondition()
     while self.at(.atSign) && attributeProgress.evaluate(currentToken) {
-      elements.append(RawSyntax(self.parseTypeAttribute()))
+      elements.append(self.parseTypeAttribute())
     }
     return RawAttributeListSyntax(elements: elements, arena: self.arena)
   }
 
   @_spi(RawSyntax)
-  public mutating func parseTypeAttribute() -> RawAttributeSyntax {
-    let at = self.eat(.atSign)
-    let ident = self.consumeIdentifier()
-    if let attr = Parser.TypeAttribute(rawValue: ident.tokenText) {
-      // Ok, it is a valid attribute, eat it, and then process it.
-      if case .convention = attr {
-        let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
-        let convention = self.consumeIdentifier()
-        let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
-        return RawAttributeSyntax(
+  public mutating func parseTypeAttribute() -> RawSyntax {
+    guard let typeAttr = Parser.TypeAttribute(rawValue: self.peek().tokenText) else {
+      return RawSyntax(self.parseCustomAttribute())
+    }
+
+    switch typeAttr {
+    case .differentiable:
+      return RawSyntax(self.parseDifferentiableAttribute())
+
+    case .convention:
+      let at = self.eat(.atSign)
+      let ident = self.consumeIdentifier()
+      let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
+      let argument = self.consumeIdentifier()
+
+      let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
+      return RawSyntax(
+        RawAttributeSyntax(
           atSignToken: at,
           attributeName: ident,
           unexpectedBeforeLeftParen,
           leftParen: leftParen,
-          argument: RawSyntax(convention),
+          argument: RawSyntax(argument),
           unexpectedBeforeRightParen,
           rightParen: rightParen,
           tokenList: nil,
-          arena: self.arena)
-      }
+          arena: self.arena
+        )
+      )
+
+    default:
+      let at = self.eat(.atSign)
+      let ident = self.consumeIdentifier()
+      return RawSyntax(
+        RawAttributeSyntax(
+          atSignToken: at,
+          attributeName: ident,
+          leftParen: nil,
+          argument: nil,
+          rightParen: nil,
+          tokenList: nil,
+          arena: self.arena
+        )
+      )
     }
-    return RawAttributeSyntax(
-      atSignToken: at,
-      attributeName: ident,
-      leftParen: nil,
-      argument: nil,
-      rightParen: nil,
-      tokenList: nil,
-      arena: self.arena)
   }
 }
 

Tests/SwiftParserTest/Attributes.swift

@@ -63,6 +63,36 @@ final class AttributeTests: XCTestCase {
       """
       @objc(zeroArg)
       class A { }
+
+      @objc(:::::)
+      func f(_: Int, _: Int, _: Int, _: Int, _: Int) { }
+      """
+    )
+  }
+
+  func testRethrowsAttribute() {
+    AssertParse(
+      """
+      @rethrows
+      protocol P { }
+      """
+    )
+  }
+
+  func testAutoclosureAttribute() {
+    AssertParse(
+      """
+      func f(in: @autoclosure () -> Int) { }
+      func g(in: @autoclosure @escaping () -> Int) { }
+      """
+    )
+  }
+
+  func testDifferentiableAttribute() {
+    AssertParse(
+      """
+      func f(in: @differentiable(reverse) (Int) -> Int) { }
+      func f(in: @differentiable(reverse, wrt: a) (Int) -> Int) { }
       """
     )
   }

Tests/SwiftParserTest/Expressions.swift

@@ -86,6 +86,7 @@ final class ExpressionTests: XCTestCase {
     AssertParse(
       """
       #file
+      #fileID
       (#line)
       #column
       #function

Tests/SwiftParserTest/LexerTests.swift

@@ -130,6 +130,7 @@ public class LexerTests: XCTestCase {
     0xff.0p2
     -0xff.0p2
     +0xff.0p2
+    0x1.921fb4p1
     """
     data.withUTF8 { buf in
       let lexemes = Lexer.lex(buf)
@@ -146,6 +147,7 @@ public class LexerTests: XCTestCase {
         lexeme(.floatingLiteral, "0xff.0p2"),
         lexeme(.prefixOperator, "\n+", leading: 1),
         lexeme(.floatingLiteral, "0xff.0p2"),
+        lexeme(.floatingLiteral, "\n0x1.921fb4p1", leading: 1),
         lexeme(.eof, ""),
       ])
     }
@@ -347,6 +349,13 @@ public class LexerTests: XCTestCase {
         lexeme(.pound, "#"),
         lexeme(.eof, ""),
       ]),
+      ("/a)/", [
+        lexeme(.prefixOperator, "/"),
+        lexeme(.identifier, "a"),
+        lexeme(.rightParen, ")"),
+        lexeme(.postfixOperator, "/"),
+        lexeme(.eof, ""),
+      ]),
     ]
     for (fixture, expectation) in fixtures {
       var fixture = fixture