Port and Update Syntax Classification Tests

CodeGeneration/Sources/SyntaxSupport/gyb_generated/Classification.swift

@@ -32,6 +32,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
   SyntaxClassification(name: "Keyword", description: "A Swift keyword, including contextual keywords."),
   SyntaxClassification(name: "Identifier", description: "A generic identifier."),
   SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."),
+  SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."),
   SyntaxClassification(name: "DollarIdentifier", description: "An identifier starting with `$` like `$0`."),
   SyntaxClassification(name: "IntegerLiteral", description: "An integer literal."),
   SyntaxClassification(name: "FloatingLiteral", description: "A floating point literal."),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

@@ -1288,7 +1288,8 @@ public let DECL_NODES: [Node] = [
                  "SpacedBinaryOperator",
                  "PrefixOperator",
                  "PostfixOperator"
-               ]),
+               ],
+               classification: "OperatorIdentifier"),
          Child(name: "OperatorPrecedenceAndTypes",
                kind: "OperatorPrecedenceAndTypes",
                description: "Optionally specify a precedence group and designated types.",

Sources/SwiftSyntax/gyb_generated/SyntaxClassification.swift

@@ -21,6 +21,8 @@ public enum SyntaxClassification {
   case identifier
   /// An identifier referring to a type. 
   case typeIdentifier
+  /// An identifier referring to an operator. 
+  case operatorIdentifier
   /// An identifier starting with `$` like `$0`. 
   case dollarIdentifier
   /// An integer literal. 
@@ -75,6 +77,8 @@ extension SyntaxClassification {
         return (.buildConfigId, false)
       case (.declModifier, 1):
         return (.attribute, false)
+      case (.operatorDecl, 7):
+        return (.operatorIdentifier, false)
       case (.precedenceGroupRelation, 1):
         return (.keyword, false)
       case (.precedenceGroupAssociativity, 1):
@@ -343,13 +347,13 @@ extension RawTokenKind {
     case .identifier:
       return .identifier
     case .unspacedBinaryOperator:
-      return .none
+      return .operatorIdentifier
     case .spacedBinaryOperator:
-      return .none
+      return .operatorIdentifier
     case .postfixOperator:
-      return .none
+      return .operatorIdentifier
     case .prefixOperator:
-      return .none
+      return .operatorIdentifier
     case .dollarIdentifier:
       return .dollarIdentifier
     case .contextualKeyword:

Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift

@@ -21,6 +21,7 @@ extension SyntaxClassification {
     case .identifier: return "id"
     case .typeIdentifier: return "type"
     case .dollarIdentifier: return "dollar"
+    case .operatorIdentifier: return "op"
     case .integerLiteral: return "int"
     case .floatingLiteral: return "float"
     case .stringLiteral: return "str"

Tests/SwiftSyntaxParserTest/ClassificationTests.swift → Tests/SwiftParserTest/ClassificationTests.swift

@@ -1,12 +1,12 @@
 import XCTest
 import SwiftSyntax
-import SwiftSyntaxParser
+import SwiftParser
 
 public class ClassificationTests: XCTestCase {
 
   public func testClassification() {
     let source = "// blah.\nlet x/*yo*/ = 0"
-    let tree = try! SyntaxParser.parse(source: source)
+    let tree = Parser.parse(source: source)
     do {
       let classif = Array(tree.classifications)
       XCTAssertEqual(classif.count, 8)
@@ -84,7 +84,7 @@ public class ClassificationTests: XCTestCase {
 
     do {
       let source = "func foo() {}"
-      let tree = try! SyntaxParser.parse(source: source)
+      let tree = Parser.parse(source: source)
       // For `classification(at:)` there's an initial walk to find the token that
       // the offset is contained in and the classified ranges are processed from that
       // token. That means that a `none` classified range would be restricted inside
@@ -97,7 +97,7 @@ public class ClassificationTests: XCTestCase {
 
   public func testTokenClassification() {
     let source = "let x: Int"
-    let tree = try! SyntaxParser.parse(source: source)
+    let tree = Parser.parse(source: source)
     do {
       let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
       XCTAssertEqual(tokens.count, 4)
@@ -121,4 +121,56 @@ public class ClassificationTests: XCTestCase {
       XCTAssertEqual(classif.kind, .typeIdentifier)
     }
   }
+
+  public func testOperatorTokenClassification() {
+    do {
+      let source = "let x: Int = 4 + 5 / 6"
+      let tree = Parser.parse(source: source)
+
+      let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
+      XCTAssertEqual(tokens.count, 10)
+      guard tokens.count == 10 else {
+        return
+      }
+      let classif = tokens.map { $0.tokenClassification }
+      XCTAssertEqual(classif[0].kind, .keyword)
+      XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 3))
+      XCTAssertEqual(classif[1].kind, .identifier)
+      XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 4, length: 1))
+      XCTAssertEqual(classif[2].kind, .none)
+      XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 5, length: 1))
+      XCTAssertEqual(classif[3].kind, .typeIdentifier)
+      XCTAssertEqual(classif[3].range, ByteSourceRange(offset: 7, length: 3))
+      XCTAssertEqual(classif[4].kind, .none)
+      XCTAssertEqual(classif[4].range, ByteSourceRange(offset: 11, length: 1))
+      XCTAssertEqual(classif[5].kind, .integerLiteral)
+      XCTAssertEqual(classif[5].range, ByteSourceRange(offset: 13, length: 1))
+      XCTAssertEqual(classif[6].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[6].range, ByteSourceRange(offset: 15, length: 1))
+      XCTAssertEqual(classif[7].kind, .integerLiteral)
+      XCTAssertEqual(classif[7].range, ByteSourceRange(offset: 17, length: 1))
+      XCTAssertEqual(classif[8].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[8].range, ByteSourceRange(offset: 19, length: 1))
+      XCTAssertEqual(classif[9].kind, .integerLiteral)
+      XCTAssertEqual(classif[9].range, ByteSourceRange(offset: 21, length: 1))
+    }
+
+    do {
+      let source = "infix operator *--*"
+      let tree = Parser.parse(source: source)
+
+      let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
+      XCTAssertEqual(tokens.count, 3)
+      guard tokens.count == 3 else {
+        return
+      }
+      let classif = tokens.map { $0.tokenClassification }
+      XCTAssertEqual(classif[0].kind, .keyword)
+      XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 5))
+      XCTAssertEqual(classif[1].kind, .keyword)
+      XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 6, length: 8))
+      XCTAssertEqual(classif[2].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 15, length: 4))
+    }
+  }
 }

gyb_syntax_support/Classification.py

@@ -26,6 +26,9 @@ def __init__(self, name, description):
     SyntaxClassification('TypeIdentifier', description='''
     An identifier referring to a type.
     '''),
+    SyntaxClassification('OperatorIdentifier', description='''
+    An identifier referring to an operator.
+    '''),
     SyntaxClassification('DollarIdentifier', description='''
     An identifier starting with `$` like `$0`.
     '''),

gyb_syntax_support/DeclNodes.py

@@ -744,6 +744,7 @@
                    '''),
              Child('OperatorKeyword', kind='OperatorToken'),
              Child('Identifier', kind='Token', name_for_diagnostics='name',
+                   classification='OperatorIdentifier',
                    token_choices=[
                        'UnspacedBinaryOperatorToken',
                        'SpacedBinaryOperatorToken',

gyb_syntax_support/Token.py

@@ -399,14 +399,16 @@ def macro_name(self):
     Misc('Identifier', 'identifier', name_for_diagnostics='identifier',
          classification='Identifier', serialization_code=105),
     Misc('UnspacedBinaryOperator', 'oper_binary_unspaced',
-         name_for_diagnostics='binary operator', serialization_code=107),
+         name_for_diagnostics='binary operator',
+         classification='OperatorIdentifier', serialization_code=107),
     Misc('SpacedBinaryOperator', 'oper_binary_spaced',
-         name_for_diagnostics='binary operator', serialization_code=108,
+         name_for_diagnostics='binary operator',
+         classification='OperatorIdentifier', serialization_code=108,
          requires_leading_space=True, requires_trailing_space=True),
     Misc('PostfixOperator', 'oper_postfix', name_for_diagnostics='postfix operator',
-         serialization_code=110),
+         classification='OperatorIdentifier', serialization_code=110),
     Misc('PrefixOperator', 'oper_prefix', name_for_diagnostics='prefix operator',
-         serialization_code=109),
+         classification='OperatorIdentifier', serialization_code=109),
     Misc('DollarIdentifier', 'dollarident', name_for_diagnostics='dollar identifier',
          classification='DollarIdentifier', serialization_code=106),
 

lit_tests/coloring.swift

@@ -60,12 +60,12 @@ class MyCls {
     subscript (i : Int, j : Int) -> Int {
       // CHECK: <kw>get</kw> {
       get {
-        // CHECK: <kw>return</kw> <id>i</id> + <id>j</id>
+        // CHECK: <kw>return</kw> <id>i</id> <op>+</op> <id>j</id>
         return i + j
       }
       // CHECK: <kw>set</kw>(<id>v</id>) {
       set(v) {
-        // CHECK: <id>v</id> + <id>i</id> - <id>j</id>
+        // CHECK: <id>v</id> <op>+</op> <id>i</id> <op>-</op> <id>j</id>
         v + i - j
       }
     }
@@ -88,13 +88,13 @@ class Attributes {
 // CHECK: <attr-builtin>@IBOutlet</attr-builtin> <attr-builtin>@objc</attr-builtin> <kw>var</kw> <id>v3</id>: <type>String</type>
   @IBOutlet @objc var v3: String
 
-// CHECK: <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <kw>func</kw> <id>f1</id>() {}
+// CHECK: <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <kw>func</kw> <id>f1</id>() {}
   @available(*, unavailable) func f1() {}
 
-// CHECK: <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <attr-builtin>@IBAction</attr-builtin> <kw>func</kw> <id>f2</id>() {}
+// CHECK: <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <attr-builtin>@IBAction</attr-builtin> <kw>func</kw> <id>f2</id>() {}
   @available(*, unavailable) @IBAction func f2() {}
 
-// CHECK: <attr-builtin>@IBAction</attr-builtin> <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <kw>func</kw> <id>f3</id>() {}
+// CHECK: <attr-builtin>@IBAction</attr-builtin> <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <kw>func</kw> <id>f3</id>() {}
   @IBAction @available(*, unavailable) func f3() {}
 
 // CHECK: <kw>mutating</kw> <kw>func</kw> <id>func_mutating_1</id>() {}
@@ -160,7 +160,7 @@ protocol Prot {
   var protocolProperty2: Int { get set }
 }
 
-// CHECK: <kw>infix</kw> <kw>operator</kw> *-* : <id>FunnyPrecedence</id>{{$}}
+// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*-*</op> : <id>FunnyPrecedence</id>{{$}}
 infix operator *-* : FunnyPrecedence
 
 // CHECK: <kw>precedencegroup</kw> <id>FunnyPrecedence</id>
@@ -171,19 +171,19 @@ precedencegroup FunnyPrecedence {
   higherThan: MultiplicationPrecedence
 }
 
-// CHECK: <kw>func</kw> *-*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
+// CHECK: <kw>func</kw> <op>*-*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
 func *-*(l: Int, r: Int) -> Int { return l }
 
-// CHECK: <kw>infix</kw> <kw>operator</kw> *-+* : <id>FunnyPrecedence</id>
+// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*-+*</op> : <id>FunnyPrecedence</id>
 infix operator *-+* : FunnyPrecedence
 
-// CHECK: <kw>func</kw> *-+*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
+// CHECK: <kw>func</kw> <op>*-+*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
 func *-+*(l: Int, r: Int) -> Int { return l }
 
-// CHECK: <kw>infix</kw> <kw>operator</kw> *--*{{$}}
+// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*--*</op>{{$}}
 infix operator *--*
 
-// CHECK: <kw>func</kw> *--*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
+// CHECK: <kw>func</kw> <op>*--*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
 func *--*(l: Int, r: Int) -> Int { return l }
 
 // CHECK: <kw>protocol</kw> <id>Prot2</id> : <type>Prot</type> {}
@@ -277,7 +277,7 @@ class Observers {
 
 // CHECK: <kw>func</kw> <id>test3</id>(<id>o</id>: <type>AnyObject</type>) {
 func test3(o: AnyObject) {
-  // CHECK: <kw>_</kw> = <id>o</id> <kw>is</kw> <type>MyCls</type> ? <id>o</id> <kw>as</kw> <type>MyCls</type> : <id>o</id> <kw>as</kw>! <type>MyCls</type> <kw>as</kw> <type>MyCls</type> + <int>1</int>
+  // CHECK: <kw>_</kw> = <id>o</id> <kw>is</kw> <type>MyCls</type> ? <id>o</id> <kw>as</kw> <type>MyCls</type> : <id>o</id> <kw>as</kw>! <type>MyCls</type> <kw>as</kw> <type>MyCls</type> <op>+</op> <int>1</int>
   _ = o is MyCls ? o as MyCls : o as! MyCls as MyCls + 1
 }
 
@@ -293,11 +293,11 @@ class MySubClass : MyCls {
 // CHECK: <kw>var</kw> <id>g1</id> = { (<id>x</id>: <type>Int</type>) -> <type>Int</type> <kw>in</kw> <kw>return</kw> <int>0</int> }
 var g1 = { (x: Int) -> Int in return 0 }
 
-// CHECK: <kw>infix</kw> <kw>operator</kw> ~~ {
+// CHECK: <kw>infix</kw> <kw>operator</kw> <op>~~</op> {
 infix operator ~~ {}
-// CHECK: <kw>prefix</kw> <kw>operator</kw> *~~ {
+// CHECK: <kw>prefix</kw> <kw>operator</kw> <op>*~~</op> {
 prefix operator *~~ {}
-// CHECK: <kw>postfix</kw> <kw>operator</kw> ~~* {
+// CHECK: <kw>postfix</kw> <kw>operator</kw> <op>~~*</op> {
 postfix operator ~~* {}
 
 func test_defer() {
@@ -322,13 +322,13 @@ func funcTakingIn(in internalName: Int) {}
 _ = 123
 // CHECK: <int>123</int>
 _ = -123
-// CHECK: -<int>123</int>
+// CHECK: <op>-</op><int>123</int>
 _ = -1
-// CHECK: -<int>1</int>
+// CHECK: <op>-</op><int>1</int>
 _ = -0x123
-// CHECK: -<int>0x123</int>
+// CHECK: <op>-</op><int>0x123</int>
 _ = -3.1e-5
-// CHECK: <float>3.1e-5</float>
+// CHECK: <op>-</op><float>3.1e-5</float>
 
 "--\"\(x) --"
 // CHECK: <str>"--\"</str>\<anchor>(</anchor><id>x</id><anchor>)</anchor><str> --"</str>
@@ -397,5 +397,5 @@ protocol FakeClassRestrictedProtocol : `class` {}
 // CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> {}
 func foo() -> some P {}
 
-// CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> & <type>Q</type> {}
+// CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> <op>&</op> <type>Q</type> {}
 func foo() -> some P & Q {}

lit_tests/coloring_comments.swift

@@ -121,7 +121,7 @@ func foo(x: Int, y: Int) -> Int { return x + y }
 // CHECK: <doc-comment-line>/// - seealso : nope</doc-comment-line>
 // CHECK: <doc-comment-line>/// - seealso nope</doc-comment-line>
 // CHECK: <doc-comment-line>/// - returns: `x + y`</doc-comment-line>
-// CHECK: <kw>func</kw> <id>foo</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> + <id>y</id> }
+// CHECK: <kw>func</kw> <id>foo</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> <op>+</op> <id>y</id> }
 
 
 /// Brief.
@@ -151,7 +151,7 @@ func bar(x: Int, y: Int) -> Int { return x + y }
 // CHECK: <doc-comment-line>/// - NOTE: NOTE2</doc-comment-line>
 // CHECK: <doc-comment-line>///   - note: Not a Note field (not at top level)</doc-comment-line>
 // CHECK: <doc-comment-line>/// - returns: `x + y`</doc-comment-line>
-// CHECK: <kw>func</kw> <id>bar</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> + <id>y</id> }
+// CHECK: <kw>func</kw> <id>bar</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> <op>+</op> <id>y</id> }
 
 /**
   Does pretty much nothing.