Classify Operator Identifiers

Author: CodaFi

CodeGeneration/Sources/SyntaxSupport/gyb_generated/Classification.swift

@@ -32,6 +32,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
   SyntaxClassification(name: "Keyword", description: "A Swift keyword, including contextual keywords."),
   SyntaxClassification(name: "Identifier", description: "A generic identifier."),
   SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."),
+  SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."),
   SyntaxClassification(name: "DollarIdentifier", description: "An identifier starting with `$` like `$0`."),
   SyntaxClassification(name: "IntegerLiteral", description: "An integer literal."),
   SyntaxClassification(name: "FloatingLiteral", description: "A floating point literal."),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

@@ -1288,7 +1288,8 @@ public let DECL_NODES: [Node] = [
                  "SpacedBinaryOperator",
                  "PrefixOperator",
                  "PostfixOperator"
-               ]),
+               ],
+               classification: "OperatorIdentifier"),
          Child(name: "OperatorPrecedenceAndTypes",
                kind: "OperatorPrecedenceAndTypes",
                description: "Optionally specify a precedence group and designated types.",

Sources/SwiftSyntax/gyb_generated/SyntaxClassification.swift

@@ -21,6 +21,8 @@ public enum SyntaxClassification {
   case identifier
   /// An identifier referring to a type. 
   case typeIdentifier
+  /// An identifier referring to an operator. 
+  case operatorIdentifier
   /// An identifier starting with `$` like `$0`. 
   case dollarIdentifier
   /// An integer literal. 
@@ -75,6 +77,8 @@ extension SyntaxClassification {
         return (.buildConfigId, false)
       case (.declModifier, 1):
         return (.attribute, false)
+      case (.operatorDecl, 7):
+        return (.operatorIdentifier, false)
       case (.precedenceGroupRelation, 1):
         return (.keyword, false)
       case (.precedenceGroupAssociativity, 1):
@@ -343,13 +347,13 @@ extension RawTokenKind {
     case .identifier:
       return .identifier
     case .unspacedBinaryOperator:
-      return .none
+      return .operatorIdentifier
     case .spacedBinaryOperator:
-      return .none
+      return .operatorIdentifier
     case .postfixOperator:
-      return .none
+      return .operatorIdentifier
     case .prefixOperator:
-      return .none
+      return .operatorIdentifier
     case .dollarIdentifier:
       return .dollarIdentifier
     case .contextualKeyword:

Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift

@@ -21,6 +21,7 @@ extension SyntaxClassification {
     case .identifier: return "id"
     case .typeIdentifier: return "type"
     case .dollarIdentifier: return "dollar"
+    case .operatorIdentifier: return "op"
     case .integerLiteral: return "int"
     case .floatingLiteral: return "float"
     case .stringLiteral: return "str"

Tests/SwiftParserTest/ClassificationTests.swift

@@ -121,4 +121,56 @@ public class ClassificationTests: XCTestCase {
       XCTAssertEqual(classif.kind, .typeIdentifier)
     }
   }
+
+  public func testOperatorTokenClassification() {
+    do {
+      let source = "let x: Int = 4 + 5 / 6"
+      let tree = Parser.parse(source: source)
+
+      let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
+      XCTAssertEqual(tokens.count, 10)
+      guard tokens.count == 10 else {
+        return
+      }
+      let classif = tokens.map { $0.tokenClassification }
+      XCTAssertEqual(classif[0].kind, .keyword)
+      XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 3))
+      XCTAssertEqual(classif[1].kind, .identifier)
+      XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 4, length: 1))
+      XCTAssertEqual(classif[2].kind, .none)
+      XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 5, length: 1))
+      XCTAssertEqual(classif[3].kind, .typeIdentifier)
+      XCTAssertEqual(classif[3].range, ByteSourceRange(offset: 7, length: 3))
+      XCTAssertEqual(classif[4].kind, .none)
+      XCTAssertEqual(classif[4].range, ByteSourceRange(offset: 11, length: 1))
+      XCTAssertEqual(classif[5].kind, .integerLiteral)
+      XCTAssertEqual(classif[5].range, ByteSourceRange(offset: 13, length: 1))
+      XCTAssertEqual(classif[6].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[6].range, ByteSourceRange(offset: 15, length: 1))
+      XCTAssertEqual(classif[7].kind, .integerLiteral)
+      XCTAssertEqual(classif[7].range, ByteSourceRange(offset: 17, length: 1))
+      XCTAssertEqual(classif[8].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[8].range, ByteSourceRange(offset: 19, length: 1))
+      XCTAssertEqual(classif[9].kind, .integerLiteral)
+      XCTAssertEqual(classif[9].range, ByteSourceRange(offset: 21, length: 1))
+    }
+
+    do {
+      let source = "infix operator *--*"
+      let tree = Parser.parse(source: source)
+
+      let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
+      XCTAssertEqual(tokens.count, 3)
+      guard tokens.count == 3 else {
+        return
+      }
+      let classif = tokens.map { $0.tokenClassification }
+      XCTAssertEqual(classif[0].kind, .keyword)
+      XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 5))
+      XCTAssertEqual(classif[1].kind, .keyword)
+      XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 6, length: 8))
+      XCTAssertEqual(classif[2].kind, .operatorIdentifier)
+      XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 15, length: 4))
+    }
+  }
 }

gyb_syntax_support/Classification.py

@@ -26,6 +26,9 @@ def __init__(self, name, description):
     SyntaxClassification('TypeIdentifier', description='''
     An identifier referring to a type.
     '''),
+    SyntaxClassification('OperatorIdentifier', description='''
+    An identifier referring to an operator.
+    '''),
     SyntaxClassification('DollarIdentifier', description='''
     An identifier starting with `$` like `$0`.
     '''),

gyb_syntax_support/DeclNodes.py

@@ -744,6 +744,7 @@
                    '''),
              Child('OperatorKeyword', kind='OperatorToken'),
              Child('Identifier', kind='Token', name_for_diagnostics='name',
+                   classification='OperatorIdentifier',
                    token_choices=[
                        'UnspacedBinaryOperatorToken',
                        'SpacedBinaryOperatorToken',

gyb_syntax_support/Token.py

@@ -399,14 +399,16 @@ def macro_name(self):
     Misc('Identifier', 'identifier', name_for_diagnostics='identifier',
          classification='Identifier', serialization_code=105),
     Misc('UnspacedBinaryOperator', 'oper_binary_unspaced',
-         name_for_diagnostics='binary operator', serialization_code=107),
+         name_for_diagnostics='binary operator',
+         classification='OperatorIdentifier', serialization_code=107),
     Misc('SpacedBinaryOperator', 'oper_binary_spaced',
-         name_for_diagnostics='binary operator', serialization_code=108,
+         name_for_diagnostics='binary operator',
+         classification='OperatorIdentifier', serialization_code=108,
          requires_leading_space=True, requires_trailing_space=True),
     Misc('PostfixOperator', 'oper_postfix', name_for_diagnostics='postfix operator',
-         serialization_code=110),
+         classification='OperatorIdentifier', serialization_code=110),
     Misc('PrefixOperator', 'oper_prefix', name_for_diagnostics='prefix operator',
-         serialization_code=109),
+         classification='OperatorIdentifier', serialization_code=109),
     Misc('DollarIdentifier', 'dollarident', name_for_diagnostics='dollar identifier',
          classification='DollarIdentifier', serialization_code=106),