Skip to content

Commit dab059c

Browse files
CodaFiCodaFi
authored
Merge pull request #970 from CodaFi/real-classy
Port and Update Syntax Classification Tests
2 parents 7e3dc35 + 3959975 commit dab059c

File tree

11 files changed

+117
-53
lines changed

11 files changed

+117
-53
lines changed

‎CodeGeneration/Sources/SyntaxSupport/gyb_generated/Classification.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
3232
SyntaxClassification(name: "Keyword", description: "A Swift keyword, including contextual keywords."),
3333
SyntaxClassification(name: "Identifier", description: "A generic identifier."),
3434
SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."),
35+
SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."),
3536
SyntaxClassification(name: "DollarIdentifier", description: "An identifier starting with `$` like `$0`."),
3637
SyntaxClassification(name: "IntegerLiteral", description: "An integer literal."),
3738
SyntaxClassification(name: "FloatingLiteral", description: "A floating point literal."),

‎CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1288,7 +1288,8 @@ public let DECL_NODES: [Node] = [
12881288
"SpacedBinaryOperator",
12891289
"PrefixOperator",
12901290
"PostfixOperator"
1291-
]),
1291+
],
1292+
classification: "OperatorIdentifier"),
12921293
Child(name: "OperatorPrecedenceAndTypes",
12931294
kind: "OperatorPrecedenceAndTypes",
12941295
description: "Optionally specify a precedence group and designated types.",

‎Sources/SwiftSyntax/gyb_generated/SyntaxClassification.swift

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ public enum SyntaxClassification {
2121
case identifier
2222
/// An identifier referring to a type.
2323
case typeIdentifier
24+
/// An identifier referring to an operator.
25+
case operatorIdentifier
2426
/// An identifier starting with `$` like `$0`.
2527
case dollarIdentifier
2628
/// An integer literal.
@@ -75,6 +77,8 @@ extension SyntaxClassification {
7577
return (.buildConfigId, false)
7678
case (.declModifier, 1):
7779
return (.attribute, false)
80+
case (.operatorDecl, 7):
81+
return (.operatorIdentifier, false)
7882
case (.precedenceGroupRelation, 1):
7983
return (.keyword, false)
8084
case (.precedenceGroupAssociativity, 1):
@@ -343,13 +347,13 @@ extension RawTokenKind {
343347
case .identifier:
344348
return .identifier
345349
case .unspacedBinaryOperator:
346-
return .none
350+
return .operatorIdentifier
347351
case .spacedBinaryOperator:
348-
return .none
352+
return .operatorIdentifier
349353
case .postfixOperator:
350-
return .none
354+
return .operatorIdentifier
351355
case .prefixOperator:
352-
return .none
356+
return .operatorIdentifier
353357
case .dollarIdentifier:
354358
return .dollarIdentifier
355359
case .contextualKeyword:

‎Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ extension SyntaxClassification {
2121
case .identifier: return "id"
2222
case .typeIdentifier: return "type"
2323
case .dollarIdentifier: return "dollar"
24+
case .operatorIdentifier: return "op"
2425
case .integerLiteral: return "int"
2526
case .floatingLiteral: return "float"
2627
case .stringLiteral: return "str"

‎Tests/SwiftSyntaxParserTest/ClassificationTests.swift renamed to ‎Tests/SwiftParserTest/ClassificationTests.swift

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
import XCTest
22
import SwiftSyntax
3-
import SwiftSyntaxParser
3+
import SwiftParser
44

55
public class ClassificationTests: XCTestCase {
66

77
public func testClassification() {
88
let source = "// blah.\nlet x/*yo*/ = 0"
9-
let tree = try! SyntaxParser.parse(source: source)
9+
let tree = Parser.parse(source: source)
1010
do {
1111
let classif = Array(tree.classifications)
1212
XCTAssertEqual(classif.count, 8)
@@ -84,7 +84,7 @@ public class ClassificationTests: XCTestCase {
8484

8585
do {
8686
let source = "func foo() {}"
87-
let tree = try! SyntaxParser.parse(source: source)
87+
let tree = Parser.parse(source: source)
8888
// For `classification(at:)` there's an initial walk to find the token that
8989
// the offset is contained in and the classified ranges are processed from that
9090
// token. That means that a `none` classified range would be restricted inside
@@ -97,7 +97,7 @@ public class ClassificationTests: XCTestCase {
9797

9898
public func testTokenClassification() {
9999
let source = "let x: Int"
100-
let tree = try! SyntaxParser.parse(source: source)
100+
let tree = Parser.parse(source: source)
101101
do {
102102
let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
103103
XCTAssertEqual(tokens.count, 4)
@@ -121,4 +121,56 @@ public class ClassificationTests: XCTestCase {
121121
XCTAssertEqual(classif.kind, .typeIdentifier)
122122
}
123123
}
124+
125+
public func testOperatorTokenClassification() {
126+
do {
127+
let source = "let x: Int = 4 + 5 / 6"
128+
let tree = Parser.parse(source: source)
129+
130+
let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
131+
XCTAssertEqual(tokens.count, 10)
132+
guard tokens.count == 10 else {
133+
return
134+
}
135+
let classif = tokens.map { $0.tokenClassification }
136+
XCTAssertEqual(classif[0].kind, .keyword)
137+
XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 3))
138+
XCTAssertEqual(classif[1].kind, .identifier)
139+
XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 4, length: 1))
140+
XCTAssertEqual(classif[2].kind, .none)
141+
XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 5, length: 1))
142+
XCTAssertEqual(classif[3].kind, .typeIdentifier)
143+
XCTAssertEqual(classif[3].range, ByteSourceRange(offset: 7, length: 3))
144+
XCTAssertEqual(classif[4].kind, .none)
145+
XCTAssertEqual(classif[4].range, ByteSourceRange(offset: 11, length: 1))
146+
XCTAssertEqual(classif[5].kind, .integerLiteral)
147+
XCTAssertEqual(classif[5].range, ByteSourceRange(offset: 13, length: 1))
148+
XCTAssertEqual(classif[6].kind, .operatorIdentifier)
149+
XCTAssertEqual(classif[6].range, ByteSourceRange(offset: 15, length: 1))
150+
XCTAssertEqual(classif[7].kind, .integerLiteral)
151+
XCTAssertEqual(classif[7].range, ByteSourceRange(offset: 17, length: 1))
152+
XCTAssertEqual(classif[8].kind, .operatorIdentifier)
153+
XCTAssertEqual(classif[8].range, ByteSourceRange(offset: 19, length: 1))
154+
XCTAssertEqual(classif[9].kind, .integerLiteral)
155+
XCTAssertEqual(classif[9].range, ByteSourceRange(offset: 21, length: 1))
156+
}
157+
158+
do {
159+
let source = "infix operator *--*"
160+
let tree = Parser.parse(source: source)
161+
162+
let tokens = Array(tree.tokens(viewMode: .sourceAccurate))
163+
XCTAssertEqual(tokens.count, 3)
164+
guard tokens.count == 3 else {
165+
return
166+
}
167+
let classif = tokens.map { $0.tokenClassification }
168+
XCTAssertEqual(classif[0].kind, .keyword)
169+
XCTAssertEqual(classif[0].range, ByteSourceRange(offset: 0, length: 5))
170+
XCTAssertEqual(classif[1].kind, .keyword)
171+
XCTAssertEqual(classif[1].range, ByteSourceRange(offset: 6, length: 8))
172+
XCTAssertEqual(classif[2].kind, .operatorIdentifier)
173+
XCTAssertEqual(classif[2].range, ByteSourceRange(offset: 15, length: 4))
174+
}
175+
}
124176
}

‎gyb_syntax_support/Classification.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ def __init__(self, name, description):
2626
SyntaxClassification('TypeIdentifier', description='''
2727
An identifier referring to a type.
2828
'''),
29+
SyntaxClassification('OperatorIdentifier', description='''
30+
An identifier referring to an operator.
31+
'''),
2932
SyntaxClassification('DollarIdentifier', description='''
3033
An identifier starting with `$` like `$0`.
3134
'''),

‎gyb_syntax_support/DeclNodes.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -744,6 +744,7 @@
744744
'''),
745745
Child('OperatorKeyword', kind='OperatorToken'),
746746
Child('Identifier', kind='Token', name_for_diagnostics='name',
747+
classification='OperatorIdentifier',
747748
token_choices=[
748749
'UnspacedBinaryOperatorToken',
749750
'SpacedBinaryOperatorToken',

‎gyb_syntax_support/Token.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -399,14 +399,16 @@ def macro_name(self):
399399
Misc('Identifier', 'identifier', name_for_diagnostics='identifier',
400400
classification='Identifier', serialization_code=105),
401401
Misc('UnspacedBinaryOperator', 'oper_binary_unspaced',
402-
name_for_diagnostics='binary operator', serialization_code=107),
402+
name_for_diagnostics='binary operator',
403+
classification='OperatorIdentifier', serialization_code=107),
403404
Misc('SpacedBinaryOperator', 'oper_binary_spaced',
404-
name_for_diagnostics='binary operator', serialization_code=108,
405+
name_for_diagnostics='binary operator',
406+
classification='OperatorIdentifier', serialization_code=108,
405407
requires_leading_space=True, requires_trailing_space=True),
406408
Misc('PostfixOperator', 'oper_postfix', name_for_diagnostics='postfix operator',
407-
serialization_code=110),
409+
classification='OperatorIdentifier', serialization_code=110),
408410
Misc('PrefixOperator', 'oper_prefix', name_for_diagnostics='prefix operator',
409-
serialization_code=109),
411+
classification='OperatorIdentifier', serialization_code=109),
410412
Misc('DollarIdentifier', 'dollarident', name_for_diagnostics='dollar identifier',
411413
classification='DollarIdentifier', serialization_code=106),
412414

‎lit_tests/coloring.swift

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ class MyCls {
6060
subscript (i : Int, j : Int) -> Int {
6161
// CHECK: <kw>get</kw> {
6262
get {
63-
// CHECK: <kw>return</kw> <id>i</id> + <id>j</id>
63+
// CHECK: <kw>return</kw> <id>i</id> <op>+</op> <id>j</id>
6464
return i + j
6565
}
6666
// CHECK: <kw>set</kw>(<id>v</id>) {
6767
set(v) {
68-
// CHECK: <id>v</id> + <id>i</id> - <id>j</id>
68+
// CHECK: <id>v</id> <op>+</op> <id>i</id> <op>-</op> <id>j</id>
6969
v + i - j
7070
}
7171
}
@@ -88,13 +88,13 @@ class Attributes {
8888
// CHECK: <attr-builtin>@IBOutlet</attr-builtin> <attr-builtin>@objc</attr-builtin> <kw>var</kw> <id>v3</id>: <type>String</type>
8989
@IBOutlet @objc var v3: String
9090

91-
// CHECK: <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <kw>func</kw> <id>f1</id>() {}
91+
// CHECK: <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <kw>func</kw> <id>f1</id>() {}
9292
@available(*, unavailable) func f1() {}
9393

94-
// CHECK: <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <attr-builtin>@IBAction</attr-builtin> <kw>func</kw> <id>f2</id>() {}
94+
// CHECK: <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <attr-builtin>@IBAction</attr-builtin> <kw>func</kw> <id>f2</id>() {}
9595
@available(*, unavailable) @IBAction func f2() {}
9696

97-
// CHECK: <attr-builtin>@IBAction</attr-builtin> <attr-builtin>@available</attr-builtin>(*, <id>unavailable</id>) <kw>func</kw> <id>f3</id>() {}
97+
// CHECK: <attr-builtin>@IBAction</attr-builtin> <attr-builtin>@available</attr-builtin>(<op>*</op>, <id>unavailable</id>) <kw>func</kw> <id>f3</id>() {}
9898
@IBAction @available(*, unavailable) func f3() {}
9999

100100
// CHECK: <kw>mutating</kw> <kw>func</kw> <id>func_mutating_1</id>() {}
@@ -160,7 +160,7 @@ protocol Prot {
160160
var protocolProperty2: Int { get set }
161161
}
162162

163-
// CHECK: <kw>infix</kw> <kw>operator</kw> *-* : <id>FunnyPrecedence</id>{{$}}
163+
// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*-*</op> : <id>FunnyPrecedence</id>{{$}}
164164
infix operator *-* : FunnyPrecedence
165165

166166
// CHECK: <kw>precedencegroup</kw> <id>FunnyPrecedence</id>
@@ -171,19 +171,19 @@ precedencegroup FunnyPrecedence {
171171
higherThan: MultiplicationPrecedence
172172
}
173173

174-
// CHECK: <kw>func</kw> *-*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
174+
// CHECK: <kw>func</kw> <op>*-*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
175175
func *-*(l: Int, r: Int) -> Int { return l }
176176

177-
// CHECK: <kw>infix</kw> <kw>operator</kw> *-+* : <id>FunnyPrecedence</id>
177+
// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*-+*</op> : <id>FunnyPrecedence</id>
178178
infix operator *-+* : FunnyPrecedence
179179

180-
// CHECK: <kw>func</kw> *-+*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
180+
// CHECK: <kw>func</kw> <op>*-+*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
181181
func *-+*(l: Int, r: Int) -> Int { return l }
182182

183-
// CHECK: <kw>infix</kw> <kw>operator</kw> *--*{{$}}
183+
// CHECK: <kw>infix</kw> <kw>operator</kw> <op>*--*</op>{{$}}
184184
infix operator *--*
185185

186-
// CHECK: <kw>func</kw> *--*(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
186+
// CHECK: <kw>func</kw> <op>*--*</op>(<id>l</id>: <type>Int</type>, <id>r</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>l</id> }{{$}}
187187
func *--*(l: Int, r: Int) -> Int { return l }
188188

189189
// CHECK: <kw>protocol</kw> <id>Prot2</id> : <type>Prot</type> {}
@@ -277,7 +277,7 @@ class Observers {
277277

278278
// CHECK: <kw>func</kw> <id>test3</id>(<id>o</id>: <type>AnyObject</type>) {
279279
func test3(o: AnyObject) {
280-
// CHECK: <kw>_</kw> = <id>o</id> <kw>is</kw> <type>MyCls</type> ? <id>o</id> <kw>as</kw> <type>MyCls</type> : <id>o</id> <kw>as</kw>! <type>MyCls</type> <kw>as</kw> <type>MyCls</type> + <int>1</int>
280+
// CHECK: <kw>_</kw> = <id>o</id> <kw>is</kw> <type>MyCls</type> ? <id>o</id> <kw>as</kw> <type>MyCls</type> : <id>o</id> <kw>as</kw>! <type>MyCls</type> <kw>as</kw> <type>MyCls</type> <op>+</op> <int>1</int>
281281
_ = o is MyCls ? o as MyCls : o as! MyCls as MyCls + 1
282282
}
283283

@@ -293,11 +293,11 @@ class MySubClass : MyCls {
293293
// CHECK: <kw>var</kw> <id>g1</id> = { (<id>x</id>: <type>Int</type>) -> <type>Int</type> <kw>in</kw> <kw>return</kw> <int>0</int> }
294294
var g1 = { (x: Int) -> Int in return 0 }
295295

296-
// CHECK: <kw>infix</kw> <kw>operator</kw> ~~ {
296+
// CHECK: <kw>infix</kw> <kw>operator</kw> <op>~~</op> {
297297
infix operator ~~ {}
298-
// CHECK: <kw>prefix</kw> <kw>operator</kw> *~~ {
298+
// CHECK: <kw>prefix</kw> <kw>operator</kw> <op>*~~</op> {
299299
prefix operator *~~ {}
300-
// CHECK: <kw>postfix</kw> <kw>operator</kw> ~~* {
300+
// CHECK: <kw>postfix</kw> <kw>operator</kw> <op>~~*</op> {
301301
postfix operator ~~* {}
302302

303303
func test_defer() {
@@ -322,13 +322,13 @@ func funcTakingIn(in internalName: Int) {}
322322
_ = 123
323323
// CHECK: <int>123</int>
324324
_ = -123
325-
// CHECK: -<int>123</int>
325+
// CHECK: <op>-</op><int>123</int>
326326
_ = -1
327-
// CHECK: -<int>1</int>
327+
// CHECK: <op>-</op><int>1</int>
328328
_ = -0x123
329-
// CHECK: -<int>0x123</int>
329+
// CHECK: <op>-</op><int>0x123</int>
330330
_ = -3.1e-5
331-
// CHECK: <float>3.1e-5</float>
331+
// CHECK: <op>-</op><float>3.1e-5</float>
332332
333333
"--\"\(x) --"
334334
// CHECK: <str>"--\"</str>\<anchor>(</anchor><id>x</id><anchor>)</anchor><str> --"</str>
@@ -397,5 +397,5 @@ protocol FakeClassRestrictedProtocol : `class` {}
397397
// CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> {}
398398
func foo() -> some P {}
399399

400-
// CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> & <type>Q</type> {}
400+
// CHECK: <kw>func</kw> <id>foo</id>() -> <kw>some</kw> <type>P</type> <op>&</op> <type>Q</type> {}
401401
func foo() -> some P & Q {}

‎lit_tests/coloring_comments.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func foo(x: Int, y: Int) -> Int { return x + y }
121121
// CHECK: <doc-comment-line>/// - seealso : nope</doc-comment-line>
122122
// CHECK: <doc-comment-line>/// - seealso nope</doc-comment-line>
123123
// CHECK: <doc-comment-line>/// - returns: `x + y`</doc-comment-line>
124-
// CHECK: <kw>func</kw> <id>foo</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> + <id>y</id> }
124+
// CHECK: <kw>func</kw> <id>foo</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> <op>+</op> <id>y</id> }
125125

126126

127127
/// Brief.
@@ -151,7 +151,7 @@ func bar(x: Int, y: Int) -> Int { return x + y }
151151
// CHECK: <doc-comment-line>/// - NOTE: NOTE2</doc-comment-line>
152152
// CHECK: <doc-comment-line>/// - note: Not a Note field (not at top level)</doc-comment-line>
153153
// CHECK: <doc-comment-line>/// - returns: `x + y`</doc-comment-line>
154-
// CHECK: <kw>func</kw> <id>bar</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> + <id>y</id> }
154+
// CHECK: <kw>func</kw> <id>bar</id>(<id>x</id>: <type>Int</type>, <id>y</id>: <type>Int</type>) -> <type>Int</type> { <kw>return</kw> <id>x</id> <op>+</op> <id>y</id> }
155155

156156
/**
157157
Does pretty much nothing.

0 commit comments

Comments
 (0)