@@ -16,10 +16,28 @@ typealias RawTriviaPieceBuffer = UnsafeBufferPointer<RawTriviaPiece>
16
16
/// Node data for RawSyntax tree. Tagged union plus common data.
17
17
internal struct RawSyntaxData {
18
18
internal enum Payload {
19
+ case parsedToken( ParsedToken )
19
20
case materializedToken( MaterializedToken )
20
21
case layout( Layout )
21
22
}
22
23
24
+ /// Token with lazy trivia parsing.
25
+ ///
26
+ /// The RawSyntax's `arena` must have a valid trivia parsing function to
27
+ /// lazily materialize the leading/trailing trivia pieces.
28
+ struct ParsedToken {
29
+ var tokenKind : RawTokenKind
30
+
31
+ /// Whole text of this token including leading/trailing trivia.
32
+ var wholeText : SyntaxText
33
+
34
+ /// Range of the actual token’s text.
35
+ ///
36
+ /// Text in `wholeText` before `textRange.lowerBound` is leading trivia and
37
+ /// after `textRange.upperBound` is trailing trivia.
38
+ var textRange : Range < SyntaxText . Index >
39
+ }
40
+
23
41
/// Token typically created with `TokenSyntax.<someToken>`.
24
42
struct MaterializedToken {
25
43
var tokenKind : RawTokenKind
@@ -42,6 +60,18 @@ internal struct RawSyntaxData {
42
60
fileprivate var arenaReference : SyntaxArenaRef
43
61
}
44
62
63
+ extension RawSyntaxData . ParsedToken {
64
+ var tokenText : SyntaxText {
65
+ SyntaxText ( rebasing: wholeText [ textRange] )
66
+ }
67
+ var leadingTriviaText : SyntaxText {
68
+ SyntaxText ( rebasing: wholeText [ ..< textRange. lowerBound] )
69
+ }
70
+ var trailingTriviaText : SyntaxText {
71
+ SyntaxText ( rebasing: wholeText [ textRange. upperBound... ] )
72
+ }
73
+ }
74
+
45
75
extension RawSyntaxData . MaterializedToken {
46
76
var leadingTrivia : RawTriviaPieceBuffer {
47
77
RawTriviaPieceBuffer ( rebasing: triviaPieces [ ..< Int ( numLeadingTrivia) ] )
@@ -94,6 +124,8 @@ extension RawSyntax {
94
124
switch rawData. payload {
95
125
case . materializedToken( let dat) :
96
126
return dat. tokenKind
127
+ case . parsedToken( let dat) :
128
+ return dat. tokenKind
97
129
case . layout( _) :
98
130
preconditionFailure ( " 'tokenKind' is not available for non-token node " )
99
131
}
@@ -102,6 +134,8 @@ extension RawSyntax {
102
134
/// Token text of this node assuming this node is a token.
103
135
var rawTokenText : SyntaxText {
104
136
switch rawData. payload {
137
+ case . parsedToken( let dat) :
138
+ return dat. tokenText
105
139
case . materializedToken( let dat) :
106
140
return dat. tokenText
107
141
case . layout( _) :
@@ -112,6 +146,8 @@ extension RawSyntax {
112
146
/// The UTF-8 byte length of the leading trivia, assuming this node is a token.
113
147
var tokenLeadingTriviaByteLength : Int {
114
148
switch rawData. payload {
149
+ case . parsedToken( let dat) :
150
+ return dat. leadingTriviaText. count
115
151
case . materializedToken( let dat) :
116
152
return dat. leadingTrivia. reduce ( 0 ) { $0 + $1. byteLength }
117
153
case . layout( _) :
@@ -122,26 +158,32 @@ extension RawSyntax {
122
158
/// The UTF-8 byte length of the trailing trivia, assuming this node is a token.
123
159
var tokenTrailingTriviaByteLength : Int {
124
160
switch rawData. payload {
161
+ case . parsedToken( let dat) :
162
+ return dat. trailingTriviaText. count
125
163
case . materializedToken( let dat) :
126
164
return dat. trailingTrivia. reduce ( 0 ) { $0 + $1. byteLength }
127
165
case . layout( _) :
128
166
preconditionFailure ( " 'tokenTrailingTriviaByteLength' is not available for non-token node " )
129
167
}
130
168
}
131
169
132
- var tokenLeadingRawTriviaPieces : RawTriviaPieceBuffer {
170
+ var tokenLeadingRawTriviaPieces : [ RawTriviaPiece ] {
133
171
switch rawData. payload {
172
+ case . parsedToken( let dat) :
173
+ return self . arena. parseTrivia ( source: dat. leadingTriviaText, position: . leading)
134
174
case . materializedToken( let dat) :
135
- return dat. leadingTrivia
175
+ return Array ( dat. leadingTrivia)
136
176
case . layout( _) :
137
177
preconditionFailure ( " 'tokenLeadingRawTriviaPieces' is called on non-token raw syntax " )
138
178
}
139
179
}
140
180
141
- var tokenTrailingRawTriviaPieces : RawTriviaPieceBuffer {
181
+ var tokenTrailingRawTriviaPieces : [ RawTriviaPiece ] {
142
182
switch rawData. payload {
183
+ case . parsedToken( let dat) :
184
+ return self . arena. parseTrivia ( source: dat. trailingTriviaText, position: . trailing)
143
185
case . materializedToken( let dat) :
144
- return dat. trailingTrivia
186
+ return Array ( dat. trailingTrivia)
145
187
case . layout( _) :
146
188
preconditionFailure ( " 'tokenTrailingRawTriviaPieces' is called on non-token raw syntax " )
147
189
}
@@ -152,6 +194,7 @@ extension RawSyntax {
152
194
/// The syntax kind of this raw syntax.
153
195
var kind : SyntaxKind {
154
196
switch rawData. payload {
197
+ case . parsedToken( _) : return . token
155
198
case . materializedToken( _) : return . token
156
199
case . layout( let dat) : return dat. kind
157
200
}
@@ -175,8 +218,11 @@ extension RawSyntax {
175
218
/// Child nodes.
176
219
var children : RawSyntaxBuffer {
177
220
switch rawData. payload {
178
- case . materializedToken( _) : return . init( start: nil , count: 0 )
179
- case . layout( let dat) : return dat. layout
221
+ case . parsedToken( _) ,
222
+ . materializedToken( _) :
223
+ return . init( start: nil , count: 0 )
224
+ case . layout( let dat) :
225
+ return dat. layout
180
226
}
181
227
}
182
228
@@ -197,8 +243,11 @@ extension RawSyntax {
197
243
/// Total number of nodes in this sub-tree, including `self` node.
198
244
var totalNodes : Int {
199
245
switch rawData. payload {
200
- case . materializedToken( _) : return 1
201
- case . layout( let dat) : return dat. descendantCount + 1
246
+ case . parsedToken( _) ,
247
+ . materializedToken( _) :
248
+ return 1
249
+ case . layout( let dat) :
250
+ return dat. descendantCount + 1
202
251
}
203
252
}
204
253
@@ -226,6 +275,7 @@ extension RawSyntax {
226
275
/// Sum of text byte lengths of all descendant token nodes.
227
276
var byteLength : Int {
228
277
switch rawData. payload {
278
+ case . parsedToken( let dat) : return dat. wholeText. count
229
279
case . materializedToken( let dat) : return Int ( dat. byteLength)
230
280
case . layout( let dat) : return dat. byteLength
231
281
}
@@ -237,6 +287,8 @@ extension RawSyntax {
237
287
238
288
func formTokenKind( ) -> TokenKind ? {
239
289
switch rawData. payload {
290
+ case . parsedToken( let dat) :
291
+ return TokenKind . fromRaw ( kind: dat. tokenKind, text: dat. tokenText)
240
292
case . materializedToken( let dat) :
241
293
return TokenKind . fromRaw ( kind: dat. tokenKind, text: dat. tokenText)
242
294
case . layout( _) :
@@ -255,13 +307,13 @@ extension RawSyntax {
255
307
}
256
308
257
309
/// Returns the leading `Trivia`, assuming this node is a token.
258
- func formTokenLeadingTrivia( ) -> Trivia ? {
310
+ func formTokenLeadingTrivia( ) -> Trivia {
259
311
return Trivia ( pieces: tokenLeadingRawTriviaPieces. map ( { TriviaPiece ( raw: $0) } ) )
260
312
}
261
313
262
314
/// Returns the trailing `Trivia`, assuming this node is a token.
263
315
/// - Returns: nil if called on a layout node.
264
- func formTokenTrailingTrivia( ) -> Trivia ? {
316
+ func formTokenTrailingTrivia( ) -> Trivia {
265
317
return Trivia ( pieces: tokenTrailingRawTriviaPieces. map ( { TriviaPiece ( raw: $0) } ) )
266
318
}
267
319
@@ -280,7 +332,14 @@ extension RawSyntax {
280
332
/// Assuming this node is a token, returns a `RawSyntax` node with the same
281
333
/// source text but with the token kind changed to `newValue`.
282
334
func withTokenKind( _ newValue: TokenKind ) -> RawSyntax {
283
- switch payload {
335
+ switch rawData. payload {
336
+ case . parsedToken( _) :
337
+ // The wholeText can't be continuous anymore. Make a materialized token.
338
+ return . makeMaterializedToken(
339
+ kind: newValue,
340
+ leadingTrivia: formTokenLeadingTrivia ( ) ,
341
+ trailingTrivia: formTokenTrailingTrivia ( ) ,
342
+ arena: arena)
284
343
case . materializedToken( var payload) :
285
344
let decomposed = newValue. decomposeToRaw ( )
286
345
let rawKind = decomposed. rawKind
@@ -304,7 +363,7 @@ extension RawSyntax {
304
363
return . makeMaterializedToken(
305
364
kind: formTokenKind ( ) !,
306
365
leadingTrivia: leadingTrivia,
307
- trailingTrivia: formTokenTrailingTrivia ( ) ! ,
366
+ trailingTrivia: formTokenTrailingTrivia ( ) ,
308
367
arena: arena)
309
368
}
310
369
@@ -324,7 +383,7 @@ extension RawSyntax {
324
383
if isToken {
325
384
return . makeMaterializedToken(
326
385
kind: formTokenKind ( ) !,
327
- leadingTrivia: formTokenLeadingTrivia ( ) ! ,
386
+ leadingTrivia: formTokenLeadingTrivia ( ) ,
328
387
trailingTrivia: trailingTrivia,
329
388
arena: arena)
330
389
}
@@ -517,6 +576,9 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
517
576
/// - Parameter stream: The stream on which to output this node.
518
577
public func write< Target: TextOutputStream > ( to target: inout Target ) {
519
578
switch rawData. payload {
579
+ case . parsedToken( let dat) :
580
+ String ( syntaxText: dat. wholeText) . write ( to: & target)
581
+ break
520
582
case . materializedToken( let dat) :
521
583
for p in dat. leadingTrivia { p. write ( to: & target) }
522
584
String ( syntaxText: dat. tokenText) . write ( to: & target)
@@ -594,6 +656,8 @@ extension RawSyntax {
594
656
/// is a token node.
595
657
var tokenTextByteLength : Int {
596
658
switch rawData. payload {
659
+ case . parsedToken( let dat) :
660
+ return dat. tokenText. count
597
661
case . materializedToken( let dat) :
598
662
return dat. tokenText. count
599
663
case . layout( _) :
@@ -640,6 +704,24 @@ private func makeRawTriviaPieces(leadingTrivia: Trivia, trailingTrivia: Trivia,
640
704
}
641
705
642
706
extension RawSyntax {
707
+ /// "Designated" factory method to create a parsed token node.
708
+ ///
709
+ /// - Parameters:
710
+ /// - kind: Token kind.
711
+ /// - wholeText: Whole text of this token including trailing/leading trivia.
712
+ /// - textRange: Range of the token text in `wholeText`.
713
+ /// - arena: SyntaxArea to the result node data resides.
714
+ internal static func parsedToken(
715
+ kind: RawTokenKind ,
716
+ wholeText: SyntaxText ,
717
+ textRange: Range < SyntaxText . Index > ,
718
+ arena: SyntaxArena
719
+ ) -> RawSyntax {
720
+ let payload = RawSyntaxData . ParsedToken (
721
+ tokenKind: kind, wholeText: wholeText, textRange: textRange)
722
+ return RawSyntax ( arena: arena, payload: . parsedToken( payload) )
723
+ }
724
+
643
725
/// "Designated" factory method to create a materialized token node.
644
726
///
645
727
/// This should not be called directly.
@@ -864,6 +946,11 @@ extension RawSyntax: CustomDebugStringConvertible {
864
946
private func debugWrite< Target: TextOutputStream > ( to target: inout Target , indent: Int , withChildren: Bool = false ) {
865
947
let childIndent = indent + 2
866
948
switch rawData. payload {
949
+ case . parsedToken( let dat) :
950
+ target. write ( " .parsedToken( " )
951
+ target. write ( String ( describing: dat. tokenKind) )
952
+ target. write ( " wholeText= \( dat. tokenText. debugDescription) " )
953
+ target. write ( " textRange= \( dat. textRange. description) " )
867
954
case . materializedToken( let dat) :
868
955
target. write ( " .materializedToken( " )
869
956
target. write ( String ( describing: dat. tokenKind) )
0 commit comments