Skip to content

Commit 1529d65

Browse files
ahoppenahoppen
authored
Merge pull request #18684 from ahoppen/enable-cache-token-nodes
[libSyntax] Reenable caching of token nodes
2 parents 6be2762 + 32a3a31 commit 1529d65

File tree

4 files changed

+98
-63
lines changed

4 files changed

+98
-63
lines changed

‎lib/Syntax/SyntaxArena.cpp

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ struct SyntaxArena::Implementation {
2929
/// List of pointers to the allocated RawSyntax
3030
std::vector<RawSyntax *> AllocatedRawSyntaxList;
3131

32+
/// Cached Tokens.
33+
llvm::FoldingSet<RawSyntaxCacheNode> CachedTokens;
34+
3235
Implementation() = default;
3336
void *Allocate(size_t size, size_t alignment) {
3437
return Allocator.Allocate(size, alignment);
@@ -122,6 +125,38 @@ RC<RawSyntax> RawSyntax::getToken(SyntaxArena &Arena, tok TokKind,
122125
llvm::ArrayRef<TriviaPiece> LeadingTrivia,
123126
llvm::ArrayRef<TriviaPiece> TrailingTrivia) {
124127

125-
return RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
126-
SourcePresence::Present, &Arena);
128+
// Determine whether this token is worth to cache.
129+
if (
130+
// Is string_literal with >16 length.
131+
(TokKind == tok::string_literal && Text.size() > 16) ||
132+
// Has leading comment trivia et al.
133+
any_of(LeadingTrivia,
134+
[](const syntax::TriviaPiece &T) { return T.getText().size(); }) ||
135+
// Has trailing comment trivia et al.
136+
any_of(TrailingTrivia,
137+
[](const syntax::TriviaPiece &T) { return T.getText().size(); })) {
138+
139+
// Do not use cache.
140+
return RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
141+
SourcePresence::Present, &Arena);
142+
}
143+
144+
// This node is cacheable. Get or create.
145+
auto &CachedTokens = Arena.Impl.CachedTokens;
146+
147+
llvm::FoldingSetNodeID ID;
148+
RawSyntax::Profile(ID, TokKind, Text, LeadingTrivia, TrailingTrivia);
149+
150+
void *insertPos = nullptr;
151+
if (auto existing = CachedTokens.FindNodeOrInsertPos(ID, insertPos))
152+
// Found in the cache. Just return it.
153+
return existing->get();
154+
155+
// Could not found in the cache. Create it.
156+
auto Raw = RawSyntax::make(TokKind, Text, LeadingTrivia, TrailingTrivia,
157+
SourcePresence::Present, &Arena);
158+
auto IDRef = ID.Intern(Arena.getAllocator());
159+
auto CacheNode = new (Arena) RawSyntaxCacheNode(Raw.get(), IDRef);
160+
CachedTokens.InsertNode(CacheNode, insertPos);
161+
return Raw;
127162
}

‎test/Syntax/Inputs/serialize_multiple_decls.json

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
2-
"id": 20,
2+
"id": 18,
33
"kind": "SourceFile",
44
"layout": [
55
{
6-
"id": 19,
6+
"id": 17,
77
"kind": "CodeBlockItemList",
88
"layout": [
99
{
@@ -110,11 +110,11 @@
110110
"presence": "Present"
111111
},
112112
{
113-
"id": 16,
113+
"id": 14,
114114
"kind": "CodeBlockItem",
115115
"layout": [
116116
{
117-
"id": 15,
117+
"id": 13,
118118
"kind": "StructDecl",
119119
"layout": [
120120
null,
@@ -157,11 +157,11 @@
157157
null,
158158
null,
159159
{
160-
"id": 14,
160+
"id": 12,
161161
"kind": "MemberDeclBlock",
162162
"layout": [
163163
{
164-
"id": 11,
164+
"id": 3,
165165
"tokenKind": {
166166
"kind": "l_brace"
167167
},
@@ -170,13 +170,13 @@
170170
"presence": "Present"
171171
},
172172
{
173-
"id": 12,
173+
"id": 11,
174174
"kind": "MemberDeclList",
175175
"layout": [],
176176
"presence": "Present"
177177
},
178178
{
179-
"id": 13,
179+
"id": 5,
180180
"tokenKind": {
181181
"kind": "r_brace"
182182
},
@@ -204,7 +204,7 @@
204204
"presence": "Present"
205205
},
206206
{
207-
"id": 18,
207+
"id": 16,
208208
"tokenKind": {
209209
"kind": "eof",
210210
"text": ""

‎test/Syntax/Inputs/serialize_struct_decl.json

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
{
2-
"id": 41,
2+
"id": 40,
33
"kind": "SourceFile",
44
"layout": [
55
{
6-
"id": 40,
6+
"id": 39,
77
"kind": "CodeBlockItemList",
88
"layout": [
99
{
10-
"id": 37,
10+
"id": 36,
1111
"kind": "CodeBlockItem",
1212
"layout": [
1313
{
14-
"id": 36,
14+
"id": 35,
1515
"kind": "StructDecl",
1616
"layout": [
1717
null,
@@ -66,7 +66,7 @@
6666
null,
6767
null,
6868
{
69-
"id": 35,
69+
"id": 34,
7070
"kind": "MemberDeclBlock",
7171
"layout": [
7272
{
@@ -79,7 +79,7 @@
7979
"presence": "Present"
8080
},
8181
{
82-
"id": 33,
82+
"id": 32,
8383
"kind": "MemberDeclList",
8484
"layout": [
8585
{
@@ -201,11 +201,11 @@
201201
"presence": "Present"
202202
},
203203
{
204-
"id": 32,
204+
"id": 31,
205205
"kind": "MemberDeclListItem",
206206
"layout": [
207207
{
208-
"id": 31,
208+
"id": 30,
209209
"kind": "VariableDecl",
210210
"layout": [
211211
null,
@@ -234,11 +234,11 @@
234234
"presence": "Present"
235235
},
236236
{
237-
"id": 30,
237+
"id": 29,
238238
"kind": "PatternBindingList",
239239
"layout": [
240240
{
241-
"id": 29,
241+
"id": 28,
242242
"kind": "PatternBinding",
243243
"layout": [
244244
{
@@ -264,11 +264,11 @@
264264
"presence": "Present"
265265
},
266266
{
267-
"id": 28,
267+
"id": 27,
268268
"kind": "TypeAnnotation",
269269
"layout": [
270270
{
271-
"id": 18,
271+
"id": 7,
272272
"tokenKind": {
273273
"kind": "colon"
274274
},
@@ -282,11 +282,11 @@
282282
"presence": "Present"
283283
},
284284
{
285-
"id": 27,
285+
"id": 26,
286286
"kind": "SimpleTypeIdentifier",
287287
"layout": [
288288
{
289-
"id": 19,
289+
"id": 18,
290290
"tokenKind": {
291291
"kind": "identifier",
292292
"text": "Array"
@@ -301,11 +301,11 @@
301301
"presence": "Present"
302302
},
303303
{
304-
"id": 26,
304+
"id": 25,
305305
"kind": "GenericArgumentClause",
306306
"layout": [
307307
{
308-
"id": 20,
308+
"id": 19,
309309
"tokenKind": {
310310
"kind": "l_angle"
311311
},
@@ -319,19 +319,19 @@
319319
"presence": "Present"
320320
},
321321
{
322-
"id": 24,
322+
"id": 23,
323323
"kind": "GenericArgumentList",
324324
"layout": [
325325
{
326-
"id": 23,
326+
"id": 22,
327327
"kind": "GenericArgument",
328328
"layout": [
329329
{
330-
"id": 22,
330+
"id": 21,
331331
"kind": "SimpleTypeIdentifier",
332332
"layout": [
333333
{
334-
"id": 21,
334+
"id": 20,
335335
"tokenKind": {
336336
"kind": "identifier",
337337
"text": "Int"
@@ -357,7 +357,7 @@
357357
"presence": "Present"
358358
},
359359
{
360-
"id": 25,
360+
"id": 24,
361361
"tokenKind": {
362362
"kind": "r_angle"
363363
},
@@ -394,7 +394,7 @@
394394
"presence": "Present"
395395
},
396396
{
397-
"id": 34,
397+
"id": 33,
398398
"tokenKind": {
399399
"kind": "r_brace"
400400
},
@@ -426,7 +426,7 @@
426426
"presence": "Present"
427427
},
428428
{
429-
"id": 39,
429+
"id": 38,
430430
"tokenKind": {
431431
"kind": "eof",
432432
"text": ""

0 commit comments

Comments
 (0)