Skip to content

Commit 4c17203

Browse files
nkcsgexinkcsgexi
authored
[libSyntax][SourceKit] Integrating libSyntax representation of a source file with several SourceKitd syntax requests (#14000)
The enhanced SourceKitd requests are EditorOpen and EdtiorReplaceText. In these two requests, the clients can specify a flag "key. enablesyntaxtree = 1" to get a serialize libSyntax tree with the response. To help this integration, we added a function in SyntaxParsingContext to explicitly finalize the creation of a SourceFileSyntax to incorporate the fact that SourceKit needs the tree before its destroying the parser instance. To test this integration, we diff the syntax tree serialized from the frontend action and the tree serialized from a SourceKitd response. They should be identical.
1 parent 6c8025b commit 4c17203

File tree

13 files changed

+127
-21
lines changed

13 files changed

+127
-21
lines changed

‎include/swift/Parse/Parser.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,13 @@ class Parser {
356356

357357
bool isInSILMode() const { return SIL != nullptr; }
358358

359+
/// Calling this function to finalize libSyntax tree creation without destroying
360+
/// the parser instance.
361+
void finalizeSyntaxTree() {
362+
assert(Tok.is(tok::eof) && "not done parsing yet");
363+
SyntaxContext->finalizeRoot();
364+
}
365+
359366
//===--------------------------------------------------------------------===//
360367
// Routines to save and restore parser state.
361368

‎include/swift/Syntax/SyntaxParsingContext.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,12 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
239239
/// Discard collected parts on this context.
240240
void setDiscard() { Mode = AccumulationMode::Discard; }
241241

242+
/// Explicitly finalizing syntax tree creation.
243+
/// This function will be called during the destroying of a root syntax
244+
/// parsing context. However, we can explicitly call this function to get
245+
/// the syntax tree before closing the root context.
246+
void finalizeRoot();
247+
242248
};
243249

244250
} // namespace syntax

‎lib/Syntax/SyntaxParsingContext.cpp

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ void finalizeSourceFile(RootContextData &RootData,
258258
}
259259
}
260260

261-
if (Parts.back()->isToken() &&
261+
if (!Parts.empty() && Parts.back()->isToken() &&
262262
cast<RawTokenSyntax>(Parts.back())->is(tok::eof)) {
263263
EOFToken.emplace(make<TokenSyntax>(Parts.back()));
264264
Parts = Parts.drop_back();
@@ -286,6 +286,19 @@ void finalizeSourceFile(RootContextData &RootData,
286286
}
287287
} // End of anonymous namespace
288288

289+
void SyntaxParsingContext::finalizeRoot() {
290+
if (!Enabled)
291+
return;
292+
assert(isTopOfContextStack() && "some sub-contexts are not destructed");
293+
assert(isRoot() && "only root context can finalize the tree");
294+
assert(Mode == AccumulationMode::Root);
295+
finalizeSourceFile(getRootData(), getParts());
296+
297+
// Clear the parts because we will call this function again when destroying
298+
// the root context.
299+
getRootData().Storage.clear();
300+
}
301+
289302
SyntaxParsingContext::~SyntaxParsingContext() {
290303
assert(isTopOfContextStack() && "destructed in wrong order");
291304

@@ -334,8 +347,7 @@ SyntaxParsingContext::~SyntaxParsingContext() {
334347

335348
// Accumulate parsed toplevel syntax onto the SourceFile.
336349
case AccumulationMode::Root:
337-
assert(isRoot() && "AccumulationMode::Root is only for root context");
338-
finalizeSourceFile(getRootData(), getParts());
350+
finalizeRoot();
339351
break;
340352

341353
// Never.

‎test/SourceKit/SyntaxTree/basic.swift

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// RUN: %swift-syntax-test -input-source-filename %s -serialize-raw-tree > %t.emit
2+
// RUN: %sourcekitd-test -req=syntax-tree %s > %t.sourcekit
3+
// RUN: diff %t.emit %t.sourcekit
4+
5+
struct Foo {
6+
let bar : Int
7+
8+
let baz : Array < Int >
9+
}

‎tools/SourceKit/include/SourceKit/Core/LangSupport.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,9 @@ class EditorConsumer {
238238

239239
virtual bool handleSourceText(StringRef Text) = 0;
240240

241+
virtual bool handleSerializedSyntaxTree(StringRef Text) = 0;
242+
virtual bool syntaxTreeEnabled() = 0;
243+
241244
virtual void finished() {}
242245
};
243246

‎tools/SourceKit/include/SourceKit/Core/ProtocolUIDs.def

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,10 @@ KEY(Severity, "key.severity")
3939
KEY(Offset, "key.offset")
4040
KEY(Length, "key.length")
4141
KEY(SourceFile, "key.sourcefile")
42+
KEY(SerializedSyntaxTree, "key.serialized_syntax_tree")
4243
KEY(SourceText, "key.sourcetext")
4344
KEY(EnableSyntaxMap, "key.enablesyntaxmap")
45+
KEY(EnableSyntaxTree, "key.enablesyntaxtree")
4446
KEY(EnableStructure, "key.enablesubstructure")
4547
KEY(Description, "key.description")
4648
KEY(TypeName, "key.typename")

‎tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
#include "swift/IDE/SourceEntityWalker.h"
3636
#include "swift/IDE/SyntaxModel.h"
3737
#include "swift/Subsystems.h"
38+
#include "swift/Syntax/Serialization/SyntaxSerialization.h"
39+
#include "swift/Syntax/SyntaxNodes.h"
3840

3941
#include "llvm/Support/ErrorHandling.h"
4042
#include "llvm/Support/MemoryBuffer.h"
@@ -669,6 +671,7 @@ class SwiftDocumentSyntaxInfo {
669671
P.parseTopLevel();
670672
Done = P.Tok.is(tok::eof);
671673
}
674+
P.finalizeSyntaxTree();
672675
}
673676

674677
SourceFile &getSourceFile() {
@@ -1791,6 +1794,15 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
17911794

17921795
ide::SyntaxModelContext ModelContext(Impl.SyntaxInfo->getSourceFile());
17931796

1797+
if (Consumer.syntaxTreeEnabled()) {
1798+
std::string SyntaxContent;
1799+
llvm::raw_string_ostream OS(SyntaxContent);
1800+
json::Output JsonOut(OS);
1801+
auto Root = Impl.SyntaxInfo->getSourceFile().getSyntaxRoot().getRaw();
1802+
JsonOut << Root;
1803+
Consumer.handleSerializedSyntaxTree(OS.str());
1804+
}
1805+
17941806
SwiftSyntaxMap NewMap = SwiftSyntaxMap(Impl.SyntaxMap.Tokens.size() + 16);
17951807

17961808
SwiftEditorSyntaxWalker SyntaxWalker(NewMap,

‎tools/SourceKit/tools/sourcekitd-test/TestOptions.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ bool TestOptions::parseArgs(llvm::ArrayRef<const char *> Args) {
117117
.Case("cursor", SourceKitRequest::CursorInfo)
118118
.Case("related-idents", SourceKitRequest::RelatedIdents)
119119
.Case("syntax-map", SourceKitRequest::SyntaxMap)
120+
.Case("syntax-tree", SourceKitRequest::SyntaxTree)
120121
.Case("structure", SourceKitRequest::Structure)
121122
.Case("format", SourceKitRequest::Format)
122123
.Case("expand-placeholder", SourceKitRequest::ExpandPlaceholder)

‎tools/SourceKit/tools/sourcekitd-test/TestOptions.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ enum class SourceKitRequest {
5757
NameTranslation,
5858
MarkupToXML,
5959
Statistics,
60+
SyntaxTree,
6061
#define SEMANTIC_REFACTORING(KIND, NAME, ID) KIND,
6162
#include "swift/IDE/RefactoringKinds.def"
6263
};

‎tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,7 @@ static int handleTestInvocation(ArrayRef<const char *> Args,
528528
sourcekitd_request_dictionary_set_string(Req, KeyName, SourceFile.c_str());
529529
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, true);
530530
sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
531+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxTree, false);
531532
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
532533
break;
533534

@@ -536,6 +537,7 @@ static int handleTestInvocation(ArrayRef<const char *> Args,
536537
sourcekitd_request_dictionary_set_string(Req, KeyName, SourceFile.c_str());
537538
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, false);
538539
sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, true);
540+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxTree, false);
539541
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
540542
break;
541543

@@ -544,6 +546,7 @@ static int handleTestInvocation(ArrayRef<const char *> Args,
544546
sourcekitd_request_dictionary_set_string(Req, KeyName, SourceFile.c_str());
545547
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, false);
546548
sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
549+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxTree, false);
547550
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
548551
break;
549552

@@ -555,6 +558,15 @@ static int handleTestInvocation(ArrayRef<const char *> Args,
555558
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
556559
break;
557560

561+
case SourceKitRequest::SyntaxTree:
562+
sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestEditorOpen);
563+
sourcekitd_request_dictionary_set_string(Req, KeyName, SourceFile.c_str());
564+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, false);
565+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
566+
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxTree, true);
567+
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, true);
568+
break;
569+
558570
case SourceKitRequest::DocInfo:
559571
sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestDocInfo);
560572
break;
@@ -875,6 +887,13 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
875887
printFoundUSR(Info, SourceBuf.get(), llvm::outs());
876888
break;
877889

890+
case SourceKitRequest::SyntaxTree: {
891+
// Print only the serialized syntax tree.
892+
llvm::outs() << sourcekitd_variant_dictionary_get_string(
893+
sourcekitd_response_get_value(Resp), KeySerializedSyntaxTree);
894+
llvm::outs() << '\n';
895+
break;
896+
}
878897
case SourceKitRequest::SyntaxMap:
879898
case SourceKitRequest::Structure:
880899
sourcekitd_response_description_dump_filedesc(Resp, STDOUT_FILENO);
@@ -897,7 +916,8 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
897916
EnableSyntaxMax);
898917
sourcekitd_request_dictionary_set_int64(EdReq, KeyEnableStructure,
899918
EnableSubStructure);
900-
sourcekitd_request_dictionary_set_int64(EdReq, KeySyntacticOnly, !Opts.UsedSema);
919+
sourcekitd_request_dictionary_set_int64(EdReq, KeySyntacticOnly,
920+
!Opts.UsedSema);
901921

902922
sourcekitd_response_t EdResp = sourcekitd_send_request_sync(EdReq);
903923
sourcekitd_response_description_dump_filedesc(EdResp, STDOUT_FILENO);

0 commit comments

Comments
 (0)