Skip to content

Commit c8a3957

Browse files
ahoppenahoppen
committed
[SourceKit] Add option to force the SyntaxMap to be generated via the syntax tree
1 parent 5c22761 commit c8a3957

File tree

8 files changed

+167
-10
lines changed

8 files changed

+167
-10
lines changed

‎tools/SourceKit/include/SourceKit/Core/LangSupport.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,10 @@ class EditorConsumer {
241241
virtual bool syntaxTreeEnabled() = 0;
242242

243243
virtual void finished() {}
244+
245+
// FIXME: This is just for bootstrapping incremental syntax tree parsing.
246+
// Remove it once when we are able to incrementally transfer the syntax tree
247+
virtual bool forceLibSyntaxBasedProcessing() = 0;
244248
};
245249

246250
class OptionsDictionary {

‎tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

Lines changed: 137 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "swift/IDE/SyntaxModel.h"
3838
#include "swift/Subsystems.h"
3939
#include "swift/Syntax/Serialization/SyntaxSerialization.h"
40+
#include "swift/Syntax/SyntaxClassifier.h"
4041
#include "swift/Syntax/SyntaxNodes.h"
4142

4243
#include "llvm/Support/ErrorHandling.h"
@@ -557,6 +558,122 @@ struct SwiftSyntaxMap {
557558
}
558559
};
559560

561+
class SyntaxToSyntaxMapConverter : public SyntaxVisitor {
562+
SwiftSyntaxMap &SyntaxMap;
563+
564+
std::map<unsigned, SyntaxClassification> TokenClassifications;
565+
566+
public:
567+
SyntaxToSyntaxMapConverter(
568+
SwiftSyntaxMap &SyntaxMap,
569+
std::map<unsigned, SyntaxClassification> TokenClassifications)
570+
: SyntaxMap(SyntaxMap), TokenClassifications(TokenClassifications) {}
571+
572+
private:
573+
void visitTrivia(Trivia Trivia, unsigned Offset) {
574+
for (auto TriviaPiece : Trivia) {
575+
visitTriviaPiece(TriviaPiece, Offset);
576+
Offset += TriviaPiece.getTextLength();
577+
}
578+
}
579+
580+
void visitTriviaPiece(TriviaPiece TriviaPiece, unsigned Offset) {
581+
llvm::Optional<SyntaxNodeKind> Kind;
582+
switch (TriviaPiece.getKind()) {
583+
case TriviaKind::Space:
584+
case TriviaKind::Tab:
585+
case TriviaKind::VerticalTab:
586+
case TriviaKind::Formfeed:
587+
case TriviaKind::Newline:
588+
case TriviaKind::CarriageReturn:
589+
case TriviaKind::CarriageReturnLineFeed:
590+
case swift::syntax::TriviaKind::Backtick:
591+
case TriviaKind::GarbageText:
592+
Kind = llvm::None;
593+
break;
594+
case swift::syntax::TriviaKind::LineComment:
595+
Kind = SyntaxNodeKind::CommentLine;
596+
break;
597+
case TriviaKind::BlockComment:
598+
Kind = SyntaxNodeKind::CommentBlock;
599+
break;
600+
case TriviaKind::DocLineComment:
601+
Kind = SyntaxNodeKind::DocCommentLine;
602+
break;
603+
case TriviaKind::DocBlockComment:
604+
Kind = SyntaxNodeKind::DocCommentBlock;
605+
break;
606+
}
607+
if (Kind.hasValue()) {
608+
SwiftSyntaxToken Token(Offset, TriviaPiece.getTextLength(),
609+
Kind.getValue());
610+
SyntaxMap.addToken(Token);
611+
}
612+
}
613+
614+
llvm::Optional<SyntaxNodeKind>
615+
getKindForSyntaxClassification(SyntaxClassification Classification) const {
616+
// FIXME: We should really use the same enum so that the conversion is not
617+
// necessary
618+
switch (Classification) {
619+
case SyntaxClassification::None:
620+
return llvm::None;
621+
case SyntaxClassification::Keyword:
622+
return SyntaxNodeKind::Keyword;
623+
case SyntaxClassification::Identifier:
624+
return SyntaxNodeKind::Identifier;
625+
case SyntaxClassification::DollarIdentifier:
626+
return SyntaxNodeKind::DollarIdent;
627+
case SyntaxClassification::IntegerLiteral:
628+
return SyntaxNodeKind::Integer;
629+
case SyntaxClassification::FloatingLiteral:
630+
return SyntaxNodeKind::Floating;
631+
case SyntaxClassification::StringLiteral:
632+
return SyntaxNodeKind::String;
633+
case SyntaxClassification::StringInterpolationAnchor:
634+
return SyntaxNodeKind::StringInterpolationAnchor;
635+
case SyntaxClassification::TypeIdentifier:
636+
return SyntaxNodeKind::TypeId;
637+
case SyntaxClassification::BuildConfigKeyword:
638+
return SyntaxNodeKind::BuildConfigKeyword;
639+
case SyntaxClassification::BuildConfigId:
640+
return SyntaxNodeKind::BuildConfigId;
641+
case SyntaxClassification::PoundDirectiveKeyword:
642+
return SyntaxNodeKind::PoundDirectiveKeyword;
643+
case SyntaxClassification::Attribute:
644+
return SyntaxNodeKind::AttributeBuiltin;
645+
case SyntaxClassification::EditorPlaceholder:
646+
return SyntaxNodeKind::EditorPlaceholder;
647+
case SyntaxClassification::ObjectLiteral:
648+
return SyntaxNodeKind::ObjectLiteral;
649+
}
650+
}
651+
652+
virtual void visit(TokenSyntax Token) override {
653+
if (Token.isMissing())
654+
return;
655+
656+
auto LeadingTriviaOffset =
657+
Token.getAbsolutePositionWithLeadingTrivia().getOffset();
658+
visitTrivia(Token.getLeadingTrivia(), LeadingTriviaOffset);
659+
660+
SyntaxClassification Classification = TokenClassifications[Token.getId()];
661+
auto Kind = getKindForSyntaxClassification(Classification);
662+
unsigned TokenStart = Token.getAbsolutePosition().getOffset();
663+
unsigned TokenLength = Token.getRaw()->getTokenText().size();
664+
if (Kind.hasValue() && TokenLength > 0) {
665+
SwiftSyntaxToken Token(TokenStart, TokenLength, Kind.getValue());
666+
SyntaxMap.addToken(Token);
667+
}
668+
669+
auto TrailingTriviaOffset = TokenStart + TokenLength;
670+
visitTrivia(Token.getTrailingTrivia(), TrailingTriviaOffset);
671+
}
672+
673+
public:
674+
void writeToSyntaxMap(Syntax Node) { Node.accept(*this); }
675+
};
676+
560677
struct EditorConsumerSyntaxMapEntry {
561678
unsigned Offset;
562679
unsigned Length;
@@ -1772,8 +1889,6 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
17721889

17731890
Impl.ParserDiagnostics = Impl.SyntaxInfo->getDiagnostics();
17741891

1775-
ide::SyntaxModelContext ModelContext(Impl.SyntaxInfo->getSourceFile());
1776-
17771892
if (Consumer.syntaxTreeEnabled()) {
17781893
std::string SyntaxContent;
17791894
llvm::raw_string_ostream OS(SyntaxContent);
@@ -1785,11 +1900,21 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
17851900

17861901
SwiftSyntaxMap NewMap = SwiftSyntaxMap(Impl.SyntaxMap.Tokens.size() + 16);
17871902

1788-
SwiftEditorSyntaxWalker SyntaxWalker(NewMap,
1789-
Impl.SyntaxInfo->getSourceManager(),
1790-
Consumer,
1791-
Impl.SyntaxInfo->getBufferID());
1792-
ModelContext.walk(SyntaxWalker);
1903+
if (Consumer.forceLibSyntaxBasedProcessing()) {
1904+
auto SyntaxTree = Impl.SyntaxInfo->getSourceFile().getSyntaxRoot();
1905+
1906+
SyntaxClassifier Classifier;
1907+
auto Classification = Classifier.classify(SyntaxTree);
1908+
SyntaxToSyntaxMapConverter Printer(NewMap, Classification);
1909+
Printer.writeToSyntaxMap(SyntaxTree);
1910+
} else {
1911+
ide::SyntaxModelContext ModelContext(Impl.SyntaxInfo->getSourceFile());
1912+
1913+
SwiftEditorSyntaxWalker SyntaxWalker(
1914+
NewMap, Impl.SyntaxInfo->getSourceManager(), Consumer,
1915+
Impl.SyntaxInfo->getBufferID());
1916+
ModelContext.walk(SyntaxWalker);
1917+
}
17931918

17941919
bool SawChanges = true;
17951920
if (Impl.Edited) {
@@ -2034,7 +2159,8 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
20342159
ArrayRef<const char *> Args) {
20352160

20362161
ImmutableTextSnapshotRef Snapshot = nullptr;
2037-
const bool BuildSyntaxTree = Consumer.syntaxTreeEnabled();
2162+
const bool BuildSyntaxTree =
2163+
Consumer.syntaxTreeEnabled() || Consumer.forceLibSyntaxBasedProcessing();
20382164
auto EditorDoc = EditorDocuments.getByUnresolvedName(Name);
20392165
if (!EditorDoc) {
20402166
EditorDoc = new SwiftEditorDocument(Name, *this);
@@ -2101,7 +2227,9 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
21012227
Snapshot = EditorDoc->replaceText(Offset, Length, Buf,
21022228
Consumer.needsSemanticInfo());
21032229
assert(Snapshot);
2104-
EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
2230+
bool BuildSyntaxTree = Consumer.syntaxTreeEnabled() ||
2231+
Consumer.forceLibSyntaxBasedProcessing();
2232+
EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
21052233
EditorDoc->readSyntaxInfo(Consumer);
21062234
} else {
21072235
Snapshot = EditorDoc->getLatestSnapshot();

‎tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ class SwiftEditorDocument :
9494

9595
void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
9696
bool BuildSyntaxTree);
97-
void readSyntaxInfo(EditorConsumer& consumer);
97+
void readSyntaxInfo(EditorConsumer &consumer);
9898
void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
9999
EditorConsumer& Consumer);
100100

‎tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,8 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
647647
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, true);
648648
sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
649649
sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxTree, false);
650+
sourcekitd_request_dictionary_set_int64(
651+
Req, KeyForceLibSyntaxBasedProcessing, true);
650652
sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
651653
break;
652654

@@ -713,6 +715,8 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
713715
sourcekitd_request_dictionary_set_int64(Req, KeyLength, Opts.Length);
714716
sourcekitd_request_dictionary_set_string(Req, KeySourceText,
715717
Opts.ReplaceText.getValue().c_str());
718+
sourcekitd_request_dictionary_set_int64(
719+
Req, KeyForceLibSyntaxBasedProcessing, true);
716720
break;
717721

718722
case SourceKitRequest::PrintAnnotations:
@@ -1061,6 +1065,8 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
10611065
EnableSubStructure);
10621066
sourcekitd_request_dictionary_set_int64(EdReq, KeySyntacticOnly,
10631067
!Opts.UsedSema);
1068+
sourcekitd_request_dictionary_set_int64(
1069+
EdReq, KeyForceLibSyntaxBasedProcessing, true);
10641070

10651071
sourcekitd_response_t EdResp = sendRequestSync(EdReq, Opts);
10661072
sourcekitd_response_description_dump_filedesc(EdResp, STDOUT_FILENO);

‎tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ struct SKEditorConsumerOptions {
7575
bool EnableDiagnostics = false;
7676
bool EnableSyntaxTree = false;
7777
bool SyntacticOnly = false;
78+
// FIXME: This is just for bootstrapping incremental syntax tree parsing.
79+
// Remove it once when we are able to incrementally transfer the syntax tree
80+
bool ForceLibSyntaxBasedProcessing = false;
7881
};
7982

8083
} // anonymous namespace
@@ -435,13 +438,17 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
435438
Req.getInt64(KeyEnableSyntaxTree, EnableSyntaxTree, /*isOptional=*/true);
436439
int64_t SyntacticOnly = false;
437440
Req.getInt64(KeySyntacticOnly, SyntacticOnly, /*isOptional=*/true);
441+
int64_t ForceLibSyntaxBasedProcessing = false;
442+
Req.getInt64(KeyForceLibSyntaxBasedProcessing,
443+
ForceLibSyntaxBasedProcessing, /*isOptional=*/true);
438444

439445
SKEditorConsumerOptions Opts;
440446
Opts.EnableSyntaxMap = EnableSyntaxMap;
441447
Opts.EnableStructure = EnableStructure;
442448
Opts.EnableDiagnostics = EnableDiagnostics;
443449
Opts.EnableSyntaxTree = EnableSyntaxTree;
444450
Opts.SyntacticOnly = SyntacticOnly;
451+
Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
445452
return Rec(editorOpen(*Name, InputBuf.get(), Opts, Args));
446453
}
447454
if (ReqUID == RequestEditorClose) {
@@ -476,13 +483,18 @@ void handleRequestImpl(sourcekitd_object_t ReqObj, ResponseReceiver Rec) {
476483
Req.getInt64(KeyEnableSyntaxTree, EnableSyntaxTree, /*isOptional=*/true);
477484
int64_t SyntacticOnly = false;
478485
Req.getInt64(KeySyntacticOnly, SyntacticOnly, /*isOptional=*/true);
486+
int64_t ForceLibSyntaxBasedProcessing = false;
487+
Req.getInt64(KeyForceLibSyntaxBasedProcessing,
488+
ForceLibSyntaxBasedProcessing,
489+
/*isOptional=*/true);
479490

480491
SKEditorConsumerOptions Opts;
481492
Opts.EnableSyntaxMap = EnableSyntaxMap;
482493
Opts.EnableStructure = EnableStructure;
483494
Opts.EnableDiagnostics = EnableDiagnostics;
484495
Opts.EnableSyntaxTree = EnableSyntaxTree;
485496
Opts.SyntacticOnly = SyntacticOnly;
497+
Opts.ForceLibSyntaxBasedProcessing = ForceLibSyntaxBasedProcessing;
486498

487499
return Rec(editorReplaceText(*Name, InputBuf.get(), Offset, Length, Opts));
488500
}
@@ -2064,6 +2076,10 @@ class SKEditorConsumer : public EditorConsumer {
20642076
if (RespReceiver)
20652077
RespReceiver(createResponse());
20662078
}
2079+
2080+
virtual bool forceLibSyntaxBasedProcessing() override {
2081+
return Opts.ForceLibSyntaxBasedProcessing;
2082+
}
20672083
};
20682084

20692085
} // end anonymous namespace

‎unittests/SourceKit/SwiftLang/CursorInfoTest.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ class NullEditorConsumer : public EditorConsumer {
9090
bool handleSourceText(StringRef Text) override { return false; }
9191
bool handleSerializedSyntaxTree(StringRef Text) override { return false; }
9292
bool syntaxTreeEnabled() override { return false; }
93+
bool forceLibSyntaxBasedProcessing() override { return false; }
9394

9495
public:
9596
bool needsSema = false;

‎unittests/SourceKit/SwiftLang/EditingTest.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ class DiagConsumer : public EditorConsumer {
9999
bool handleSourceText(StringRef Text) override { return false; }
100100
bool handleSerializedSyntaxTree(StringRef Text) override { return false; }
101101
bool syntaxTreeEnabled() override { return false; }
102+
bool forceLibSyntaxBasedProcessing() override { return false; }
102103
};
103104

104105
struct DocUpdateMutexState {

‎utils/gyb_sourcekit_support/UIDs.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ def __init__(self, internal_name, external_name):
4848
KEY('SourceFile', 'key.sourcefile'),
4949
KEY('SerializedSyntaxTree', 'key.serialized_syntax_tree'),
5050
KEY('SourceText', 'key.sourcetext'),
51+
KEY('ForceLibSyntaxBasedProcessing', 'key.forcelibsyntaxbasedprocessing'),
5152
KEY('EnableSyntaxMap', 'key.enablesyntaxmap'),
5253
KEY('EnableSyntaxTree', 'key.enablesyntaxtree'),
5354
KEY('EnableStructure', 'key.enablesubstructure'),

0 commit comments

Comments
 (0)