Skip to content

Commit fdc8e7d

Browse files
ahoppenahoppen
authored
Merge pull request #69294 from ahoppen/ahoppen/semantic-tokens
[SourceKit] Add a request to get the semantic tokens of a file
2 parents a058e98 + 2507e0e commit fdc8e7d

File tree

13 files changed

+284
-42
lines changed

13 files changed

+284
-42
lines changed

‎lib/IDE/SourceEntityWalker.cpp

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -609,14 +609,19 @@ ASTWalker::PreWalkResult<Expr *> SemaAnnotator::walkToExprPre(Expr *E) {
609609
// We already visited the children.
610610
return doSkipChildren();
611611
} else if (auto ME = dyn_cast<MacroExpansionExpr>(E)) {
612-
// Add a reference to the macro
613-
auto macroRef = ME->getMacroRef();
614-
if (auto *macroDecl = dyn_cast_or_null<MacroDecl>(macroRef.getDecl())) {
615-
auto macroRefType = macroDecl->getDeclaredInterfaceType();
616-
if (!passReference(
617-
macroDecl, macroRefType, ME->getMacroNameLoc(),
618-
ReferenceMetaData(SemaReferenceKind::DeclRef, llvm::None)))
619-
return Action::Stop();
612+
// Add a reference to the macro if this is a true macro expansion *expression*.
613+
// If this is a `MacroExpansionExpr` that expands a declaration macro, the
614+
// substitute decl will be visited by ASTWalker and we would be passing its
615+
// reference if we didn't have this check.
616+
if (!ME->getSubstituteDecl()) {
617+
auto macroRef = ME->getMacroRef();
618+
if (auto *macroDecl = dyn_cast_or_null<MacroDecl>(macroRef.getDecl())) {
619+
auto macroRefType = macroDecl->getDeclaredInterfaceType();
620+
if (!passReference(
621+
macroDecl, macroRefType, ME->getMacroNameLoc(),
622+
ReferenceMetaData(SemaReferenceKind::DeclRef, llvm::None)))
623+
return Action::Stop();
624+
}
620625
}
621626
}
622627

‎test/SourceKit/Macros/macro_semantic_token.swift

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
2+
@freestanding(declaration)
3+
macro anonymousTypes(_: () -> String) = #externalMacro(module: "MacroDefinition", type: "DefineAnonymousTypesMacro")
4+
5+
#anonymousTypes { "hello" }
6+
7+
// REQUIRES: swift_swift_parser, executable_test
8+
9+
// RUN: %empty-directory(%t)
10+
11+
//##-- Prepare the macro plugin.
12+
// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroDefinition) -module-name=MacroDefinition %S/../../Macros/Inputs/syntax_macro_definitions.swift -g -no-toolchain-stdlib-rpath
13+
14+
// Check the output of the the `#anonymousTypes` macro
15+
// RUN: %sourcekitd-test -req=semantic-tokens @__swiftmacro_9MacroUser33_8C2BB8A10AE555140C0EDFDEB4A9572DLl14anonymousTypesfMf0_.swift -primary-file %s -- -swift-version 5 -load-plugin-library %t/%target-library-name(MacroDefinition) -module-name MacroUser %s | %FileCheck %s --check-prefix IN_BUFFER
16+
17+
// Check that we get some semantic tokens. Checking exact offsets is brittle.
18+
// IN_BUFFER: source.lang.swift.ref.struct
19+
// IN_BUFFER: key.kind: source.lang.swift.ref.class
20+
21+
// Check that we don't get semantic tokens inside the buffer when requesting semantic tokens for the outer file
22+
// RUN: %sourcekitd-test -req=semantic-tokens %s -- -swift-version 5 -load-plugin-library %t/%target-library-name(MacroDefinition) -module-name MacroUser %s | %FileCheck %s --check-prefix PRIMARY_FILE
23+
// Reference to String in line 2
24+
// PRIMARY_FILE: key.kind: source.lang.swift.ref.struct
25+
// Reference to macro in line 4
26+
// PRIMARY_FILE: key.kind: source.lang.swift.ref.macro
27+
// There should be no other references
28+
// PRIMARY_FILE-NOT: key.kind

‎test/SourceKit/SemanticTokens/semantic_tokens.swift

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// RUN: %sourcekitd-test -req=semantic-tokens %s -- %s | %FileCheck %s
2+
3+
class MyClass {}
4+
5+
let x: String = "test"
6+
var y = MyClass()
7+
8+
9+
// String in line 5
10+
// CHECK: key.kind: source.lang.swift.ref.struct
11+
// CHECK: key.length: 6,
12+
// CHECK key.is_system: 1
13+
// CHECK-LABEL: },
14+
15+
// MyClass in line 6
16+
// CHECK: key.kind: source.lang.swift.ref.class,
17+
// CHECK: key.length: 7

‎tools/SourceKit/include/SourceKit/Core/LangSupport.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "SourceKit/Support/CancellationToken.h"
1818
#include "SourceKit/Support/UIdent.h"
1919
#include "swift/AST/Type.h"
20+
#include "swift/IDE/CodeCompletionResult.h"
2021
#include "llvm/ADT/ArrayRef.h"
2122
#include "llvm/ADT/IntrusiveRefCntPtr.h"
2223
#include "llvm/ADT/Optional.h"
@@ -331,6 +332,35 @@ struct DiagnosticEntryInfo : DiagnosticEntryInfoBase {
331332
SmallVector<DiagnosticEntryInfoBase, 1> Notes;
332333
};
333334

335+
struct SwiftSemanticToken {
336+
unsigned ByteOffset;
337+
unsigned Length : 24;
338+
// The code-completion kinds are a good match for the semantic kinds we want.
339+
// FIXME: Maybe rename CodeCompletionDeclKind to a more general concept ?
340+
swift::ide::CodeCompletionDeclKind Kind : 6;
341+
unsigned IsRef : 1;
342+
unsigned IsSystem : 1;
343+
344+
SwiftSemanticToken(swift::ide::CodeCompletionDeclKind Kind,
345+
unsigned ByteOffset, unsigned Length, bool IsRef,
346+
bool IsSystem)
347+
: ByteOffset(ByteOffset), Length(Length), Kind(Kind), IsRef(IsRef),
348+
IsSystem(IsSystem) {}
349+
350+
bool getIsRef() const { return static_cast<bool>(IsRef); }
351+
352+
bool getIsSystem() const { return static_cast<bool>(IsSystem); }
353+
354+
UIdent getUIdentForKind();
355+
};
356+
357+
#if !defined(_MSC_VER)
358+
static_assert(sizeof(SwiftSemanticToken) == 8, "Too big");
359+
// FIXME: MSVC doesn't pack bitfields with different underlying types.
360+
// Giving up to check this in MSVC for now, because static_assert is only for
361+
// keeping low memory usage.
362+
#endif
363+
334364
struct SourceFileRange {
335365
/// The byte offset at which the range begins
336366
uintptr_t Start;
@@ -710,6 +740,9 @@ struct CursorInfoData {
710740
/// The result type of `LangSupport::getDiagnostics`
711741
typedef ArrayRef<DiagnosticEntryInfo> DiagnosticsResult;
712742

743+
/// The result of `LangSupport::getSemanticTokens`.
744+
typedef std::vector<SwiftSemanticToken> SemanticTokensResult;
745+
713746
struct RangeInfo {
714747
UIdent RangeKind;
715748
StringRef ExprType;
@@ -1104,6 +1137,13 @@ class LangSupport {
11041137
std::function<void(const RequestResult<DiagnosticsResult> &)>
11051138
Receiver) = 0;
11061139

1140+
virtual void getSemanticTokens(
1141+
StringRef PrimaryFilePath, StringRef InputBufferName,
1142+
ArrayRef<const char *> Args, llvm::Optional<VFSOptions> VfsOptions,
1143+
SourceKitCancellationToken CancellationToken,
1144+
std::function<void(const RequestResult<SemanticTokensResult> &)>
1145+
Receiver) = 0;
1146+
11071147
virtual void
11081148
getNameInfo(StringRef PrimaryFilePath, StringRef InputBufferName,
11091149
unsigned Offset, NameTranslatingInfo &Input,

‎tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

Lines changed: 102 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -643,36 +643,6 @@ struct EditorConsumerSyntaxMapEntry {
643643
:Offset(Offset), Length(Length), Kind(Kind) { }
644644
};
645645

646-
struct SwiftSemanticToken {
647-
unsigned ByteOffset;
648-
unsigned Length : 24;
649-
// The code-completion kinds are a good match for the semantic kinds we want.
650-
// FIXME: Maybe rename CodeCompletionDeclKind to a more general concept ?
651-
CodeCompletionDeclKind Kind : 6;
652-
unsigned IsRef : 1;
653-
unsigned IsSystem : 1;
654-
655-
SwiftSemanticToken(CodeCompletionDeclKind Kind,
656-
unsigned ByteOffset, unsigned Length,
657-
bool IsRef, bool IsSystem)
658-
: ByteOffset(ByteOffset), Length(Length), Kind(Kind),
659-
IsRef(IsRef), IsSystem(IsSystem) { }
660-
661-
bool getIsRef() const { return static_cast<bool>(IsRef); }
662-
663-
bool getIsSystem() const { return static_cast<bool>(IsSystem); }
664-
665-
UIdent getUIdentForKind() const {
666-
return SwiftLangSupport::getUIDForCodeCompletionDeclKind(Kind, getIsRef());
667-
}
668-
};
669-
#if !defined(_MSC_VER)
670-
static_assert(sizeof(SwiftSemanticToken) == 8, "Too big");
671-
// FIXME: MSVC doesn't pack bitfields with different underlying types.
672-
// Giving up to check this in MSVC for now, because static_assert is only for
673-
// keeping low memory usage.
674-
#endif
675-
676646
class SwiftDocumentSemanticInfo :
677647
public ThreadSafeRefCountedBase<SwiftDocumentSemanticInfo> {
678648

@@ -982,9 +952,39 @@ class SemanticAnnotator : public SourceEntityWalker {
982952
public:
983953

984954
std::vector<SwiftSemanticToken> SemaToks;
955+
bool IsWalkingMacroExpansionBuffer = false;
985956

986957
SemanticAnnotator(SourceManager &SM, unsigned BufferID)
987-
: SM(SM), BufferID(BufferID) {}
958+
: SM(SM), BufferID(BufferID) {
959+
if (auto GeneratedSourceInfo = SM.getGeneratedSourceInfo(BufferID)) {
960+
switch (GeneratedSourceInfo->kind) {
961+
case GeneratedSourceInfo::ExpressionMacroExpansion:
962+
case GeneratedSourceInfo::FreestandingDeclMacroExpansion:
963+
case GeneratedSourceInfo::AccessorMacroExpansion:
964+
case GeneratedSourceInfo::MemberAttributeMacroExpansion:
965+
case GeneratedSourceInfo::MemberMacroExpansion:
966+
case GeneratedSourceInfo::PeerMacroExpansion:
967+
case GeneratedSourceInfo::ConformanceMacroExpansion:
968+
case GeneratedSourceInfo::ExtensionMacroExpansion:
969+
IsWalkingMacroExpansionBuffer = true;
970+
break;
971+
case GeneratedSourceInfo::ReplacedFunctionBody:
972+
case GeneratedSourceInfo::PrettyPrinted:
973+
break;
974+
}
975+
}
976+
}
977+
978+
MacroWalking getMacroWalkingBehavior() const override {
979+
if (IsWalkingMacroExpansionBuffer) {
980+
// When we are walking a macro expansion buffer, we need to set the macro
981+
// walking behavior to walk the expansion, otherwise we skip over all the
982+
// declarations in the buffer.
983+
return MacroWalking::ArgumentsAndExpansion;
984+
} else {
985+
return SourceEntityWalker::getMacroWalkingBehavior();
986+
}
987+
}
988988

989989
bool visitDeclReference(ValueDecl *D, CharSourceRange Range,
990990
TypeDecl *CtorTyRef, ExtensionDecl *ExtTyRef, Type T,
@@ -1026,6 +1026,12 @@ class SemanticAnnotator : public SourceEntityWalker {
10261026
if (!Range.isValid())
10271027
return;
10281028

1029+
// If we are walking into macro expansions, make sure we only report ranges
1030+
// from the requested buffer, not any buffers of child macro expansions.
1031+
if (IsWalkingMacroExpansionBuffer &&
1032+
SM.findBufferContainingLoc(Range.getStart()) != BufferID) {
1033+
return;
1034+
}
10291035
unsigned ByteOffset = SM.getLocOffsetInBuffer(Range.getStart(), BufferID);
10301036
unsigned Length = Range.getByteLength();
10311037
auto Kind = ContextFreeCodeCompletionResult::getCodeCompletionDeclKind(D);
@@ -2535,3 +2541,68 @@ void SwiftLangSupport::editorExpandPlaceholder(StringRef Name, unsigned Offset,
25352541
}
25362542
EditorDoc->expandPlaceholder(Offset, Length, Consumer);
25372543
}
2544+
2545+
//===----------------------------------------------------------------------===//
2546+
// Semantic Tokens
2547+
//===----------------------------------------------------------------------===//
2548+
2549+
void SwiftLangSupport::getSemanticTokens(
2550+
StringRef PrimaryFilePath, StringRef InputBufferName,
2551+
ArrayRef<const char *> Args, llvm::Optional<VFSOptions> VfsOptions,
2552+
SourceKitCancellationToken CancellationToken,
2553+
std::function<void(const RequestResult<SemanticTokensResult> &)> Receiver) {
2554+
std::string FileSystemError;
2555+
auto FileSystem = getFileSystem(VfsOptions, PrimaryFilePath, FileSystemError);
2556+
if (!FileSystem) {
2557+
Receiver(RequestResult<SemanticTokensResult>::fromError(FileSystemError));
2558+
return;
2559+
}
2560+
2561+
std::string InvocationError;
2562+
SwiftInvocationRef Invok = ASTMgr->getTypecheckInvocation(
2563+
Args, PrimaryFilePath, FileSystem, InvocationError);
2564+
if (!InvocationError.empty()) {
2565+
LOG_WARN_FUNC("error creating ASTInvocation: " << InvocationError);
2566+
}
2567+
if (!Invok) {
2568+
Receiver(RequestResult<SemanticTokensResult>::fromError(InvocationError));
2569+
return;
2570+
}
2571+
2572+
class SemanticTokensConsumer : public SwiftASTConsumer {
2573+
StringRef InputBufferName;
2574+
std::function<void(const RequestResult<SemanticTokensResult> &)> Receiver;
2575+
2576+
public:
2577+
SemanticTokensConsumer(
2578+
StringRef InputBufferName,
2579+
std::function<void(const RequestResult<SemanticTokensResult> &)>
2580+
Receiver)
2581+
: InputBufferName(InputBufferName), Receiver(Receiver) {}
2582+
2583+
void handlePrimaryAST(ASTUnitRef AstUnit) override {
2584+
auto &CompIns = AstUnit->getCompilerInstance();
2585+
SourceFile *SF = retrieveInputFile(InputBufferName, CompIns);
2586+
if (!SF) {
2587+
Receiver(RequestResult<SemanticTokensResult>::fromError(
2588+
"Unable to find input file"));
2589+
return;
2590+
}
2591+
SemanticAnnotator Annotator(CompIns.getSourceMgr(), *SF->getBufferID());
2592+
Annotator.walk(SF);
2593+
Receiver(
2594+
RequestResult<SemanticTokensResult>::fromResult(Annotator.SemaToks));
2595+
}
2596+
2597+
void cancelled() override {
2598+
Receiver(RequestResult<SemanticTokensResult>::cancelled());
2599+
}
2600+
};
2601+
2602+
auto Consumer = std::make_shared<SemanticTokensConsumer>(InputBufferName,
2603+
std::move(Receiver));
2604+
2605+
getASTManager()->processASTAsync(Invok, std::move(Consumer),
2606+
/*OncePerASTToken=*/nullptr,
2607+
CancellationToken, FileSystem);
2608+
}

‎tools/SourceKit/lib/SwiftLang/SwiftLangSupport.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,10 @@ UIdent SwiftLangSupport::getUIDForRefactoringKind(ide::RefactoringKind Kind){
397397
}
398398
}
399399

400+
UIdent SwiftSemanticToken::getUIdentForKind() {
401+
return SwiftLangSupport::getUIDForCodeCompletionDeclKind(Kind, IsRef);
402+
}
403+
400404
UIdent SwiftLangSupport::getUIDForCodeCompletionDeclKind(
401405
ide::CodeCompletionDeclKind Kind, bool IsRef) {
402406
if (IsRef) {

‎tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -665,6 +665,13 @@ class SwiftLangSupport : public LangSupport {
665665
std::function<void(const RequestResult<DiagnosticsResult> &)>
666666
Receiver) override;
667667

668+
void getSemanticTokens(
669+
StringRef PrimaryFilePath, StringRef InputBufferName,
670+
ArrayRef<const char *> Args, llvm::Optional<VFSOptions> VfsOptions,
671+
SourceKitCancellationToken CancellationToken,
672+
std::function<void(const RequestResult<SemanticTokensResult> &)> Receiver)
673+
override;
674+
668675
void getNameInfo(
669676
StringRef PrimaryFilePath, StringRef InputBufferName, unsigned Offset,
670677
NameTranslatingInfo &Input, ArrayRef<const char *> Args,
@@ -812,6 +819,10 @@ class CloseClangModuleFiles {
812819
/// Disable expensive SIL options which do not affect indexing or diagnostics.
813820
void disableExpensiveSILOptions(swift::SILOptions &Opts);
814821

822+
swift::SourceFile *retrieveInputFile(StringRef inputBufferName,
823+
const swift::CompilerInstance &CI,
824+
bool haveRealPath = false);
825+
815826
} // namespace SourceKit
816827

817828
#endif

‎tools/SourceKit/lib/SwiftLang/SwiftSourceDocInfo.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1537,9 +1537,9 @@ class CursorRangeInfoConsumer : public SwiftASTConsumer {
15371537
}
15381538
};
15391539

1540-
static SourceFile *retrieveInputFile(StringRef inputBufferName,
1541-
const CompilerInstance &CI,
1542-
bool haveRealPath = false) {
1540+
SourceFile *SourceKit::retrieveInputFile(StringRef inputBufferName,
1541+
const CompilerInstance &CI,
1542+
bool haveRealPath) {
15431543
// Don't bother looking up if we have the same file as the primary file or
15441544
// we weren't given a separate input file
15451545
if (inputBufferName.empty() ||

‎tools/SourceKit/tools/sourcekitd-test/TestOptions.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ bool TestOptions::parseArgs(llvm::ArrayRef<const char *> Args) {
155155
.Case("global-config", SourceKitRequest::GlobalConfiguration)
156156
.Case("dependency-updated", SourceKitRequest::DependencyUpdated)
157157
.Case("diags", SourceKitRequest::Diagnostics)
158+
.Case("semantic-tokens", SourceKitRequest::SemanticTokens)
158159
.Case("compile", SourceKitRequest::Compile)
159160
.Case("compile.close", SourceKitRequest::CompileClose)
160161
.Case("syntactic-expandmacro", SourceKitRequest::SyntacticMacroExpansion)

‎tools/SourceKit/tools/sourcekitd-test/TestOptions.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ enum class SourceKitRequest {
6868
GlobalConfiguration,
6969
DependencyUpdated,
7070
Diagnostics,
71+
SemanticTokens,
7172
Compile,
7273
CompileClose,
7374
SyntacticMacroExpansion,

‎tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,10 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
11371137
case SourceKitRequest::Diagnostics:
11381138
sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestDiagnostics);
11391139
break;
1140+
case SourceKitRequest::SemanticTokens:
1141+
sourcekitd_request_dictionary_set_uid(Req, KeyRequest,
1142+
RequestSemanticTokens);
1143+
break;
11401144

11411145
case SourceKitRequest::Compile:
11421146
sourcekitd_request_dictionary_set_string(Req, KeyName, SemaName.c_str());
@@ -1427,6 +1431,7 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
14271431
case SourceKitRequest::ConformingMethodList:
14281432
case SourceKitRequest::DependencyUpdated:
14291433
case SourceKitRequest::Diagnostics:
1434+
case SourceKitRequest::SemanticTokens:
14301435
printRawResponse(Resp);
14311436
break;
14321437
case SourceKitRequest::Compile:

0 commit comments

Comments
 (0)