Skip to content

Commit e9dfdea

Browse files
nkcsgexinkcsgexi
committed
SerializeLoc: serialize basic decl source location information to .swiftsourceinfo file
After setting up the .swiftsourceinfo file, this patch starts to actually serialize and de-serialize source locations for declaration. The binary format of .swiftsourceinfo currently contains these three records: BasicDeclLocs: a hash table mapping from a USR ID to a list of basic source locations. The USR id could be retrieved from the following DeclUSRs record using an actual decl USR. The basic source locations include a file ID and the results from Decl::getLoc(), ValueDecl::getNameLoc(), Decl::getStartLoc() and Decl::getEndLoc(). The file ID could be used to retrieve the actual file name from the following SourceFilePaths record. Each location is encoded as a line:column pair. DeclUSRS: a hash table mapping from USR to a USR ID used by location records. SourceFilePaths: a hash table mapping from a file ID to actual file name. BasicDeclLocs should be sufficient for most diagnostic cases. If additional source locations are needed, we could always add new source location records without breaking the backward compatibility. When de-serializing the source location from a module-imported decl, we calculate its USR, retrieve the USR ID from the DeclUSRS record, and use the USR ID to look up the basic location list in the BasicDeclLocs record. For more details about .swiftsourceinfo file: https://forums.swift.org/t/proposal-emitting-source-information-file-during-compilation
1 parent 0587470 commit e9dfdea

File tree

22 files changed

+1163
-132
lines changed

22 files changed

+1163
-132
lines changed

‎include/swift/AST/FileUnit.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,10 @@ class FileUnit : public DeclContext {
131131
return None;
132132
}
133133

134+
virtual Optional<BasicDeclLocs> getBasicLocsForDecl(const Decl *D) const {
135+
return None;
136+
}
137+
134138
virtual void collectAllGroups(std::vector<StringRef> &Names) const {}
135139

136140
/// Returns an implementation-defined "discriminator" for \p D, which

‎include/swift/AST/RawComment.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,19 @@ struct CommentInfo {
7878
uint32_t SourceOrder;
7979
};
8080

81+
struct LineColumn {
82+
uint32_t Line;
83+
uint32_t Column;
84+
};
85+
86+
struct BasicDeclLocs {
87+
StringRef SourceFilePath;
88+
Optional<LineColumn> Loc;
89+
Optional<LineColumn> NameLoc;
90+
Optional<LineColumn> StartLoc;
91+
Optional<LineColumn> EndLoc;
92+
};
93+
8194
} // namespace swift
8295

8396
#endif // LLVM_SWIFT_AST_RAW_COMMENT_H

‎include/swift/AST/USRGeneration.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "swift/Basic/LLVM.h"
2323

2424
namespace swift {
25+
class Decl;
2526
class AbstractStorageDecl;
2627
class ValueDecl;
2728
class ExtensionDecl;
@@ -56,6 +57,10 @@ bool printAccessorUSR(const AbstractStorageDecl *D, AccessorKind AccKind,
5657
/// \returns true if it failed, false on success.
5758
bool printExtensionUSR(const ExtensionDecl *ED, raw_ostream &OS);
5859

60+
/// Prints out the Decl USRs suitable for keys .swiftdoc and .swiftsourceinfo files.
61+
/// \returns true if it failed, false on success.
62+
bool printDeclUSRForModuleDoc(const Decl *D, raw_ostream &OS);
63+
5964
} // namespace ide
6065
} // namespace swift
6166

‎include/swift/Frontend/Frontend.h

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@ class CompilerInstance {
395395
struct PartialModuleInputs {
396396
std::unique_ptr<llvm::MemoryBuffer> ModuleBuffer;
397397
std::unique_ptr<llvm::MemoryBuffer> ModuleDocBuffer;
398+
std::unique_ptr<llvm::MemoryBuffer> ModuleSourceInfoBuffer;
398399
};
399400

400401
/// Contains \c MemoryBuffers for partial serialized module files and
@@ -555,20 +556,29 @@ class CompilerInstance {
555556

556557
Optional<unsigned> getRecordedBufferID(const InputFile &input, bool &failed);
557558

559+
struct ModuleBuffers {
560+
std::unique_ptr<llvm::MemoryBuffer> ModuleBuffer;
561+
std::unique_ptr<llvm::MemoryBuffer> ModuleDocBuffer;
562+
std::unique_ptr<llvm::MemoryBuffer> ModuleSourceInfoBuffer;
563+
};
564+
558565
/// Given an input file, return a buffer to use for its contents,
559566
/// and a buffer for the corresponding module doc file if one exists.
560567
/// On failure, return a null pointer for the first element of the returned
561568
/// pair.
562-
std::pair<std::unique_ptr<llvm::MemoryBuffer>,
563-
std::unique_ptr<llvm::MemoryBuffer>>
564-
getInputBufferAndModuleDocBufferIfPresent(const InputFile &input);
569+
ModuleBuffers getInputBuffersIfPresent(const InputFile &input);
565570

566571
/// Try to open the module doc file corresponding to the input parameter.
567572
/// Return None for error, nullptr if no such file exists, or the buffer if
568573
/// one was found.
569574
Optional<std::unique_ptr<llvm::MemoryBuffer>>
570575
openModuleDoc(const InputFile &input);
571576

577+
/// Try to open the module source info file corresponding to the input parameter.
578+
/// Return None for error, nullptr if no such file exists, or the buffer if
579+
/// one was found.
580+
Optional<std::unique_ptr<llvm::MemoryBuffer>>
581+
openModuleSourceInfo(const InputFile &input);
572582
public:
573583
/// Parses and type-checks all input files.
574584
void performSema();

‎include/swift/Frontend/ModuleInterfaceLoader.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,10 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
149149
std::error_code findModuleFilesInDirectory(
150150
AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
151151
StringRef ModuleDocFilename,
152+
StringRef ModuleSourceInfoFilename,
152153
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
153-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
154+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
155+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) override;
154156

155157
bool isCached(StringRef DepPath) override;
156158

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
5353
bool findModule(AccessPathElem moduleID,
5454
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
5555
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
56+
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
5657
bool &isFramework, bool &isSystemModule);
5758

5859
/// Attempts to search the provided directory for a loadable serialized
@@ -70,20 +71,30 @@ class SerializedModuleLoaderBase : public ModuleLoader {
7071
virtual std::error_code findModuleFilesInDirectory(
7172
AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
7273
StringRef ModuleDocFilename,
74+
StringRef ModuleSourceInfoFilename,
7375
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
74-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) = 0;
76+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
77+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) = 0;
7578

7679
std::error_code
7780
openModuleFiles(AccessPathElem ModuleID,
7881
StringRef ModulePath, StringRef ModuleDocPath,
82+
StringRef ModuleSourceInfoPath,
7983
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
80-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer);
84+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
85+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer);
8186

8287
std::error_code
8388
openModuleDocFile(AccessPathElem ModuleID,
8489
StringRef ModuleDocPath,
8590
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer);
8691

92+
std::error_code
93+
openModuleSourceInfoFile(AccessPathElem ModuleID,
94+
StringRef ModulePath,
95+
StringRef ModuleSourceInfoPath,
96+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer);
97+
8798
/// If the module loader subclass knows that all options have been tried for
8899
/// loading an architecture-specific file out of a swiftmodule bundle, try
89100
/// to list the architectures that \e are present.
@@ -116,6 +127,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
116127
FileUnit *loadAST(ModuleDecl &M, Optional<SourceLoc> diagLoc,
117128
std::unique_ptr<llvm::MemoryBuffer> moduleInputBuffer,
118129
std::unique_ptr<llvm::MemoryBuffer> moduleDocInputBuffer,
130+
std::unique_ptr<llvm::MemoryBuffer> moduleSourceInfoInputBuffer,
119131
bool isFramework, bool treatAsPartialModule);
120132

121133
/// Check whether the module with a given name can be imported without
@@ -163,8 +175,10 @@ class SerializedModuleLoader : public SerializedModuleLoaderBase {
163175
std::error_code findModuleFilesInDirectory(
164176
AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
165177
StringRef ModuleDocFilename,
178+
StringRef ModuleSourceInfoFilename,
166179
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
167-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
180+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
181+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) override;
168182

169183
bool maybeDiagnoseTargetMismatch(SourceLoc sourceLocation,
170184
StringRef moduleName,
@@ -203,8 +217,10 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
203217
std::error_code findModuleFilesInDirectory(
204218
AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
205219
StringRef ModuleDocFilename,
220+
StringRef ModuleSourceInfoFilename,
206221
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
207-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
222+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
223+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) override;
208224

209225
bool maybeDiagnoseTargetMismatch(SourceLoc sourceLocation,
210226
StringRef moduleName,
@@ -317,6 +333,8 @@ class SerializedASTFile final : public LoadedFile {
317333

318334
Optional<StringRef> getGroupNameByUSR(StringRef USR) const override;
319335

336+
Optional<BasicDeclLocs> getBasicLocsForDecl(const Decl *D) const override;
337+
320338
void collectAllGroups(std::vector<StringRef> &Names) const override;
321339

322340
virtual void getTopLevelDecls(SmallVectorImpl<Decl*> &results) const override;

‎lib/AST/USRGeneration.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,3 +341,19 @@ bool ide::printExtensionUSR(const ExtensionDecl *ED, raw_ostream &OS) {
341341
return true;
342342
}
343343

344+
bool ide::printDeclUSRForModuleDoc(const Decl *D, raw_ostream &OS) {
345+
if (D->isImplicit())
346+
return true;
347+
if (auto *VD = dyn_cast<ValueDecl>(D)) {
348+
if (ide::printDeclUSR(VD, OS)) {
349+
return true;
350+
}
351+
} else if (auto *ED = dyn_cast<ExtensionDecl>(D)) {
352+
if (ide::printExtensionUSR(ED, OS)) {
353+
return true;
354+
}
355+
} else {
356+
return true;
357+
}
358+
return false;
359+
}

‎lib/Frontend/Frontend.cpp

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -468,38 +468,35 @@ Optional<unsigned> CompilerInstance::getRecordedBufferID(const InputFile &input,
468468
return existingBufferID;
469469
}
470470
}
471-
std::pair<std::unique_ptr<llvm::MemoryBuffer>,
472-
std::unique_ptr<llvm::MemoryBuffer>>
473-
buffers = getInputBufferAndModuleDocBufferIfPresent(input);
471+
auto buffers = getInputBuffersIfPresent(input);
474472

475-
if (!buffers.first) {
473+
if (!buffers.ModuleBuffer) {
476474
failed = true;
477475
return None;
478476
}
479477

480478
// FIXME: The fact that this test happens twice, for some cases,
481479
// suggests that setupInputs could use another round of refactoring.
482-
if (serialization::isSerializedAST(buffers.first->getBuffer())) {
480+
if (serialization::isSerializedAST(buffers.ModuleBuffer->getBuffer())) {
483481
PartialModules.push_back(
484-
{std::move(buffers.first), std::move(buffers.second)});
482+
{std::move(buffers.ModuleBuffer), std::move(buffers.ModuleDocBuffer),
483+
std::move(buffers.ModuleSourceInfoBuffer)});
485484
return None;
486485
}
487-
assert(buffers.second.get() == nullptr);
486+
assert(buffers.ModuleDocBuffer.get() == nullptr);
487+
assert(buffers.ModuleSourceInfoBuffer.get() == nullptr);
488488
// Transfer ownership of the MemoryBuffer to the SourceMgr.
489-
unsigned bufferID = SourceMgr.addNewSourceBuffer(std::move(buffers.first));
489+
unsigned bufferID = SourceMgr.addNewSourceBuffer(std::move(buffers.ModuleBuffer));
490490

491491
InputSourceCodeBufferIDs.push_back(bufferID);
492492
return bufferID;
493493
}
494494

495-
std::pair<std::unique_ptr<llvm::MemoryBuffer>,
496-
std::unique_ptr<llvm::MemoryBuffer>>
497-
CompilerInstance::getInputBufferAndModuleDocBufferIfPresent(
495+
CompilerInstance::ModuleBuffers CompilerInstance::getInputBuffersIfPresent(
498496
const InputFile &input) {
499497
if (auto b = input.buffer()) {
500-
return std::make_pair(llvm::MemoryBuffer::getMemBufferCopy(
501-
b->getBuffer(), b->getBufferIdentifier()),
502-
nullptr);
498+
return {llvm::MemoryBuffer::getMemBufferCopy(b->getBuffer(), b->getBufferIdentifier()),
499+
nullptr, nullptr};
503500
}
504501
// FIXME: Working with filenames is fragile, maybe use the real path
505502
// or have some kind of FileManager.
@@ -509,17 +506,37 @@ CompilerInstance::getInputBufferAndModuleDocBufferIfPresent(
509506
if (!inputFileOrErr) {
510507
Diagnostics.diagnose(SourceLoc(), diag::error_open_input_file, input.file(),
511508
inputFileOrErr.getError().message());
512-
return std::make_pair(nullptr, nullptr);
509+
return {nullptr, nullptr, nullptr};
513510
}
514511
if (!serialization::isSerializedAST((*inputFileOrErr)->getBuffer()))
515-
return std::make_pair(std::move(*inputFileOrErr), nullptr);
512+
return {std::move(*inputFileOrErr), nullptr, nullptr};
516513

517-
if (Optional<std::unique_ptr<llvm::MemoryBuffer>> moduleDocBuffer =
518-
openModuleDoc(input)) {
519-
return std::make_pair(std::move(*inputFileOrErr),
520-
std::move(*moduleDocBuffer));
521-
}
522-
return std::make_pair(nullptr, nullptr);
514+
Optional<std::unique_ptr<llvm::MemoryBuffer>> moduleDocBuffer = openModuleDoc(input);
515+
Optional<std::unique_ptr<llvm::MemoryBuffer>> moduleSourceInfoBuffer = openModuleSourceInfo(input);
516+
517+
return {
518+
std::move(*inputFileOrErr),
519+
moduleDocBuffer.hasValue() ? std::move(*moduleDocBuffer): nullptr,
520+
moduleSourceInfoBuffer.hasValue() ? std::move(*moduleSourceInfoBuffer): nullptr
521+
};
522+
}
523+
524+
Optional<std::unique_ptr<llvm::MemoryBuffer>>
525+
CompilerInstance::openModuleSourceInfo(const InputFile &input) {
526+
llvm::SmallString<128> moduleSourceInfoFilePath(input.file());
527+
llvm::sys::path::replace_extension(moduleSourceInfoFilePath,
528+
file_types::getExtension(file_types::TY_SwiftSourceInfoFile));
529+
std::string NonPrivatePath = moduleSourceInfoFilePath.str().str();
530+
StringRef fileName = llvm::sys::path::filename(NonPrivatePath);
531+
llvm::sys::path::remove_filename(moduleSourceInfoFilePath);
532+
llvm::sys::path::append(moduleSourceInfoFilePath, "Private");
533+
llvm::sys::path::append(moduleSourceInfoFilePath, fileName);
534+
if (auto sourceInfoFileOrErr = swift::vfs::getFileOrSTDIN(getFileSystem(),
535+
moduleSourceInfoFilePath))
536+
return std::move(*sourceInfoFileOrErr);
537+
if (auto sourceInfoFileOrErr = swift::vfs::getFileOrSTDIN(getFileSystem(), NonPrivatePath))
538+
return std::move(*sourceInfoFileOrErr);
539+
return None;
523540
}
524541

525542
Optional<std::unique_ptr<llvm::MemoryBuffer>>
@@ -896,7 +913,8 @@ bool CompilerInstance::parsePartialModulesAndLibraryFiles(
896913
for (auto &PM : PartialModules) {
897914
assert(PM.ModuleBuffer);
898915
if (!SML->loadAST(*MainModule, SourceLoc(), std::move(PM.ModuleBuffer),
899-
std::move(PM.ModuleDocBuffer), /*isFramework*/false,
916+
std::move(PM.ModuleDocBuffer),
917+
std::move(PM.ModuleSourceInfoBuffer), /*isFramework*/false,
900918
/*treatAsPartialModule*/true))
901919
hadLoadError = true;
902920
}

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -988,8 +988,10 @@ bool ModuleInterfaceLoader::isCached(StringRef DepPath) {
988988
std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
989989
AccessPathElem ModuleID, StringRef DirPath, StringRef ModuleFilename,
990990
StringRef ModuleDocFilename,
991+
StringRef ModuleSourceInfoFilename,
991992
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
992-
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) {
993+
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
994+
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) {
993995

994996
// If running in OnlySerialized mode, ModuleInterfaceLoader
995997
// should not have been constructed at all.

0 commit comments

Comments
 (0)