Skip to content

Commit 0559ec4

Browse files
BigcheeseBigcheese
committed
[clang][DepScan] Remove unused -ivfsoverlay files
`-ivfsoverlay` files are unused when building most modules. Enable removing them by, * adding a way to visit the filesystem tree with extensible RTTI to access each `RedirectingFileSystem`. * Adding tracking to `RedirectingFileSystem` to record when it actually redirects a file access. * Storing this information in each PCM. Usage tracking is only enabled when iterating over the source manager and affecting modulemaps. Here each path is stated to cause an access. During scanning these stats all hit the cache.
1 parent 5bd87e6 commit 0559ec4

25 files changed

+932
-77
lines changed

‎clang/include/clang/Basic/DiagnosticSerializationKinds.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ def err_pch_diagopt_mismatch : Error<"%0 is currently enabled, but was not in "
4444
"the PCH file">;
4545
def err_pch_modulecache_mismatch : Error<"PCH was compiled with module cache "
4646
"path '%0', but the path is currently '%1'">;
47+
def err_pch_vfsoverlay_mismatch : Error<"PCH was compiled with different VFS overlay files than are currently in use">;
48+
def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has the following VFS overlays:\n%1">;
49+
def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">;
4750

4851
def err_pch_version_too_old : Error<
4952
"PCH file uses an older PCH format that is no longer supported">;

‎clang/include/clang/Basic/FileManager.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,10 @@ class FileManager : public RefCountedBase<FileManager> {
248248
return FS;
249249
}
250250

251+
/// Enable or disable tracking of VFS usage. Used to not track full header
252+
/// search and implicit modulemap lookup.
253+
void trackVFSUsage(bool Active);
254+
251255
void setVirtualFileSystem(IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
252256
this->FS = std::move(FS);
253257
}

‎clang/include/clang/Lex/HeaderSearch.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,6 +576,12 @@ class HeaderSearch {
576576
/// Note: implicit module maps don't contribute to entry usage.
577577
std::vector<bool> computeUserEntryUsage() const;
578578

579+
/// Determine which HeaderSearchOptions::VFSOverlayFiles have been
580+
/// successfully used so far and mark their index with 'true' in the resulting
581+
/// bit vector.
582+
/// Note: implicit module maps don't contribute to entry usage.
583+
std::vector<bool> computeVFSUsageAndClear() const;
584+
579585
/// This method returns a HeaderMap for the specified
580586
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
581587
const HeaderMap *CreateHeaderMap(FileEntryRef FE);

‎clang/include/clang/Lex/HeaderSearchOptions.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,10 @@ class HeaderSearchOptions {
263263
LLVM_PREFERRED_TYPE(bool)
264264
unsigned ModulesStrictContextHash : 1;
265265

266+
/// Whether to include ivfsoverlay usage information in written AST files.
267+
LLVM_PREFERRED_TYPE(bool)
268+
unsigned ModulesIncludeVFSUsage : 1;
269+
266270
HeaderSearchOptions(StringRef _Sysroot = "/")
267271
: Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(false),
268272
ImplicitModuleMaps(false), ModuleMapFileHomeIsCwd(false),
@@ -277,7 +281,7 @@ class HeaderSearchOptions {
277281
ModulesSkipDiagnosticOptions(false),
278282
ModulesSkipHeaderSearchPaths(false),
279283
ModulesSkipPragmaDiagnosticMappings(false), ModulesHashContent(false),
280-
ModulesStrictContextHash(false) {}
284+
ModulesStrictContextHash(false), ModulesIncludeVFSUsage(false) {}
281285

282286
/// AddPath - Add the \p Path path to the specified \p Group list.
283287
void AddPath(StringRef Path, frontend::IncludeDirGroup Group,

‎clang/include/clang/Serialization/ASTBitCodes.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,9 @@ enum UnhashedControlBlockRecordTypes {
405405

406406
/// Record code for the indices of used header search entries.
407407
HEADER_SEARCH_ENTRY_USAGE,
408+
409+
/// Record code for the indices of used VFSs.
410+
VFS_USAGE,
408411
};
409412

410413
/// Record code for extension blocks.

‎clang/include/clang/Serialization/ASTReader.h

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1780,12 +1780,13 @@ class ASTReader
17801780
/// Read the control block for the named AST file.
17811781
///
17821782
/// \returns true if an error occurred, false otherwise.
1783-
static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr,
1784-
const InMemoryModuleCache &ModuleCache,
1785-
const PCHContainerReader &PCHContainerRdr,
1786-
bool FindModuleFileExtensions,
1787-
ASTReaderListener &Listener,
1788-
bool ValidateDiagnosticOptions);
1783+
static bool readASTFileControlBlock(
1784+
StringRef Filename, FileManager &FileMgr,
1785+
const InMemoryModuleCache &ModuleCache,
1786+
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
1787+
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
1788+
unsigned DefaultClientLoadCapabilities = ARR_ConfigurationMismatch |
1789+
ARR_OutOfDate);
17891790

17901791
/// Determine whether the given AST file is acceptable to load into a
17911792
/// translation unit with the given language and target options.
@@ -2270,6 +2271,9 @@ class ASTReader
22702271
SourceRange ReadSourceRange(ModuleFile &F, const RecordData &Record,
22712272
unsigned &Idx, LocSeq *Seq = nullptr);
22722273

2274+
static llvm::BitVector ReadBitVector(const RecordData &Record,
2275+
const StringRef Blob);
2276+
22732277
// Read a string
22742278
static std::string ReadString(const RecordDataImpl &Record, unsigned &Idx);
22752279

‎clang/include/clang/Serialization/ModuleFile.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ class ModuleFile {
189189
/// The bit vector denoting usage of each header search entry (true = used).
190190
llvm::BitVector SearchPathUsage;
191191

192+
/// The bit vector denoting usage of each VFS entry (true = used).
193+
llvm::BitVector VFSUsage;
194+
192195
/// Whether this module has been directly imported by the
193196
/// user.
194197
bool DirectlyImported = false;

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,12 @@ class EntryRef {
280280
/// This is not a thread safe VFS. A single instance is meant to be used only in
281281
/// one thread. Multiple instances are allowed to service multiple threads
282282
/// running in parallel.
283-
class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
283+
class DependencyScanningWorkerFilesystem
284+
: public llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
285+
llvm::vfs::ProxyFileSystem> {
284286
public:
287+
static const char ID;
288+
285289
DependencyScanningWorkerFilesystem(
286290
DependencyScanningFilesystemSharedCache &SharedCache,
287291
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ enum class ScanningOutputFormat {
4545
P1689,
4646
};
4747

48+
#define DSS_LAST_BITMASK_ENUM(Id) \
49+
LLVM_MARK_AS_BITMASK_ENUM(Id), All = llvm::NextPowerOf2(Id) - 1
50+
4851
enum class ScanningOptimizations {
4952
None = 0,
5053

@@ -54,11 +57,15 @@ enum class ScanningOptimizations {
5457
/// Remove warnings from system modules.
5558
SystemWarnings = 2,
5659

57-
LLVM_MARK_AS_BITMASK_ENUM(SystemWarnings),
58-
All = HeaderSearch | SystemWarnings,
60+
/// Remove unused -ivfsoverlay arguments.
61+
VFS = 4,
62+
63+
DSS_LAST_BITMASK_ENUM(VFS),
5964
Default = All
6065
};
6166

67+
#undef DSS_LAST_BITMASK_ENUM
68+
6269
/// The dependency scanning service contains shared configuration and state that
6370
/// is used by the individual dependency scanning workers.
6471
class DependencyScanningService {

‎clang/lib/Basic/FileManager.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,13 @@ llvm::Expected<FileEntryRef> FileManager::getSTDIN() {
363363
return *STDIN;
364364
}
365365

366+
void FileManager::trackVFSUsage(bool Active) {
367+
FS->visit([Active](llvm::vfs::FileSystem &FileSys) {
368+
if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FileSys))
369+
RFS->setUsageTrackingActive(Active);
370+
});
371+
}
372+
366373
const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size,
367374
time_t ModificationTime) {
368375
return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry();

‎clang/lib/Frontend/CompilerInvocation.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4763,6 +4763,7 @@ std::string CompilerInvocation::getModuleHash() const {
47634763
if (hsOpts.ModulesStrictContextHash) {
47644764
HBuilder.addRange(hsOpts.SystemHeaderPrefixes);
47654765
HBuilder.addRange(hsOpts.UserEntries);
4766+
HBuilder.addRange(hsOpts.VFSOverlayFiles);
47664767

47674768
const DiagnosticOptions &diagOpts = getDiagnosticOpts();
47684769
#define DIAGOPT(Name, Bits, Default) HBuilder.add(diagOpts.Name);

‎clang/lib/Lex/HeaderSearch.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,22 @@ std::vector<bool> HeaderSearch::computeUserEntryUsage() const {
141141
return UserEntryUsage;
142142
}
143143

144+
std::vector<bool> HeaderSearch::computeVFSUsageAndClear() const {
145+
std::vector<bool> VFSUsage;
146+
llvm::vfs::FileSystem &RootFS = FileMgr.getVirtualFileSystem();
147+
// TODO: This only works if the `RedirectingFileSystem`s were all created by
148+
// `createVFSFromOverlayFiles`.
149+
RootFS.visit([&](llvm::vfs::FileSystem &FS) {
150+
if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FS)) {
151+
VFSUsage.push_back(RFS->hasBeenUsed());
152+
RFS->clearHasBeenUsed();
153+
}
154+
});
155+
// VFS visit order is the opposite of VFSOverlayFiles order.
156+
std::reverse(VFSUsage.begin(), VFSUsage.end());
157+
return VFSUsage;
158+
}
159+
144160
/// CreateHeaderMap - This method returns a HeaderMap for the specified
145161
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
146162
const HeaderMap *HeaderSearch::CreateHeaderMap(FileEntryRef FE) {

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4977,7 +4977,7 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
49774977
}
49784978
case HEADER_SEARCH_PATHS: {
49794979
bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
4980-
if (!AllowCompatibleConfigurationMismatch &&
4980+
if (Listener && !AllowCompatibleConfigurationMismatch &&
49814981
ParseHeaderSearchPaths(Record, Complain, *Listener))
49824982
Result = ConfigurationMismatch;
49834983
break;
@@ -4991,18 +4991,17 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
49914991
F->PragmaDiagMappings.insert(F->PragmaDiagMappings.end(),
49924992
Record.begin(), Record.end());
49934993
break;
4994-
case HEADER_SEARCH_ENTRY_USAGE:
4995-
if (!F)
4996-
break;
4997-
unsigned Count = Record[0];
4998-
const char *Byte = Blob.data();
4999-
F->SearchPathUsage = llvm::BitVector(Count, false);
5000-
for (unsigned I = 0; I < Count; ++Byte)
5001-
for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
5002-
if (*Byte & (1 << Bit))
5003-
F->SearchPathUsage[I] = true;
4994+
case HEADER_SEARCH_ENTRY_USAGE: {
4995+
if (F)
4996+
F->SearchPathUsage = ReadBitVector(Record, Blob);
4997+
break;
4998+
}
4999+
case VFS_USAGE: {
5000+
if (F)
5001+
F->VFSUsage = ReadBitVector(Record, Blob);
50045002
break;
50055003
}
5004+
}
50065005
}
50075006
}
50085007

@@ -5398,7 +5397,8 @@ bool ASTReader::readASTFileControlBlock(
53985397
StringRef Filename, FileManager &FileMgr,
53995398
const InMemoryModuleCache &ModuleCache,
54005399
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
5401-
ASTReaderListener &Listener, bool ValidateDiagnosticOptions) {
5400+
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
5401+
unsigned DefaultClientLoadCapabilities) {
54025402
// Open the AST file.
54035403
std::unique_ptr<llvm::MemoryBuffer> OwnedBuffer;
54045404
llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename);
@@ -5453,7 +5453,7 @@ bool ASTReader::readASTFileControlBlock(
54535453
switch (Entry.ID) {
54545454
case OPTIONS_BLOCK_ID: {
54555455
std::string IgnoredSuggestedPredefines;
5456-
if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate,
5456+
if (ReadOptionsBlock(Stream, DefaultClientLoadCapabilities,
54575457
/*AllowCompatibleConfigurationMismatch*/ false,
54585458
Listener, IgnoredSuggestedPredefines) != Success)
54595459
return true;
@@ -5679,7 +5679,7 @@ bool ASTReader::readASTFileControlBlock(
56795679

56805680
// Scan for the UNHASHED_CONTROL_BLOCK_ID block.
56815681
if (readUnhashedControlBlockImpl(
5682-
nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate,
5682+
nullptr, Bytes, DefaultClientLoadCapabilities,
56835683
/*AllowCompatibleConfigurationMismatch*/ false, &Listener,
56845684
ValidateDiagnosticOptions) != Success)
56855685
return true;
@@ -9312,6 +9312,19 @@ SourceRange ASTReader::ReadSourceRange(ModuleFile &F, const RecordData &Record,
93129312
return SourceRange(beg, end);
93139313
}
93149314

9315+
llvm::BitVector ASTReader::ReadBitVector(const RecordData &Record,
9316+
const StringRef Blob) {
9317+
llvm::BitVector Ret;
9318+
unsigned Count = Record[0];
9319+
const char *Byte = Blob.data();
9320+
Ret = llvm::BitVector(Count, false);
9321+
for (unsigned I = 0; I < Count; ++Byte)
9322+
for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
9323+
if (*Byte & (1 << Bit))
9324+
Ret[I] = true;
9325+
return Ret;
9326+
}
9327+
93159328
/// Read a floating-point value
93169329
llvm::APFloat ASTRecordReader::readAPFloat(const llvm::fltSemantics &Sem) {
93179330
return llvm::APFloat(Sem, readAPInt());

‎clang/lib/Serialization/ASTWriter.cpp

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,18 +1265,30 @@ void ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
12651265
WritePragmaDiagnosticMappings(Diags, /* isModule = */ WritingModule);
12661266

12671267
// Header search entry usage.
1268-
auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
1269-
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1270-
Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
1271-
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1272-
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1273-
unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
12741268
{
1269+
auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
1270+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1271+
Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
1272+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1273+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1274+
unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
12751275
RecordData::value_type Record[] = {HEADER_SEARCH_ENTRY_USAGE,
12761276
HSEntryUsage.size()};
12771277
Stream.EmitRecordWithBlob(HSUsageAbbrevCode, Record, bytes(HSEntryUsage));
12781278
}
12791279

1280+
// VFS usage.
1281+
{
1282+
auto VFSUsage = PP.getHeaderSearchInfo().computeVFSUsageAndClear();
1283+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1284+
Abbrev->Add(BitCodeAbbrevOp(VFS_USAGE));
1285+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1286+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1287+
unsigned VFSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
1288+
RecordData::value_type Record[] = {VFS_USAGE, VFSUsage.size()};
1289+
Stream.EmitRecordWithBlob(VFSUsageAbbrevCode, Record, bytes(VFSUsage));
1290+
}
1291+
12801292
// Leave the options block.
12811293
Stream.ExitBlock();
12821294
UnhashedControlBlockRange.second = Stream.GetCurrentBitNo() >> 3;
@@ -4687,6 +4699,15 @@ void ASTWriter::collectNonAffectingInputFiles() {
46874699
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
46884700
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
46894701

4702+
const bool IncludeVFSUsage =
4703+
PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage;
4704+
FileManager &FileMgr = PP->getFileManager();
4705+
FileMgr.trackVFSUsage(true);
4706+
if (IncludeVFSUsage)
4707+
for (StringRef Path :
4708+
PP->getHeaderSearchInfo().getHeaderSearchOpts().VFSOverlayFiles)
4709+
FileMgr.getVirtualFileSystem().exists(Path);
4710+
46904711
for (unsigned I = 1; I != N; ++I) {
46914712
const SrcMgr::SLocEntry *SLoc = &SrcMgr.getLocalSLocEntry(I);
46924713
FileID FID = FileID::get(I);
@@ -4701,8 +4722,13 @@ void ASTWriter::collectNonAffectingInputFiles() {
47014722

47024723
if (!isModuleMap(File.getFileCharacteristic()) ||
47034724
AffectingModuleMaps.empty() ||
4704-
llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry))
4725+
llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry)) {
4726+
// Lookup the path in the VFS to trigger `-ivfsoverlay` usage tracking.
4727+
if (IncludeVFSUsage)
4728+
FileMgr.getVirtualFileSystem().exists(
4729+
File.getContentCache().OrigEntry->getNameAsRequested());
47054730
continue;
4731+
}
47064732

47074733
IsSLocAffecting[I] = false;
47084734

@@ -4727,6 +4753,7 @@ void ASTWriter::collectNonAffectingInputFiles() {
47274753
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
47284754
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
47294755
}
4756+
FileMgr.trackVFSUsage(false);
47304757
}
47314758

47324759
ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,

‎clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,9 @@ static bool shouldCacheStatFailures(StringRef Filename) {
194194
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
195195
DependencyScanningFilesystemSharedCache &SharedCache,
196196
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
197-
: ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
197+
: llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
198+
llvm::vfs::ProxyFileSystem>(std::move(FS)),
199+
SharedCache(SharedCache),
198200
WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
199201
updateWorkingDirForCacheLookup();
200202
}
@@ -379,3 +381,5 @@ void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
379381
assert(!WorkingDirForCacheLookup ||
380382
llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
381383
}
384+
385+
const char DependencyScanningWorkerFilesystem::ID = 0;

0 commit comments

Comments
 (0)