Skip to content

Commit 136fc48

Browse files
BigcheeseBigcheese
committed
[clang][DependencyScanner] Remove unused -ivfsoverlay files
`-ivfsoverlay` files are unused when building most modules. Enable removing them by, * adding a way to visit the filesystem tree with extensible RTTI to access each `RedirectingFileSystem`. * Adding tracking to `RedirectingFileSystem` to record when it actually redirects a file access. * Storing this information in each PCM. Usage tracking is only enabled when iterating over the source manager and affecting modulemaps. Here each path is stated to cause an access. During scanning these stats all hit the cache. llvm#73734 rdar://118030978
1 parent 5b1084a commit 136fc48

27 files changed

+945
-82
lines changed

‎clang/include/clang/Basic/DiagnosticSerializationKinds.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ def err_pch_diagopt_mismatch : Error<"%0 is currently enabled, but was not in "
4444
"the PCH file">;
4545
def err_pch_modulecache_mismatch : Error<"PCH was compiled with module cache "
4646
"path '%0', but the path is currently '%1'">;
47+
def err_pch_vfsoverlay_mismatch : Error<"PCH was compiled with different VFS overlay files than are currently in use">;
48+
def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has the following VFS overlays:\n%1">;
49+
def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">;
4750

4851
def err_pch_version_too_old : Error<
4952
"PCH file uses an older PCH format that is no longer supported">;

‎clang/include/clang/Basic/FileManager.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,10 @@ class FileManager : public RefCountedBase<FileManager> {
248248
return FS;
249249
}
250250

251+
/// Enable or disable tracking of VFS usage. Used to not track full header
252+
/// search and implicit modulemap lookup.
253+
void trackVFSUsage(bool Active);
254+
251255
void setVirtualFileSystem(IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
252256
this->FS = std::move(FS);
253257
}

‎clang/include/clang/Lex/HeaderSearch.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,13 @@ class HeaderSearch {
571571
/// Note: implicit module maps don't contribute to entry usage.
572572
std::vector<bool> computeUserEntryUsage() const;
573573

574+
/// Collect which HeaderSearchOptions::VFSOverlayFiles have been meaningfully
575+
/// used so far and mark their index with 'true' in the resulting bit vector.
576+
///
577+
/// Note: this ignores VFSs that redirect non-affecting files such as unused
578+
/// modulemaps.
579+
std::vector<bool> collectVFSUsageAndClear() const;
580+
574581
/// This method returns a HeaderMap for the specified
575582
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
576583
const HeaderMap *CreateHeaderMap(const FileEntry *FE);

‎clang/include/clang/Lex/HeaderSearchOptions.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,9 @@ class HeaderSearchOptions {
237237
/// diagnostics.
238238
unsigned ModulesStrictContextHash : 1;
239239

240+
/// Whether to include ivfsoverlay usage information in written AST files.
241+
unsigned ModulesIncludeVFSUsage : 1;
242+
240243
HeaderSearchOptions(StringRef _Sysroot = "/")
241244
: Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(false),
242245
ImplicitModuleMaps(false), ModuleMapFileHomeIsCwd(false),
@@ -250,7 +253,7 @@ class HeaderSearchOptions {
250253
ModulesSkipDiagnosticOptions(false),
251254
ModulesSkipHeaderSearchPaths(false),
252255
ModulesSkipPragmaDiagnosticMappings(false), ModulesHashContent(false),
253-
ModulesStrictContextHash(false) {}
256+
ModulesStrictContextHash(false), ModulesIncludeVFSUsage(false) {}
254257

255258
/// AddPath - Add the \p Path path to the specified \p Group list.
256259
void AddPath(StringRef Path, frontend::IncludeDirGroup Group,

‎clang/include/clang/Serialization/ASTBitCodes.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,9 @@ enum UnhashedControlBlockRecordTypes {
416416

417417
/// Record code for the indices of used header search entries.
418418
HEADER_SEARCH_ENTRY_USAGE,
419+
420+
/// Record code for the indices of used VFSs.
421+
VFS_USAGE,
419422
};
420423

421424
/// Record code for extension blocks.

‎clang/include/clang/Serialization/ASTReader.h

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1791,12 +1791,13 @@ class ASTReader
17911791
/// Read the control block for the named AST file.
17921792
///
17931793
/// \returns true if an error occurred, false otherwise.
1794-
static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr,
1795-
const InMemoryModuleCache &ModuleCache,
1796-
const PCHContainerReader &PCHContainerRdr,
1797-
bool FindModuleFileExtensions,
1798-
ASTReaderListener &Listener,
1799-
bool ValidateDiagnosticOptions);
1794+
static bool readASTFileControlBlock(
1795+
StringRef Filename, FileManager &FileMgr,
1796+
const InMemoryModuleCache &ModuleCache,
1797+
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
1798+
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
1799+
unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch |
1800+
ARR_OutOfDate);
18001801

18011802
/// Determine whether the given AST file is acceptable to load into a
18021803
/// translation unit with the given language and target options.
@@ -2278,6 +2279,9 @@ class ASTReader
22782279
SourceRange ReadSourceRange(ModuleFile &F, const RecordData &Record,
22792280
unsigned &Idx, LocSeq *Seq = nullptr);
22802281

2282+
static llvm::BitVector ReadBitVector(const RecordData &Record,
2283+
const StringRef Blob);
2284+
22812285
// Read a string
22822286
static std::string ReadString(const RecordDataImpl &Record, unsigned &Idx);
22832287

‎clang/include/clang/Serialization/ASTWriter.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,10 +467,10 @@ class ASTWriter : public ASTDeserializationListener,
467467
std::vector<SourceRange> NonAffectingRanges;
468468
std::vector<SourceLocation::UIntTy> NonAffectingOffsetAdjustments;
469469

470-
/// Collects input files that didn't affect compilation of the current module,
470+
/// Computes input files that didn't affect compilation of the current module,
471471
/// and initializes data structures necessary for leaving those files out
472472
/// during \c SourceManager serialization.
473-
void collectNonAffectingInputFiles();
473+
void computeNonAffectingInputFiles();
474474

475475
/// Returns an adjusted \c FileID, accounting for any non-affecting input
476476
/// files.

‎clang/include/clang/Serialization/ModuleFile.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ class ModuleFile {
200200
/// The bit vector denoting usage of each header search entry (true = used).
201201
llvm::BitVector SearchPathUsage;
202202

203+
/// The bit vector denoting usage of each VFS entry (true = used).
204+
llvm::BitVector VFSUsage;
205+
203206
/// Whether this module has been directly imported by the
204207
/// user.
205208
bool DirectlyImported = false;

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,8 +296,12 @@ class EntryRef {
296296
/// This is not a thread safe VFS. A single instance is meant to be used only in
297297
/// one thread. Multiple instances are allowed to service multiple threads
298298
/// running in parallel.
299-
class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
299+
class DependencyScanningWorkerFilesystem
300+
: public llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
301+
llvm::vfs::ProxyFileSystem> {
300302
public:
303+
static const char ID;
304+
301305
DependencyScanningWorkerFilesystem(
302306
DependencyScanningFilesystemSharedCache &SharedCache,
303307
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ enum class ScanningOutputFormat {
6161
P1689,
6262
};
6363

64+
#define DSS_LAST_BITMASK_ENUM(Id) \
65+
LLVM_MARK_AS_BITMASK_ENUM(Id), All = llvm::NextPowerOf2(Id) - 1
66+
6467
enum class ScanningOptimizations {
6568
None = 0,
6669

@@ -70,11 +73,15 @@ enum class ScanningOptimizations {
7073
/// Remove warnings from system modules.
7174
SystemWarnings = 2,
7275

73-
LLVM_MARK_AS_BITMASK_ENUM(SystemWarnings),
74-
All = HeaderSearch | SystemWarnings,
76+
/// Remove unused -ivfsoverlay arguments.
77+
VFS = 4,
78+
79+
DSS_LAST_BITMASK_ENUM(VFS),
7580
Default = All
7681
};
7782

83+
#undef DSS_LAST_BITMASK_ENUM
84+
7885
/// The dependency scanning service contains shared configuration and state that
7986
/// is used by the individual dependency scanning workers.
8087
class DependencyScanningService {

‎clang/lib/Basic/FileManager.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,13 @@ llvm::Expected<FileEntryRef> FileManager::getSTDIN() {
388388
return *STDIN;
389389
}
390390

391+
void FileManager::trackVFSUsage(bool Active) {
392+
FS->visit([Active](llvm::vfs::FileSystem &FileSys) {
393+
if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FileSys))
394+
RFS->setUsageTrackingActive(Active);
395+
});
396+
}
397+
391398
const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size,
392399
time_t ModificationTime) {
393400
return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry();

‎clang/lib/Lex/HeaderSearch.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,28 @@ std::vector<bool> HeaderSearch::computeUserEntryUsage() const {
142142
return UserEntryUsage;
143143
}
144144

145+
std::vector<bool> HeaderSearch::collectVFSUsageAndClear() const {
146+
std::vector<bool> VFSUsage;
147+
if (!getHeaderSearchOpts().ModulesIncludeVFSUsage)
148+
return VFSUsage;
149+
150+
llvm::vfs::FileSystem &RootFS = FileMgr.getVirtualFileSystem();
151+
// TODO: This only works if the `RedirectingFileSystem`s were all created by
152+
// `createVFSFromOverlayFiles`.
153+
RootFS.visit([&](llvm::vfs::FileSystem &FS) {
154+
if (auto *RFS = dyn_cast<llvm::vfs::RedirectingFileSystem>(&FS)) {
155+
VFSUsage.push_back(RFS->hasBeenUsed());
156+
RFS->clearHasBeenUsed();
157+
}
158+
});
159+
assert(VFSUsage.size() == getHeaderSearchOpts().VFSOverlayFiles.size() &&
160+
"A different number of RedirectingFileSystem's were present than "
161+
"-ivfsoverlay options passed to Clang!");
162+
// VFS visit order is the opposite of VFSOverlayFiles order.
163+
std::reverse(VFSUsage.begin(), VFSUsage.end());
164+
return VFSUsage;
165+
}
166+
145167
/// CreateHeaderMap - This method returns a HeaderMap for the specified
146168
/// FileEntry, uniquing them through the 'HeaderMaps' datastructure.
147169
const HeaderMap *HeaderSearch::CreateHeaderMap(const FileEntry *FE) {

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4961,7 +4961,7 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
49614961
}
49624962
case HEADER_SEARCH_PATHS: {
49634963
bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
4964-
if (!AllowCompatibleConfigurationMismatch &&
4964+
if (Listener && !AllowCompatibleConfigurationMismatch &&
49654965
ParseHeaderSearchPaths(Record, Complain, *Listener))
49664966
Result = ConfigurationMismatch;
49674967
break;
@@ -4984,15 +4984,12 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl(
49844984
Record.begin(), Record.end());
49854985
break;
49864986
case HEADER_SEARCH_ENTRY_USAGE:
4987-
if (!F)
4988-
break;
4989-
unsigned Count = Record[0];
4990-
const char *Byte = Blob.data();
4991-
F->SearchPathUsage = llvm::BitVector(Count, false);
4992-
for (unsigned I = 0; I < Count; ++Byte)
4993-
for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
4994-
if (*Byte & (1 << Bit))
4995-
F->SearchPathUsage[I] = true;
4987+
if (F)
4988+
F->SearchPathUsage = ReadBitVector(Record, Blob);
4989+
break;
4990+
case VFS_USAGE:
4991+
if (F)
4992+
F->VFSUsage = ReadBitVector(Record, Blob);
49964993
break;
49974994
}
49984995
}
@@ -5390,7 +5387,8 @@ bool ASTReader::readASTFileControlBlock(
53905387
StringRef Filename, FileManager &FileMgr,
53915388
const InMemoryModuleCache &ModuleCache,
53925389
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
5393-
ASTReaderListener &Listener, bool ValidateDiagnosticOptions) {
5390+
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
5391+
unsigned ClientLoadCapabilities) {
53945392
// Open the AST file.
53955393
std::unique_ptr<llvm::MemoryBuffer> OwnedBuffer;
53965394
llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename);
@@ -5445,7 +5443,7 @@ bool ASTReader::readASTFileControlBlock(
54455443
switch (Entry.ID) {
54465444
case OPTIONS_BLOCK_ID: {
54475445
std::string IgnoredSuggestedPredefines;
5448-
if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate,
5446+
if (ReadOptionsBlock(Stream, ClientLoadCapabilities,
54495447
/*AllowCompatibleConfigurationMismatch*/ false,
54505448
Listener, IgnoredSuggestedPredefines) != Success)
54515449
return true;
@@ -5659,7 +5657,7 @@ bool ASTReader::readASTFileControlBlock(
56595657

56605658
// Scan for the UNHASHED_CONTROL_BLOCK_ID block.
56615659
if (readUnhashedControlBlockImpl(
5662-
nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate,
5660+
nullptr, Bytes, ClientLoadCapabilities,
56635661
/*AllowCompatibleConfigurationMismatch*/ false, &Listener,
56645662
ValidateDiagnosticOptions) != Success)
56655663
return true;
@@ -9288,6 +9286,18 @@ SourceRange ASTReader::ReadSourceRange(ModuleFile &F, const RecordData &Record,
92889286
return SourceRange(beg, end);
92899287
}
92909288

9289+
llvm::BitVector ASTReader::ReadBitVector(const RecordData &Record,
9290+
const StringRef Blob) {
9291+
unsigned Count = Record[0];
9292+
const char *Byte = Blob.data();
9293+
llvm::BitVector Ret = llvm::BitVector(Count, false);
9294+
for (unsigned I = 0; I < Count; ++Byte)
9295+
for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I)
9296+
if (*Byte & (1 << Bit))
9297+
Ret[I] = true;
9298+
return Ret;
9299+
}
9300+
92919301
/// Read a floating-point value
92929302
llvm::APFloat ASTRecordReader::readAPFloat(const llvm::fltSemantics &Sem) {
92939303
return llvm::APFloat(Sem, readAPInt());

‎clang/lib/Serialization/ASTWriter.cpp

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1271,18 +1271,30 @@ void ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
12711271
WritePragmaDiagnosticMappings(Diags, /* isModule = */ WritingModule);
12721272

12731273
// Header search entry usage.
1274-
auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
1275-
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1276-
Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
1277-
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1278-
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1279-
unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
12801274
{
1275+
auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage();
1276+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1277+
Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE));
1278+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1279+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1280+
unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
12811281
RecordData::value_type Record[] = {HEADER_SEARCH_ENTRY_USAGE,
12821282
HSEntryUsage.size()};
12831283
Stream.EmitRecordWithBlob(HSUsageAbbrevCode, Record, bytes(HSEntryUsage));
12841284
}
12851285

1286+
// VFS usage.
1287+
{
1288+
auto VFSUsage = PP.getHeaderSearchInfo().collectVFSUsageAndClear();
1289+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1290+
Abbrev->Add(BitCodeAbbrevOp(VFS_USAGE));
1291+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits.
1292+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector.
1293+
unsigned VFSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
1294+
RecordData::value_type Record[] = {VFS_USAGE, VFSUsage.size()};
1295+
Stream.EmitRecordWithBlob(VFSUsageAbbrevCode, Record, bytes(VFSUsage));
1296+
}
1297+
12861298
// Leave the options block.
12871299
Stream.ExitBlock();
12881300
UnhashedControlBlockRange.second = Stream.GetCurrentBitNo() >> 3;
@@ -4639,7 +4651,7 @@ static void AddLazyVectorDecls(ASTWriter &Writer, Vector &Vec,
46394651
}
46404652
}
46414653

4642-
void ASTWriter::collectNonAffectingInputFiles() {
4654+
void ASTWriter::computeNonAffectingInputFiles() {
46434655
SourceManager &SrcMgr = PP->getSourceManager();
46444656
unsigned N = SrcMgr.local_sloc_entry_size();
46454657

@@ -4699,6 +4711,30 @@ void ASTWriter::collectNonAffectingInputFiles() {
46994711
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
47004712
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
47014713
}
4714+
4715+
if (!PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage)
4716+
return;
4717+
4718+
FileManager &FileMgr = PP->getFileManager();
4719+
FileMgr.trackVFSUsage(true);
4720+
// Lookup the paths in the VFS to trigger `-ivfsoverlay` usage tracking.
4721+
for (StringRef Path :
4722+
PP->getHeaderSearchInfo().getHeaderSearchOpts().VFSOverlayFiles)
4723+
FileMgr.getVirtualFileSystem().exists(Path);
4724+
for (unsigned I = 1; I != N; ++I) {
4725+
if (IsSLocAffecting[I]) {
4726+
const SrcMgr::SLocEntry *SLoc = &SrcMgr.getLocalSLocEntry(I);
4727+
if (!SLoc->isFile())
4728+
continue;
4729+
const SrcMgr::FileInfo &File = SLoc->getFile();
4730+
const SrcMgr::ContentCache *Cache = &File.getContentCache();
4731+
if (!Cache->OrigEntry)
4732+
continue;
4733+
FileMgr.getVirtualFileSystem().exists(
4734+
Cache->OrigEntry->getNameAsRequested());
4735+
}
4736+
}
4737+
FileMgr.trackVFSUsage(false);
47024738
}
47034739

47044740
ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
@@ -4716,7 +4752,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
47164752

47174753
// This needs to be done very early, since everything that writes
47184754
// SourceLocations or FileIDs depends on it.
4719-
collectNonAffectingInputFiles();
4755+
computeNonAffectingInputFiles();
47204756

47214757
writeUnhashedControlBlock(PP, Context);
47224758

‎clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,9 @@ static bool shouldCacheStatFailures(StringRef Filename) {
200200
DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
201201
DependencyScanningFilesystemSharedCache &SharedCache,
202202
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
203-
: ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
203+
: llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
204+
llvm::vfs::ProxyFileSystem>(std::move(FS)),
205+
SharedCache(SharedCache),
204206
WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
205207
updateWorkingDirForCacheLookup();
206208
}
@@ -392,3 +394,5 @@ void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
392394
assert(!WorkingDirForCacheLookup ||
393395
llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
394396
}
397+
398+
const char DependencyScanningWorkerFilesystem::ID = 0;

0 commit comments

Comments
 (0)