Skip to content

Commit 0532abb

Browse files
benlangmuirbenlangmuir
authored
Merge pull request #5645 from benlangmuir/eng/blangmuir/cache-modules-v7-next-v3-stable-20221013
[stable/20221013] Initial support for caching modules using casfs
2 parents 87e1e3d + cdd72b7 commit 0532abb

31 files changed

+1562
-66
lines changed

‎clang/include/clang/Basic/DiagnosticCASKinds.td

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ def err_cas_depscan_failed: Error<
2929
def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal;
3030
def err_cas_cannot_get_module_cache_key : Error<
3131
"CAS cannot load module with key '%0' from %1: %2">, DefaultFatal;
32+
def err_cas_missing_root_id : Error<
33+
"CAS missing expected root-id '%0'">, DefaultFatal;
34+
def err_cas_cannot_parse_root_id_for_module : Error<
35+
"CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal;
3236

3337
def warn_clang_cache_disabled_caching: Warning<
3438
"caching disabled because %0">,

‎clang/include/clang/Basic/Module.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ class alignas(8) Module {
192192
/// The \c ActionCache key for this module, if any.
193193
Optional<std::string> ModuleCacheKey;
194194

195+
/// The CAS filesystem root ID for implicit modules built with the dependency
196+
/// scanner, if any.
197+
Optional<std::string> CASFileSystemRootID;
198+
195199
/// The top-level headers associated with this module.
196200
llvm::SmallSetVector<const FileEntry *, 2> TopHeaders;
197201

@@ -632,6 +636,15 @@ class alignas(8) Module {
632636
getTopLevelModule()->ModuleCacheKey = std::move(Key);
633637
}
634638

639+
Optional<std::string> getCASFileSystemRootID() const {
640+
return getTopLevelModule()->CASFileSystemRootID;
641+
}
642+
643+
void setCASFileSystemRootID(std::string ID) {
644+
assert(!getCASFileSystemRootID() || *getCASFileSystemRootID() == ID);
645+
getTopLevelModule()->CASFileSystemRootID = std::move(ID);
646+
}
647+
635648
/// Retrieve the directory for which this module serves as the
636649
/// umbrella.
637650
DirectoryName getUmbrellaDir() const;

‎clang/include/clang/Frontend/CompilerInstance.h

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,13 @@ class CompilerInstance : public ModuleLoader {
175175
/// The list of active output files.
176176
std::list<llvm::vfs::OutputFile> OutputFiles;
177177

178-
/// \brief An optional callback function used to wrap all FrontendActions
178+
using GenModuleActionWrapperFunc =
179+
std::function<std::unique_ptr<FrontendAction>(
180+
const FrontendOptions &opts, std::unique_ptr<FrontendAction> action)>;
181+
182+
/// An optional callback function used to wrap all FrontendActions
179183
/// produced to generate imported modules before they are executed.
180-
std::function<std::unique_ptr<FrontendAction>
181-
(const FrontendOptions &opts, std::unique_ptr<FrontendAction> action)>
182-
GenModuleActionWrapper;
184+
GenModuleActionWrapperFunc GenModuleActionWrapper;
183185

184186
/// Force an output buffer.
185187
std::unique_ptr<llvm::raw_pwrite_stream> OutputStream;
@@ -858,14 +860,13 @@ class CompilerInstance : public ModuleLoader {
858860

859861
bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override;
860862

861-
void setGenModuleActionWrapper(std::function<std::unique_ptr<FrontendAction>
862-
(const FrontendOptions &Opts, std::unique_ptr<FrontendAction> Action)> Wrapper) {
863+
void setGenModuleActionWrapper(GenModuleActionWrapperFunc Wrapper) {
863864
GenModuleActionWrapper = Wrapper;
864-
};
865+
}
865866

866-
std::function<std::unique_ptr<FrontendAction>
867-
(const FrontendOptions &Opts, std::unique_ptr<FrontendAction> Action)>
868-
getGenModuleActionWrapper() const { return GenModuleActionWrapper; }
867+
GenModuleActionWrapperFunc getGenModuleActionWrapper() const {
868+
return GenModuleActionWrapper;
869+
}
869870

870871
void addDependencyCollector(std::shared_ptr<DependencyCollector> Listener) {
871872
DependencyCollectors.push_back(std::move(Listener));

‎clang/include/clang/Serialization/ASTBitCodes.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,10 @@ enum ControlRecordTypes {
363363

364364
/// Record code for the (optional) \c ActionCache key for this module.
365365
MODULE_CACHE_KEY,
366+
367+
/// Record code for the (optional) CAS filesystem root ID for implicit modules
368+
/// built with the dependency scanner.
369+
CASFS_ROOT_ID,
366370
};
367371

368372
/// Record types that occur within the options block inside

‎clang/include/clang/Serialization/ModuleFile.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,10 @@ class ModuleFile {
128128
/// The \c ActionCache key for this module, or empty.
129129
std::string ModuleCacheKey;
130130

131+
/// The CAS filesystem root ID for implicit modules built with the dependency
132+
/// scanner, or empty.
133+
std::string CASFileSystemRootID;
134+
131135
/// The name of the module.
132136
std::string ModuleName;
133137

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem {
6363
createThreadSafeProxyFS() override;
6464

6565
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
66+
bool exists(const Twine &Path) override;
6667
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
6768
openFileForRead(const Twine &Path) override;
6869

‎clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "clang/Serialization/ASTReader.h"
1919
#include "llvm/ADT/DenseMap.h"
2020
#include "llvm/ADT/StringSet.h"
21+
#include "llvm/CAS/CASID.h"
2122
#include "llvm/Support/raw_ostream.h"
2223
#include <string>
2324
#include <unordered_map>
@@ -33,11 +34,13 @@ struct PrebuiltModuleDep {
3334
std::string ModuleName;
3435
std::string PCMFile;
3536
std::string ModuleMapFile;
37+
Optional<std::string> ModuleCacheKey;
3638

3739
explicit PrebuiltModuleDep(const Module *M)
3840
: ModuleName(M->getTopLevelModuleName()),
3941
PCMFile(M->getASTFile()->getName()),
40-
ModuleMapFile(M->PresumedModuleMapFile) {}
42+
ModuleMapFile(M->PresumedModuleMapFile),
43+
ModuleCacheKey(M->getModuleCacheKey()) {}
4144
};
4245

4346
/// This is used to identify a specific module.
@@ -113,6 +116,12 @@ struct ModuleDeps {
113116
// the primary TU.
114117
bool ImportedByMainFile = false;
115118

119+
/// The CASID for the module input dependency tree, if any.
120+
llvm::Optional<llvm::cas::CASID> CASFileSystemRootID;
121+
122+
/// The \c ActionCache key for this module, if any.
123+
llvm::Optional<std::string> ModuleCacheKey;
124+
116125
/// Compiler invocation that can be used to build this module. Does not
117126
/// include argv[0].
118127
std::vector<std::string> BuildArguments;
@@ -191,6 +200,10 @@ class ModuleDepCollector final : public DependencyCollector {
191200
/// invocation, (e.g. disable implicit modules, add explicit module paths).
192201
void applyDiscoveredDependencies(CompilerInvocation &CI);
193202

203+
/// Set a CAS filesystem root ID for the main file, which will be included by
204+
/// \c applyDiscoveredDependencies.
205+
void setMainFileCASFileSystemRootID(cas::CASID ID);
206+
194207
private:
195208
friend ModuleDepCollectorPP;
196209

@@ -216,6 +229,13 @@ class ModuleDepCollector final : public DependencyCollector {
216229
std::unique_ptr<DependencyOutputOptions> Opts;
217230
/// The original Clang invocation passed to dependency scanner.
218231
CompilerInvocation OriginalInvocation;
232+
/// The CAS filesystem root ID for the main input file, if any. This is used
233+
/// in \c applyDiscoveredDependencies.
234+
Optional<cas::CASID> MainFileCASFileSystemRootID;
235+
/// CAS options to apply to invocations.
236+
CASOptions CASOpts;
237+
/// CAS options to apply to invocations.
238+
bool CacheCompileJob;
219239
/// Whether to optimize the modules' command-line arguments.
220240
bool OptimizeArgs;
221241
/// Whether to set up command-lines to load PCM files eagerly.

‎clang/lib/Frontend/CompileJobCacheKey.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,7 @@ using namespace llvm::cas;
2626
static llvm::cas::CASID
2727
createCompileJobCacheKeyForArgs(ObjectStore &CAS,
2828
ArrayRef<const char *> CC1Args,
29-
llvm::cas::CASID RootID, bool IsIncludeTree) {
30-
Optional<llvm::cas::ObjectRef> RootRef = CAS.getReference(RootID);
31-
if (!RootRef)
32-
report_fatal_error(createStringError(
33-
inconvertibleErrorCode(),
34-
"cannot handle unknown compile-job cache key: " + RootID.toString()));
29+
ObjectRef RootRef, bool IsIncludeTree) {
3530
assert(!CC1Args.empty() && StringRef(CC1Args[0]) == "-cc1");
3631
SmallString<256> CommandLine;
3732
for (StringRef Arg : CC1Args) {
@@ -41,9 +36,9 @@ createCompileJobCacheKeyForArgs(ObjectStore &CAS,
4136

4237
llvm::cas::HierarchicalTreeBuilder Builder;
4338
if (IsIncludeTree) {
44-
Builder.push(*RootRef, llvm::cas::TreeEntry::Regular, "include-tree");
39+
Builder.push(RootRef, llvm::cas::TreeEntry::Regular, "include-tree");
4540
} else {
46-
Builder.push(*RootRef, llvm::cas::TreeEntry::Tree, "filesystem");
41+
Builder.push(RootRef, llvm::cas::TreeEntry::Tree, "filesystem");
4742
}
4843
Builder.push(
4944
llvm::cantFail(CAS.storeFromString(None, CommandLine)),
@@ -103,8 +98,13 @@ createCompileJobCacheKeyImpl(ObjectStore &CAS, DiagnosticsEngine &Diags,
10398
Diags.Report(diag::err_cas_cannot_parse_root_id) << RootIDString;
10499
return None;
105100
}
101+
Optional<llvm::cas::ObjectRef> RootRef = CAS.getReference(*RootID);
102+
if (!RootRef) {
103+
Diags.Report(diag::err_cas_missing_root_id) << RootIDString;
104+
return None;
105+
}
106106

107-
return createCompileJobCacheKeyForArgs(CAS, Argv, *RootID, IsIncludeTree);
107+
return createCompileJobCacheKeyForArgs(CAS, Argv, *RootRef, IsIncludeTree);
108108
}
109109

110110
static CompileJobCachingOptions

‎clang/lib/Frontend/CompilerInstance.cpp

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,8 +1264,8 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
12641264
FullSourceLoc(ImportLoc, ImportingInstance.getSourceManager()));
12651265

12661266
// Pass along the GenModuleActionWrapper callback
1267-
auto wrapGenModuleAction = ImportingInstance.getGenModuleActionWrapper();
1268-
Instance.setGenModuleActionWrapper(wrapGenModuleAction);
1267+
auto WrapGenModuleAction = ImportingInstance.getGenModuleActionWrapper();
1268+
Instance.setGenModuleActionWrapper(WrapGenModuleAction);
12691269

12701270
// Share an output manager.
12711271
assert(ImportingInstance.hasOutputBackend() &&
@@ -1288,13 +1288,10 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
12881288
// thread so that we get a stack large enough.
12891289
bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnThread(
12901290
[&]() {
1291-
// FIXME: I have no idea what the best way to do this is, but it's
1292-
// probably not this. Interfaces changed upstream.
12931291
std::unique_ptr<FrontendAction> Action(
12941292
new GenerateModuleFromModuleMapAction);
1295-
if (wrapGenModuleAction) {
1296-
Action = wrapGenModuleAction(FrontendOpts, std::move(Action));
1297-
}
1293+
if (WrapGenModuleAction)
1294+
Action = WrapGenModuleAction(FrontendOpts, std::move(Action));
12981295
Instance.ExecuteAction(*Action);
12991296
},
13001297
DesiredStackSize);

‎clang/lib/Frontend/CompilerInvocation.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4970,6 +4970,10 @@ std::string CompilerInvocation::getModuleHash(DiagnosticsEngine &Diags) const {
49704970
}
49714971
}
49724972

4973+
// Caching + implicit modules, which is only set in clang-scan-deps, puts
4974+
// additional CASIDs in the pcm.
4975+
HBuilder.add(getFrontendOpts().CacheCompileJob);
4976+
49734977
llvm::MD5::MD5Result Result;
49744978
HBuilder.getHasher().final(Result);
49754979
uint64_t Hash = Result.high() ^ Result.low();

‎clang/lib/Lex/ModuleMap.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -962,7 +962,8 @@ static void inferFrameworkLink(Module *Mod, const DirectoryEntry *FrameworkDir,
962962
// for both before we give up.
963963
for (const char *extension : {"", ".tbd"}) {
964964
llvm::sys::path::replace_extension(LibName, extension);
965-
if (FileMgr.getFile(LibName)) {
965+
// Use VFS exists to avoid depending on the file's contents in cached builds
966+
if (FileMgr.getVirtualFileSystem().exists(LibName)) {
966967
Mod->LinkLibraries.push_back(Module::LinkLibrary(Mod->Name,
967968
/*IsFramework=*/true));
968969
return;

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2961,6 +2961,10 @@ ASTReader::ReadControlBlock(ModuleFile &F,
29612961
case MODULE_CACHE_KEY:
29622962
F.ModuleCacheKey = Blob.str();
29632963
break;
2964+
2965+
case CASFS_ROOT_ID:
2966+
F.CASFileSystemRootID = Blob.str();
2967+
break;
29642968
}
29652969
}
29662970
}
@@ -5387,7 +5391,8 @@ bool ASTReader::readASTFileControlBlock(
53875391
std::string Filename = ReadString(Record, Idx);
53885392
ResolveImportedPath(Filename, ModuleDir);
53895393
std::string CacheKey = ReadString(Record, Idx);
5390-
Listener.readModuleCacheKey(ModuleName, Filename, CacheKey);
5394+
if (!CacheKey.empty())
5395+
Listener.readModuleCacheKey(ModuleName, Filename, CacheKey);
53915396
Listener.visitImport(ModuleName, Filename);
53925397
}
53935398
break;
@@ -5602,6 +5607,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
56025607
CurrentModule->PresumedModuleMapFile = F.ModuleMapPath;
56035608
if (!F.ModuleCacheKey.empty())
56045609
CurrentModule->setModuleCacheKey(F.ModuleCacheKey);
5610+
if (!F.CASFileSystemRootID.empty())
5611+
CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID);
56055612
}
56065613

56075614
CurrentModule->Kind = Kind;

‎clang/lib/Serialization/ASTWriter.cpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,7 @@ void ASTWriter::WriteBlockInfoBlock() {
788788
RECORD(ORIGINAL_FILE_ID);
789789
RECORD(INPUT_FILE_OFFSETS);
790790
RECORD(MODULE_CACHE_KEY);
791+
RECORD(CASFS_ROOT_ID);
791792

792793
BLOCK(OPTIONS_BLOCK);
793794
RECORD(LANGUAGE_OPTIONS);
@@ -1327,8 +1328,8 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
13271328
Stream.EmitRecord(MODULE_MAP_FILE, Record);
13281329
}
13291330

1330-
// Module Cache Key
13311331
if (WritingModule) {
1332+
// Module Cache Key
13321333
if (auto Key = WritingModule->getModuleCacheKey()) {
13331334
auto Abbrev = std::make_shared<BitCodeAbbrev>();
13341335
Abbrev->Add(BitCodeAbbrevOp(MODULE_CACHE_KEY));
@@ -1337,6 +1338,15 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
13371338
RecordData::value_type Record[] = {MODULE_CACHE_KEY};
13381339
Stream.EmitRecordWithBlob(AbbrevCode, Record, *Key);
13391340
}
1341+
// CAS filesystem root id, for the scanner.
1342+
if (auto ID = WritingModule->getCASFileSystemRootID()) {
1343+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1344+
Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID));
1345+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
1346+
unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
1347+
RecordData::value_type Record[] = {CASFS_ROOT_ID};
1348+
Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID);
1349+
}
13401350
}
13411351

13421352
// Imports

‎clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,13 @@ DependencyScanningCASFilesystem::status(const Twine &Path) {
299299
return Result.Entry->Status;
300300
}
301301

302+
bool DependencyScanningCASFilesystem::exists(const Twine &Path) {
303+
// Existence check does not require caching the result at the dependency
304+
// scanning level. The CachingOnDiskFileSystem tracks the exists call, which
305+
// ensures it is included in any resulting CASFileSystem.
306+
return getCachingFS().exists(Path);
307+
}
308+
302309
IntrusiveRefCntPtr<llvm::cas::ThreadSafeFileSystem>
303310
DependencyScanningCASFilesystem::createThreadSafeProxyFS() {
304311
llvm::report_fatal_error("not implemented");

0 commit comments

Comments
 (0)