Skip to content

Commit d1d4601

Browse files
jansvoboda11jansvoboda11
committed
[clang][modules] Introduce new ModuleCache interface (llvm#131193)
This PR adds new `ModuleCache` interface to Clang's implicitly-built modules machinery. The main motivation for this change is to create a second implementation that uses a more efficient kind of `llvm::AdvisoryLock` during dependency scanning. In addition to the lock abstraction, the `ModuleCache` interface also manages the existing `InMemoryModuleCache` instance. I found that compared to keeping these separate/independent, the code is a bit simpler now, since these are two tightly coupled concepts. I can envision a more efficient implementation of the `InMemoryModuleCache` for the single-process case too, which will be much easier to implement with the current setup. This is not intended to be a functional change. (cherry picked from commit c84d8e8)
1 parent e38be0f commit d1d4601

File tree

17 files changed

+220
-121
lines changed

17 files changed

+220
-121
lines changed

‎clang/include/clang/Frontend/ASTUnit.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ class FileManager;
7070
class FrontendAction;
7171
class HeaderSearch;
7272
class InputKind;
73-
class InMemoryModuleCache;
73+
class ModuleCache;
7474
class PCHContainerOperations;
7575
class PCHContainerReader;
7676
class Preprocessor;
@@ -110,7 +110,7 @@ class ASTUnit {
110110
IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
111111
IntrusiveRefCntPtr<FileManager> FileMgr;
112112
IntrusiveRefCntPtr<SourceManager> SourceMgr;
113-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
113+
IntrusiveRefCntPtr<ModuleCache> ModCache;
114114
std::unique_ptr<HeaderSearch> HeaderInfo;
115115
IntrusiveRefCntPtr<TargetInfo> Target;
116116
std::shared_ptr<Preprocessor> PP;

‎clang/include/clang/Frontend/CompilerInstance.h

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ class DiagnosticsEngine;
5454
class DiagnosticConsumer;
5555
class FileManager;
5656
class FrontendAction;
57-
class InMemoryModuleCache;
5857
class Module;
58+
class ModuleCache;
5959
class Preprocessor;
6060
class Sema;
6161
class SourceManager;
@@ -115,7 +115,7 @@ class CompilerInstance : public ModuleLoader {
115115
IntrusiveRefCntPtr<SourceManager> SourceMgr;
116116

117117
/// The cache of PCM files.
118-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
118+
IntrusiveRefCntPtr<ModuleCache> ModCache;
119119

120120
/// The preprocessor.
121121
std::shared_ptr<Preprocessor> PP;
@@ -223,7 +223,7 @@ class CompilerInstance : public ModuleLoader {
223223
explicit CompilerInstance(
224224
std::shared_ptr<PCHContainerOperations> PCHContainerOps =
225225
std::make_shared<PCHContainerOperations>(),
226-
InMemoryModuleCache *SharedModuleCache = nullptr);
226+
ModuleCache *ModCache = nullptr);
227227
~CompilerInstance() override;
228228

229229
/// @name High-Level Operations
@@ -793,9 +793,8 @@ class CompilerInstance : public ModuleLoader {
793793
static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource(
794794
StringRef Path, StringRef Sysroot,
795795
DisableValidationForModuleKind DisableValidation,
796-
bool AllowPCHWithCompilerErrors, Preprocessor &PP,
797-
InMemoryModuleCache &ModuleCache, ASTContext &Context,
798-
const PCHContainerReader &PCHContainerRdr,
796+
bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache,
797+
ASTContext &Context, const PCHContainerReader &PCHContainerRdr,
799798
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
800799
ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
801800
void *DeserializationListener, bool OwnDeserializationListener,
@@ -966,7 +965,7 @@ class CompilerInstance : public ModuleLoader {
966965
bool addCachedModuleFile(StringRef Path, StringRef CacheKey,
967966
StringRef Provider);
968967

969-
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
968+
ModuleCache &getModuleCache() const { return *ModCache; }
970969
};
971970

972971
} // end namespace clang

‎clang/include/clang/Serialization/ASTReader.h

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ struct HeaderFileInfo;
8888
class HeaderSearchOptions;
8989
class LangOptions;
9090
class MacroInfo;
91-
class InMemoryModuleCache;
91+
class ModuleCache;
9292
class NamedDecl;
9393
class NamespaceDecl;
9494
class ObjCCategoryDecl;
@@ -1641,8 +1641,8 @@ class ASTReader
16411641
///
16421642
/// \param ReadTimer If non-null, a timer used to track the time spent
16431643
/// deserializing.
1644-
ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
1645-
ASTContext *Context, const PCHContainerReader &PCHContainerRdr,
1644+
ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context,
1645+
const PCHContainerReader &PCHContainerRdr,
16461646
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
16471647
StringRef isysroot = "",
16481648
DisableValidationForModuleKind DisableValidationKind =
@@ -1853,8 +1853,7 @@ class ASTReader
18531853
///
18541854
/// \returns true if an error occurred, false otherwise.
18551855
static bool readASTFileControlBlock(
1856-
StringRef Filename, FileManager &FileMgr,
1857-
const InMemoryModuleCache &ModuleCache,
1856+
StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
18581857
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
18591858
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
18601859
unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch |
@@ -1863,7 +1862,7 @@ class ASTReader
18631862
/// Determine whether the given AST file is acceptable to load into a
18641863
/// translation unit with the given language and target options.
18651864
static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
1866-
const InMemoryModuleCache &ModuleCache,
1865+
const ModuleCache &ModCache,
18671866
const PCHContainerReader &PCHContainerRdr,
18681867
const LangOptions &LangOpts,
18691868
const TargetOptions &TargetOpts,

‎clang/include/clang/Serialization/ASTWriter.h

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class LangOptions;
6060
class MacroDefinitionRecord;
6161
class MacroInfo;
6262
class Module;
63-
class InMemoryModuleCache;
63+
class ModuleCache;
6464
class ModuleFileExtension;
6565
class ModuleFileExtensionWriter;
6666
class NamedDecl;
@@ -117,7 +117,7 @@ class ASTWriter : public ASTDeserializationListener,
117117
const SmallVectorImpl<char> &Buffer;
118118

119119
/// The PCM manager which manages memory buffers for pcm files.
120-
InMemoryModuleCache &ModuleCache;
120+
ModuleCache &ModCache;
121121

122122
/// The preprocessor we're writing.
123123
Preprocessor *PP = nullptr;
@@ -638,7 +638,7 @@ class ASTWriter : public ASTDeserializationListener,
638638
/// Create a new precompiled header writer that outputs to
639639
/// the given bitstream.
640640
ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
641-
InMemoryModuleCache &ModuleCache,
641+
ModuleCache &ModCache,
642642
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
643643
bool IncludeTimestamps = true, bool BuildingImplicitModule = false,
644644
bool GeneratingReducedBMI = false);
@@ -948,9 +948,8 @@ class PCHGenerator : public SemaConsumer {
948948
virtual Module *getEmittingModule(ASTContext &Ctx);
949949

950950
public:
951-
PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
952-
StringRef OutputFile, StringRef isysroot,
953-
std::shared_ptr<PCHBuffer> Buffer,
951+
PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile,
952+
StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
954953
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
955954
bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
956955
bool BuildingImplicitModule = false,
@@ -972,13 +971,13 @@ class CXX20ModulesGenerator : public PCHGenerator {
972971
protected:
973972
virtual Module *getEmittingModule(ASTContext &Ctx) override;
974973

975-
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
974+
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
976975
StringRef OutputFile, bool GeneratingReducedBMI);
977976

978977
public:
979-
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
978+
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
980979
StringRef OutputFile)
981-
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
980+
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
982981
/*GeneratingReducedBMI=*/false) {}
983982

984983
void HandleTranslationUnit(ASTContext &Ctx) override;
@@ -988,9 +987,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator {
988987
void anchor() override;
989988

990989
public:
991-
ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
990+
ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache,
992991
StringRef OutputFile)
993-
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
992+
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
994993
/*GeneratingReducedBMI=*/true) {}
995994
};
996995

‎clang/include/clang/Serialization/ModuleCache.h

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H
10+
#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H
11+
12+
#include "clang/Basic/LLVM.h"
13+
#include "llvm/ADT/IntrusiveRefCntPtr.h"
14+
15+
namespace llvm {
16+
class AdvisoryLock;
17+
} // namespace llvm
18+
19+
namespace clang {
20+
class InMemoryModuleCache;
21+
22+
/// The module cache used for compiling modules implicitly. This centralizes the
23+
/// operations the compiler might want to perform on the cache.
24+
class ModuleCache : public RefCountedBase<ModuleCache> {
25+
public:
26+
/// May perform any work that only needs to be performed once for multiple
27+
/// calls \c getLock() with the same module filename.
28+
virtual void prepareForGetLock(StringRef ModuleFilename) = 0;
29+
30+
/// Returns lock for the given module file. The lock is initially unlocked.
31+
virtual std::unique_ptr<llvm::AdvisoryLock>
32+
getLock(StringRef ModuleFilename) = 0;
33+
34+
/// Returns this process's view of the module cache.
35+
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
36+
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
37+
38+
// TODO: Virtualize writing/reading PCM files, timestamping, pruning, etc.
39+
40+
virtual ~ModuleCache() = default;
41+
};
42+
43+
/// Creates new \c ModuleCache backed by a file system directory that may be
44+
/// operated on by multiple processes. This instance must be used across all
45+
/// \c CompilerInstance instances participating in building modules for single
46+
/// translation unit in order to share the same \c InMemoryModuleCache.
47+
IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
48+
} // namespace clang
49+
50+
#endif

‎clang/include/clang/Serialization/ModuleManager.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class FileEntry;
3737
class FileManager;
3838
class GlobalModuleIndex;
3939
class HeaderSearch;
40-
class InMemoryModuleCache;
40+
class ModuleCache;
4141
class PCHContainerReader;
4242

4343
namespace serialization {
@@ -65,7 +65,7 @@ class ModuleManager {
6565
FileManager &FileMgr;
6666

6767
/// Cache of PCM files.
68-
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
68+
IntrusiveRefCntPtr<ModuleCache> ModCache;
6969

7070
/// Knows how to unwrap module containers.
7171
const PCHContainerReader &PCHContainerRdr;
@@ -133,9 +133,9 @@ class ModuleManager {
133133
SmallVectorImpl<std::unique_ptr<ModuleFile>>::reverse_iterator>;
134134
using ModuleOffset = std::pair<uint32_t, StringRef>;
135135

136-
explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache,
137-
const PCHContainerReader &PCHContainerRdr,
138-
const HeaderSearch &HeaderSearchInfo);
136+
ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
137+
const PCHContainerReader &PCHContainerRdr,
138+
const HeaderSearch &HeaderSearchInfo);
139139

140140
/// Forward iterator to traverse all loaded modules.
141141
ModuleIterator begin() { return Chain.begin(); }
@@ -306,7 +306,7 @@ class ModuleManager {
306306
/// View the graphviz representation of the module graph.
307307
void viewGraph();
308308

309-
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
309+
ModuleCache &getModuleCache() const { return *ModCache; }
310310
};
311311

312312
} // namespace serialization

‎clang/lib/Frontend/ASTUnit.cpp

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
#include "clang/Serialization/ASTReader.h"
6262
#include "clang/Serialization/ASTWriter.h"
6363
#include "clang/Serialization/ContinuousRangeMap.h"
64-
#include "clang/Serialization/InMemoryModuleCache.h"
64+
#include "clang/Serialization/ModuleCache.h"
6565
#include "clang/Serialization/ModuleFile.h"
6666
#include "clang/Serialization/PCHContainerOperations.h"
6767
#include "llvm/ADT/ArrayRef.h"
@@ -219,8 +219,8 @@ struct ASTUnit::ASTWriterData {
219219
llvm::BitstreamWriter Stream;
220220
ASTWriter Writer;
221221

222-
ASTWriterData(InMemoryModuleCache &ModuleCache)
223-
: Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {}
222+
ASTWriterData(ModuleCache &ModCache)
223+
: Stream(Buffer), Writer(Stream, Buffer, ModCache, {}) {}
224224
};
225225

226226
void ASTUnit::clearFileLevelDecls() {
@@ -825,7 +825,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
825825
AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
826826
AST->getFileManager(),
827827
UserFilesAreVolatile);
828-
AST->ModuleCache = new InMemoryModuleCache;
828+
AST->ModCache = createCrossProcessModuleCache();
829829
AST->HSOpts = HSOpts ? HSOpts : std::make_shared<HeaderSearchOptions>();
830830
AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front());
831831
AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
@@ -857,8 +857,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
857857
if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
858858
disableValid = DisableValidationForModuleKind::All;
859859
AST->Reader = new ASTReader(
860-
PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {},
861-
/*isysroot=*/"",
860+
PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, {}, /*isysroot=*/"",
862861
/*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors);
863862

864863
unsigned Counter = 0;
@@ -1542,7 +1541,7 @@ ASTUnit::create(std::shared_ptr<CompilerInvocation> CI,
15421541
AST->UserFilesAreVolatile = UserFilesAreVolatile;
15431542
AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
15441543
UserFilesAreVolatile);
1545-
AST->ModuleCache = new InMemoryModuleCache;
1544+
AST->ModCache = createCrossProcessModuleCache();
15461545

15471546
return AST;
15481547
}
@@ -1829,7 +1828,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
18291828
AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
18301829
AST->StorePreamblesInMemory = StorePreamblesInMemory;
18311830
AST->PreambleStoragePath = PreambleStoragePath;
1832-
AST->ModuleCache = new InMemoryModuleCache;
1831+
AST->ModCache = createCrossProcessModuleCache();
18331832
AST->OnlyLocalDecls = OnlyLocalDecls;
18341833
AST->CaptureDiagnostics = CaptureDiagnostics;
18351834
AST->TUKind = TUKind;
@@ -1840,7 +1839,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
18401839
AST->Invocation = CI;
18411840
AST->SkipFunctionBodies = SkipFunctionBodies;
18421841
if (ForSerialization)
1843-
AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache));
1842+
AST->WriterData.reset(new ASTWriterData(*AST->ModCache));
18441843
// Zero out now to ease cleanup during crash recovery.
18451844
CI = nullptr;
18461845
Diags = nullptr;
@@ -2373,8 +2372,8 @@ bool ASTUnit::serialize(raw_ostream &OS) {
23732372

23742373
SmallString<128> Buffer;
23752374
llvm::BitstreamWriter Stream(Buffer);
2376-
InMemoryModuleCache ModuleCache;
2377-
ASTWriter Writer(Stream, Buffer, ModuleCache, {});
2375+
IntrusiveRefCntPtr<ModuleCache> ModCache = createCrossProcessModuleCache();
2376+
ASTWriter Writer(Stream, Buffer, *ModCache, {});
23782377
return serializeUnit(Writer, Buffer, getSema(), OS);
23792378
}
23802379

0 commit comments

Comments
 (0)