Skip to content

Commit 47b29b6

Browse files
xymusxymus
authored
Merge pull request #61649 from xymus/index-swiftinterfaces
[Index] Force indexing of system modules to read only from swiftinterfaces
2 parents 4ce1ebb + 5d59a8f commit 47b29b6

28 files changed

+268
-50
lines changed

‎include/swift/AST/ASTContext.h

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,10 @@ class ASTContext final {
309309
/// The name of the SwiftShims module "SwiftShims".
310310
Identifier SwiftShimsModuleName;
311311

312+
/// Should we globally ignore swiftmodule files adjacent to swiftinterface
313+
/// files?
314+
bool IgnoreAdjacentModules = false;
315+
312316
// Define the set of known identifiers.
313317
#define IDENTIFIER_WITH_NAME(Name, IdStr) Identifier Id_##Name;
314318
#include "swift/AST/KnownIdentifiers.def"
@@ -1144,9 +1148,12 @@ class ASTContext final {
11441148
///
11451149
/// \param ModulePath The module's \c ImportPath which describes
11461150
/// the name of the module being loaded, possibly including submodules.
1147-
1151+
/// \param AllowMemoryCached Should we allow reuse of an already loaded
1152+
/// module or force reloading from disk, defaults to true.
1153+
///
11481154
/// \returns The requested module, or NULL if the module cannot be found.
1149-
ModuleDecl *getModule(ImportPath::Module ModulePath);
1155+
ModuleDecl *
1156+
getModule(ImportPath::Module ModulePath, bool AllowMemoryCached = true);
11501157

11511158
/// Attempts to load the matching overlay module for the given clang
11521159
/// module into this ASTContext.
@@ -1173,7 +1180,10 @@ class ASTContext final {
11731180
/// in this context.
11741181
void addLoadedModule(ModuleDecl *M);
11751182

1176-
public:
1183+
/// Change the behavior of all loaders to ignore swiftmodules next to
1184+
/// swiftinterfaces.
1185+
void setIgnoreAdjacentModules(bool value);
1186+
11771187
/// Retrieve the current generation number, which reflects the
11781188
/// number of times a module import has caused mass invalidation of
11791189
/// lookup tables.

‎include/swift/AST/DiagnosticsFrontend.def

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,10 @@ ERROR(error_index_failed_status_check,none,
255255
"failed file status check: %0", (StringRef))
256256
ERROR(error_index_inputs_more_than_outputs,none,
257257
"index output filenames do not match input source files", ())
258+
REMARK(remark_indexing_system_module,none,
259+
"indexing system module at %0"
260+
"%select{|; skipping because of a broken swiftinterface}1",
261+
(StringRef, bool))
258262

259263
ERROR(error_wrong_number_of_arguments,none,
260264
"wrong number of '%0' arguments (expected %1, got %2)",

‎include/swift/AST/Module.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -545,7 +545,7 @@ class ModuleDecl
545545
}
546546

547547
/// Returns true if the module was rebuilt from a module interface instead
548-
/// of being build from the full source.
548+
/// of being built from the full source.
549549
bool isBuiltFromInterface() const {
550550
return Bits.ModuleDecl.IsBuiltFromInterface;
551551
}

‎include/swift/AST/ModuleLoader.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,10 +249,14 @@ class ModuleLoader {
249249
/// \param path A sequence of (identifier, location) pairs that denote
250250
/// the dotted module name to load, e.g., AppKit.NSWindow.
251251
///
252+
/// \param AllowMemoryCache Enables preserving the loaded module in the
253+
/// in-memory cache for the next loading attempt.
254+
///
252255
/// \returns the module referenced, if it could be loaded. Otherwise,
253256
/// emits a diagnostic and returns NULL.
254257
virtual
255-
ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path) = 0;
258+
ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path,
259+
bool AllowMemoryCache = true) = 0;
256260

257261
/// Load extensions to the given nominal type.
258262
///

‎include/swift/Basic/LangOptions.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,9 @@ namespace swift {
216216

217217
/// Emit a remark after loading a module.
218218
bool EnableModuleLoadingRemarks = false;
219+
220+
/// Emit a remark when indexing a system module.
221+
bool EnableIndexingSystemModuleRemarks = false;
219222

220223
/// Emit a remark on early exit in explicit interface build
221224
bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,11 +222,15 @@ class ClangImporter final : public ClangModuleLoader {
222222
/// \param path A sequence of (identifier, location) pairs that denote
223223
/// the dotted module name to load, e.g., AppKit.NSWindow.
224224
///
225+
/// \param AllowMemoryCache Affects only loading serialized Swift modules,
226+
/// this parameter has no effect in the ClangImporter.
227+
///
225228
/// \returns the module referenced, if it could be loaded. Otherwise,
226229
/// emits a diagnostic and returns NULL.
227230
virtual ModuleDecl *loadModule(
228231
SourceLoc importLoc,
229-
ImportPath::Module path)
232+
ImportPath::Module path,
233+
bool AllowMemoryCache = true)
230234
override;
231235

232236
/// Determine whether \c overlayDC is within an overlay module for the

‎include/swift/Frontend/ModuleInterfaceLoader.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -439,7 +439,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
439439
StringRef OutPath, StringRef ABIOutputPath,
440440
bool SerializeDependencyHashes,
441441
bool TrackSystemDependencies, ModuleInterfaceLoaderOptions Opts,
442-
RequireOSSAModules_t RequireOSSAModules);
442+
RequireOSSAModules_t RequireOSSAModules,
443+
bool silenceInterfaceDiagnostics);
443444

444445
/// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
445446
/// a swiftmodule file).

‎include/swift/Option/Options.td

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,10 @@ def remark_loading_module : Flag<["-"], "Rmodule-loading">,
343343
Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
344344
HelpText<"Emit a remark and file path of each loaded module">;
345345

346+
def remark_indexing_system_module : Flag<["-"], "Rindexing-system-module">,
347+
Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
348+
HelpText<"Emit a remark when indexing a system module">;
349+
346350
def remark_skip_explicit_interface_build : Flag<["-"], "Rskip-explicit-interface-build">,
347351
Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
348352
HelpText<"Emit a remark if an explicit module interface invocation has an early exit because the expected output is up-to-date">;

‎include/swift/Sema/SourceLoader.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ class SourceLoader : public ModuleLoader {
7373
/// returns NULL.
7474
virtual ModuleDecl *
7575
loadModule(SourceLoc importLoc,
76-
ImportPath::Module path) override;
76+
ImportPath::Module path,
77+
bool AllowMemoryCache) override;
7778

7879
/// Load extensions to the given nominal type.
7980
///

‎include/swift/Serialization/SerializedModuleLoader.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
185185
/// emits a diagnostic and returns a FailedImportModule object.
186186
virtual ModuleDecl *
187187
loadModule(SourceLoc importLoc,
188-
ImportPath::Module path) override;
188+
ImportPath::Module path,
189+
bool AllowMemoryCache) override;
189190

190191

191192
virtual void loadExtensions(NominalTypeDecl *nominal,
@@ -294,7 +295,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
294295

295296
ModuleDecl *
296297
loadModule(SourceLoc importLoc,
297-
ImportPath::Module path) override;
298+
ImportPath::Module path,
299+
bool AllowMemoryCache = true) override;
298300

299301
/// Register a memory buffer that contains the serialized module for the given
300302
/// access path. This API is intended to be used by LLDB to add swiftmodules

‎lib/AST/ASTContext.cpp

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2201,6 +2201,10 @@ void ASTContext::addLoadedModule(ModuleDecl *M) {
22012201
getImpl().LoadedModules[M->getRealName()] = M;
22022202
}
22032203

2204+
void ASTContext::setIgnoreAdjacentModules(bool value) {
2205+
IgnoreAdjacentModules = value;
2206+
}
2207+
22042208
rewriting::RewriteContext &
22052209
ASTContext::getRewriteContext() {
22062210
auto &rewriteCtx = getImpl().TheRewriteContext;
@@ -2451,17 +2455,19 @@ bool ASTContext::canImportModule(ImportPath::Module ModuleName,
24512455
}
24522456

24532457
ModuleDecl *
2454-
ASTContext::getModule(ImportPath::Module ModulePath) {
2458+
ASTContext::getModule(ImportPath::Module ModulePath, bool AllowMemoryCached) {
24552459
assert(!ModulePath.empty());
24562460

2457-
if (auto *M = getLoadedModule(ModulePath))
2458-
return M;
2461+
if (AllowMemoryCached)
2462+
if (auto *M = getLoadedModule(ModulePath))
2463+
return M;
24592464

24602465
auto moduleID = ModulePath[0];
24612466
if (PreModuleImportCallback)
24622467
PreModuleImportCallback(moduleID.Item.str(), false /*=IsOverlay*/);
24632468
for (auto &importer : getImpl().ModuleLoaders) {
2464-
if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath)) {
2469+
if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath,
2470+
AllowMemoryCached)) {
24652471
if (LangOpts.EnableModuleLoadingRemarks) {
24662472
Diags.diagnose(ModulePath.getSourceRange().Start,
24672473
diag::module_loaded,

‎lib/ClangImporter/ClangImporter.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2023,7 +2023,8 @@ ModuleDecl *ClangImporter::Implementation::loadModuleClang(
20232023

20242024
ModuleDecl *
20252025
ClangImporter::loadModule(SourceLoc importLoc,
2026-
ImportPath::Module path) {
2026+
ImportPath::Module path,
2027+
bool AllowMemoryCache) {
20272028
return Impl.loadModule(importLoc, path);
20282029
}
20292030

‎lib/Frontend/CompilerInvocation.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,8 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
822822

823823
Opts.EnableModuleLoadingRemarks = Args.hasArg(OPT_remark_loading_module);
824824

825+
Opts.EnableIndexingSystemModuleRemarks = Args.hasArg(OPT_remark_indexing_system_module);
826+
825827
Opts.EnableSkipExplicitInterfaceModuleBuildRemarks = Args.hasArg(OPT_remark_skip_explicit_interface_build);
826828

827829
llvm::Triple Target = Opts.Target;

‎lib/Frontend/ModuleInterfaceBuilder.cpp

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,11 +320,19 @@ bool ImplicitModuleInterfaceBuilder::buildSwiftModuleInternal(
320320
llvm::RestorePrettyStackState(savedInnerPrettyStackState);
321321
};
322322

323+
Optional<DiagnosticEngine> localDiags;
324+
DiagnosticEngine *rebuildDiags = diags;
325+
if (silenceInterfaceDiagnostics) {
326+
// To silence diagnostics, use a local temporary engine.
327+
localDiags.emplace(sourceMgr);
328+
rebuildDiags = &*localDiags;
329+
}
330+
323331
SubError = (bool)subASTDelegate.runInSubCompilerInstance(
324332
moduleName, interfacePath, OutPath, diagnosticLoc,
325333
[&](SubCompilerInstanceInfo &info) {
326334
auto EBuilder = ExplicitModuleInterfaceBuilder(
327-
*info.Instance, diags, sourceMgr, moduleCachePath, backupInterfaceDir,
335+
*info.Instance, rebuildDiags, sourceMgr, moduleCachePath, backupInterfaceDir,
328336
prebuiltCachePath, ABIDescriptorPath, extraDependencies, diagnosticLoc,
329337
dependencyTracker);
330338
return EBuilder.buildSwiftModuleFromInterface(

‎lib/Frontend/ModuleInterfaceBuilder.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ class ImplicitModuleInterfaceBuilder {
4747
const StringRef backupInterfaceDir;
4848
const StringRef ABIDescriptorPath;
4949
const bool disableInterfaceFileLock;
50+
const bool silenceInterfaceDiagnostics;
5051
const SourceLoc diagnosticLoc;
5152
DependencyTracker *const dependencyTracker;
5253
SmallVector<StringRef, 3> extraDependencies;
@@ -87,6 +88,7 @@ class ImplicitModuleInterfaceBuilder {
8788
StringRef moduleName, StringRef moduleCachePath,
8889
StringRef backupInterfaceDir, StringRef prebuiltCachePath,
8990
StringRef ABIDescriptorPath, bool disableInterfaceFileLock = false,
91+
bool silenceInterfaceDiagnostics = false,
9092
SourceLoc diagnosticLoc = SourceLoc(),
9193
DependencyTracker *tracker = nullptr)
9294
: sourceMgr(sourceMgr), diags(diags), subASTDelegate(subASTDelegate),
@@ -95,6 +97,7 @@ class ImplicitModuleInterfaceBuilder {
9597
backupInterfaceDir(backupInterfaceDir),
9698
ABIDescriptorPath(ABIDescriptorPath),
9799
disableInterfaceFileLock(disableInterfaceFileLock),
100+
silenceInterfaceDiagnostics(silenceInterfaceDiagnostics),
98101
diagnosticLoc(diagnosticLoc), dependencyTracker(tracker) {}
99102

100103
/// Ensures the requested file name is added as a dependency of the resulting

‎lib/Frontend/ModuleInterfaceLoader.cpp

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ class ModuleInterfaceLoaderImpl {
677677
std::pair<std::string, std::string> getCompiledModuleCandidates() {
678678
std::pair<std::string, std::string> result;
679679
// Should we attempt to load a swiftmodule adjacent to the swiftinterface?
680-
bool shouldLoadAdjacentModule = true;
680+
bool shouldLoadAdjacentModule = !ctx.IgnoreAdjacentModules;
681681

682682
// Don't use the adjacent swiftmodule for frameworks from the public
683683
// Frameworks folder of the SDK.
@@ -1031,7 +1031,8 @@ class ModuleInterfaceLoaderImpl {
10311031
ctx.SourceMgr, diagsToUse,
10321032
astDelegate, interfacePath, moduleName, cacheDir,
10331033
prebuiltCacheDir, backupInterfaceDir, StringRef(),
1034-
Opts.disableInterfaceLock, diagnosticLoc,
1034+
Opts.disableInterfaceLock,
1035+
ctx.IgnoreAdjacentModules, diagnosticLoc,
10351036
dependencyTracker);
10361037
// If we found an out-of-date .swiftmodule, we still want to add it as
10371038
// a dependency of the .swiftinterface. That way if it's updated, but
@@ -1063,7 +1064,8 @@ class ModuleInterfaceLoaderImpl {
10631064
ImplicitModuleInterfaceBuilder fallbackBuilder(
10641065
ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath, moduleName, cacheDir,
10651066
prebuiltCacheDir, backupInterfaceDir, StringRef(),
1066-
Opts.disableInterfaceLock, diagnosticLoc,
1067+
Opts.disableInterfaceLock,
1068+
ctx.IgnoreAdjacentModules, diagnosticLoc,
10671069
dependencyTracker);
10681070
if (rebuildInfo.sawOutOfDateModule(modulePath))
10691071
fallbackBuilder.addExtraDependency(modulePath);
@@ -1247,7 +1249,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
12471249
StringRef OutPath, StringRef ABIOutputPath,
12481250
bool SerializeDependencyHashes,
12491251
bool TrackSystemDependencies, ModuleInterfaceLoaderOptions LoaderOpts,
1250-
RequireOSSAModules_t RequireOSSAModules) {
1252+
RequireOSSAModules_t RequireOSSAModules,
1253+
bool silenceInterfaceDiagnostics) {
12511254
InterfaceSubContextDelegateImpl astDelegate(
12521255
SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, LoaderOpts,
12531256
/*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir,
@@ -1256,7 +1259,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
12561259
ImplicitModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath,
12571260
ModuleName, CacheDir, PrebuiltCacheDir,
12581261
BackupInterfaceDir, ABIOutputPath,
1259-
LoaderOpts.disableInterfaceLock);
1262+
LoaderOpts.disableInterfaceLock,
1263+
silenceInterfaceDiagnostics);
12601264
// FIXME: We really only want to serialize 'important' dependencies here, if
12611265
// we want to ship the built swiftmodules to another machine.
12621266
auto failed = builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
@@ -1274,7 +1278,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
12741278
ImplicitModuleInterfaceBuilder backupBuilder(SourceMgr, &Diags, astDelegate, backInPath,
12751279
ModuleName, CacheDir, PrebuiltCacheDir,
12761280
BackupInterfaceDir, ABIOutputPath,
1277-
LoaderOpts.disableInterfaceLock);
1281+
LoaderOpts.disableInterfaceLock,
1282+
silenceInterfaceDiagnostics);
12781283
// Ensure we can rebuild module after user changed the original interface file.
12791284
backupBuilder.addExtraDependency(InPath);
12801285
// FIXME: We really only want to serialize 'important' dependencies here, if

‎lib/FrontendTool/FrontendTool.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
422422
ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
423423
StringRef ABIPath = Instance.getPrimarySpecificPathsForAtMostOnePrimary()
424424
.SupplementaryOutputs.ABIDescriptorOutputPath;
425+
bool IgnoreAdjacentModules = Instance.hasASTContext() &&
426+
Instance.getASTContext().IgnoreAdjacentModules;
425427

426428
// If an explicit interface build was requested, bypass the creation of a new
427429
// sub-instance from the interface which will build it in a separate thread,
@@ -443,7 +445,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
443445
Invocation.getOutputFilename(), ABIPath,
444446
FEOpts.SerializeModuleInterfaceDependencyHashes,
445447
FEOpts.shouldTrackSystemDependencies(), LoaderOpts,
446-
RequireOSSAModules_t(Invocation.getSILOptions()));
448+
RequireOSSAModules_t(Invocation.getSILOptions()),
449+
IgnoreAdjacentModules);
447450
}
448451

449452
static bool compileLLVMIR(CompilerInstance &Instance) {

‎lib/Index/IndexRecord.cpp

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,14 +513,40 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
513513
if (*isUptodateOpt)
514514
return false;
515515

516-
// FIXME: Would be useful for testing if swift had clang's -Rremark system so
517-
// we could output a remark here that we are going to create index data for
518-
// a module file.
516+
// Reload resilient modules from swiftinterface to avoid indexing
517+
// internal details.
518+
bool skipIndexingModule = false;
519+
if (module->getResilienceStrategy() == ResilienceStrategy::Resilient &&
520+
!module->isBuiltFromInterface() &&
521+
!module->isStdlibModule()) {
522+
module->getASTContext().setIgnoreAdjacentModules(true);
523+
524+
ImportPath::Module::Builder builder(module->getName());
525+
ASTContext &ctx = module->getASTContext();
526+
auto reloadedModule = ctx.getModule(builder.get(),
527+
/*AllowMemoryCached=*/false);
528+
529+
if (reloadedModule) {
530+
module = reloadedModule;
531+
} else {
532+
// If we can't rebuild from the swiftinterface, don't index this module.
533+
skipIndexingModule = true;
534+
}
535+
}
536+
537+
if (module->getASTContext().LangOpts.EnableIndexingSystemModuleRemarks) {
538+
diags.diagnose(SourceLoc(),
539+
diag::remark_indexing_system_module,
540+
filename, skipIndexingModule);
541+
}
519542

520543
// Pairs of (recordFile, groupName).
521544
std::vector<std::pair<std::string, std::string>> records;
522545

523-
if (!module->isStdlibModule()) {
546+
if (skipIndexingModule) {
547+
// Don't add anything to records but keep going so we still mark the module
548+
// as indexed to avoid rebuilds of broken swiftinterfaces.
549+
} else if (!module->isStdlibModule()) {
524550
std::string recordFile;
525551
bool failed = false;
526552
auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(),

‎lib/Sema/SourceLoader.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ bool SourceLoader::canImportModule(ImportPath::Module path,
9494
}
9595

9696
ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc,
97-
ImportPath::Module path) {
97+
ImportPath::Module path,
98+
bool AllowMemoryCache) {
9899
// FIXME: Swift submodules?
99100
if (path.size() > 1)
100101
return nullptr;

0 commit comments

Comments
 (0)