[Index] Force indexing of system modules to read only from swiftinterfaces

include/swift/AST/ASTContext.h

@@ -307,6 +307,10 @@ class ASTContext final {
   /// The name of the SwiftShims module "SwiftShims".
   Identifier SwiftShimsModuleName;
 
+  /// Should we globally ignore swiftmodule files adjacent to swiftinterface
+  /// files?
+  bool IgnoreAdjacentModules = false;
+
   // Define the set of known identifiers.
 #define IDENTIFIER_WITH_NAME(Name, IdStr) Identifier Id_##Name;
 #include "swift/AST/KnownIdentifiers.def"
@@ -1136,9 +1140,12 @@ class ASTContext final {
   ///
   /// \param ModulePath The module's \c ImportPath which describes
   /// the name of the module being loaded, possibly including submodules.
-
+  /// \param AllowMemoryCached Should we allow reuse of an already loaded
+  /// module or force reloading from disk, defaults to true.
+  ///
   /// \returns The requested module, or NULL if the module cannot be found.
-  ModuleDecl *getModule(ImportPath::Module ModulePath);
+  ModuleDecl *
+  getModule(ImportPath::Module ModulePath, bool AllowMemoryCached = true);
 
   /// Attempts to load the matching overlay module for the given clang
   /// module into this ASTContext.
@@ -1165,7 +1172,10 @@ class ASTContext final {
   /// in this context.
   void addLoadedModule(ModuleDecl *M);
 
-public:
+  /// Change the behavior of all loaders to ignore swiftmodules next to
+  /// swiftinterfaces.
+  void setIgnoreAdjacentModules(bool value);
+
   /// Retrieve the current generation number, which reflects the
   /// number of times a module import has caused mass invalidation of
   /// lookup tables.

include/swift/AST/DiagnosticsFrontend.def

@@ -255,6 +255,10 @@ ERROR(error_index_failed_status_check,none,
       "failed file status check: %0", (StringRef))
 ERROR(error_index_inputs_more_than_outputs,none,
       "index output filenames do not match input source files", ())
+REMARK(remark_indexing_system_module,none,
+       "indexing system module at %0"
+       "%select{|; skipping because of a broken swiftinterface}1",
+       (StringRef, bool))
 
 ERROR(error_wrong_number_of_arguments,none,
       "wrong number of '%0' arguments (expected %1, got %2)",

include/swift/AST/Module.h

@@ -545,7 +545,7 @@ class ModuleDecl
   }
 
   /// Returns true if the module was rebuilt from a module interface instead
-  /// of being build from the full source.
+  /// of being built from the full source.
   bool isBuiltFromInterface() const {
     return Bits.ModuleDecl.IsBuiltFromInterface;
   }

include/swift/AST/ModuleLoader.h

@@ -249,10 +249,14 @@ class ModuleLoader {
   /// \param path A sequence of (identifier, location) pairs that denote
   /// the dotted module name to load, e.g., AppKit.NSWindow.
   ///
+  /// \param AllowMemoryCache Enables preserving the loaded module in the
+  /// in-memory cache for the next loading attempt.
+  ///
   /// \returns the module referenced, if it could be loaded. Otherwise,
   /// emits a diagnostic and returns NULL.
   virtual
-  ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path) = 0;
+  ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path,
+                         bool AllowMemoryCache = true) = 0;
 
   /// Load extensions to the given nominal type.
   ///

include/swift/Basic/LangOptions.h

@@ -216,6 +216,9 @@ namespace swift {
 
     /// Emit a remark after loading a module.
     bool EnableModuleLoadingRemarks = false;
+
+    /// Emit a remark when indexing a system module.
+    bool EnableIndexingSystemModuleRemarks = false;
 
     /// Emit a remark on early exit in explicit interface build
     bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;

include/swift/ClangImporter/ClangImporter.h

@@ -220,11 +220,15 @@ class ClangImporter final : public ClangModuleLoader {
   /// \param path A sequence of (identifier, location) pairs that denote
   /// the dotted module name to load, e.g., AppKit.NSWindow.
   ///
+  /// \param AllowMemoryCache Affects only loading serialized Swift modules,
+  /// this parameter has no effect in the ClangImporter.
+  ///
   /// \returns the module referenced, if it could be loaded. Otherwise,
   /// emits a diagnostic and returns NULL.
   virtual ModuleDecl *loadModule(
                         SourceLoc importLoc,
-                        ImportPath::Module path)
+                        ImportPath::Module path,
+                        bool AllowMemoryCache = true)
                       override;
 
   /// Determine whether \c overlayDC is within an overlay module for the

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -439,7 +439,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
       StringRef OutPath, StringRef ABIOutputPath,
       bool SerializeDependencyHashes,
       bool TrackSystemDependencies, ModuleInterfaceLoaderOptions Opts,
-      RequireOSSAModules_t RequireOSSAModules);
+      RequireOSSAModules_t RequireOSSAModules,
+      bool silenceInterfaceDiagnostics);
 
   /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
   /// a swiftmodule file).

include/swift/Option/Options.td

@@ -343,6 +343,10 @@ def remark_loading_module : Flag<["-"], "Rmodule-loading">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   HelpText<"Emit a remark and file path of each loaded module">;
 
+def remark_indexing_system_module : Flag<["-"], "Rindexing-system-module">,
+  Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
+  HelpText<"Emit a remark when indexing a system module">;
+
 def remark_skip_explicit_interface_build : Flag<["-"], "Rskip-explicit-interface-build">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   HelpText<"Emit a remark if an explicit module interface invocation has an early exit because the expected output is up-to-date">;

include/swift/Sema/SourceLoader.h

@@ -73,7 +73,8 @@ class SourceLoader : public ModuleLoader {
   /// returns NULL.
   virtual ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache) override;
 
   /// Load extensions to the given nominal type.
   ///

include/swift/Serialization/SerializedModuleLoader.h

@@ -185,7 +185,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// emits a diagnostic and returns a FailedImportModule object.
   virtual ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache) override;
 
 
   virtual void loadExtensions(NominalTypeDecl *nominal,
@@ -294,7 +295,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
 
   ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache = true) override;
 
   /// Register a memory buffer that contains the serialized module for the given
   /// access path. This API is intended to be used by LLDB to add swiftmodules

lib/AST/ASTContext.cpp

@@ -2191,6 +2191,10 @@ void ASTContext::addLoadedModule(ModuleDecl *M) {
   getImpl().LoadedModules[M->getRealName()] = M;
 }
 
+void ASTContext::setIgnoreAdjacentModules(bool value) {
+  IgnoreAdjacentModules = value;
+}
+
 rewriting::RewriteContext &
 ASTContext::getRewriteContext() {
   auto &rewriteCtx = getImpl().TheRewriteContext;
@@ -2441,17 +2445,19 @@ bool ASTContext::canImportModule(ImportPath::Module ModuleName,
 }
 
 ModuleDecl *
-ASTContext::getModule(ImportPath::Module ModulePath) {
+ASTContext::getModule(ImportPath::Module ModulePath, bool AllowMemoryCached) {
   assert(!ModulePath.empty());
 
-  if (auto *M = getLoadedModule(ModulePath))
-    return M;
+  if (AllowMemoryCached)
+    if (auto *M = getLoadedModule(ModulePath))
+      return M;
 
   auto moduleID = ModulePath[0];
   if (PreModuleImportCallback)
     PreModuleImportCallback(moduleID.Item.str(), false /*=IsOverlay*/);
   for (auto &importer : getImpl().ModuleLoaders) {
-    if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath)) {
+    if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath,
+                                             AllowMemoryCached)) {
       if (LangOpts.EnableModuleLoadingRemarks) {
         Diags.diagnose(ModulePath.getSourceRange().Start,
                        diag::module_loaded,

lib/ClangImporter/ClangImporter.cpp

@@ -2018,7 +2018,8 @@ ModuleDecl *ClangImporter::Implementation::loadModuleClang(
 
 ModuleDecl *
 ClangImporter::loadModule(SourceLoc importLoc,
-                          ImportPath::Module path) {
+                          ImportPath::Module path,
+                          bool AllowMemoryCache) {
   return Impl.loadModule(importLoc, path);
 }
 

lib/Frontend/CompilerInvocation.cpp

@@ -822,6 +822,8 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
   Opts.EnableModuleLoadingRemarks = Args.hasArg(OPT_remark_loading_module);
 
+  Opts.EnableIndexingSystemModuleRemarks = Args.hasArg(OPT_remark_indexing_system_module);
+
   Opts.EnableSkipExplicitInterfaceModuleBuildRemarks = Args.hasArg(OPT_remark_skip_explicit_interface_build);
 
   llvm::Triple Target = Opts.Target;

lib/Frontend/ModuleInterfaceBuilder.cpp

@@ -320,11 +320,19 @@ bool ImplicitModuleInterfaceBuilder::buildSwiftModuleInternal(
       llvm::RestorePrettyStackState(savedInnerPrettyStackState);
     };
 
+    Optional<DiagnosticEngine> localDiags;
+    DiagnosticEngine *rebuildDiags = diags;
+    if (silenceInterfaceDiagnostics) {
+      // To silence diagnostics, use a local temporary engine.
+      localDiags.emplace(sourceMgr);
+      rebuildDiags = &*localDiags;
+    }
+
     SubError = (bool)subASTDelegate.runInSubCompilerInstance(
         moduleName, interfacePath, OutPath, diagnosticLoc,
         [&](SubCompilerInstanceInfo &info) {
           auto EBuilder = ExplicitModuleInterfaceBuilder(
-              *info.Instance, diags, sourceMgr, moduleCachePath, backupInterfaceDir,
+              *info.Instance, rebuildDiags, sourceMgr, moduleCachePath, backupInterfaceDir,
               prebuiltCachePath, ABIDescriptorPath, extraDependencies, diagnosticLoc,
               dependencyTracker);
           return EBuilder.buildSwiftModuleFromInterface(

lib/Frontend/ModuleInterfaceBuilder.h

@@ -47,6 +47,7 @@ class ImplicitModuleInterfaceBuilder {
   const StringRef backupInterfaceDir;
   const StringRef ABIDescriptorPath;
   const bool disableInterfaceFileLock;
+  const bool silenceInterfaceDiagnostics;
   const SourceLoc diagnosticLoc;
   DependencyTracker *const dependencyTracker;
   SmallVector<StringRef, 3> extraDependencies;
@@ -87,6 +88,7 @@ class ImplicitModuleInterfaceBuilder {
       StringRef moduleName, StringRef moduleCachePath,
       StringRef backupInterfaceDir, StringRef prebuiltCachePath,
       StringRef ABIDescriptorPath, bool disableInterfaceFileLock = false,
+      bool silenceInterfaceDiagnostics = false,
       SourceLoc diagnosticLoc = SourceLoc(),
       DependencyTracker *tracker = nullptr)
       : sourceMgr(sourceMgr), diags(diags), subASTDelegate(subASTDelegate),
@@ -95,6 +97,7 @@ class ImplicitModuleInterfaceBuilder {
         backupInterfaceDir(backupInterfaceDir),
         ABIDescriptorPath(ABIDescriptorPath),
         disableInterfaceFileLock(disableInterfaceFileLock),
+        silenceInterfaceDiagnostics(silenceInterfaceDiagnostics),
         diagnosticLoc(diagnosticLoc), dependencyTracker(tracker) {}
 
   /// Ensures the requested file name is added as a dependency of the resulting

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -677,7 +677,7 @@ class ModuleInterfaceLoaderImpl {
   std::pair<std::string, std::string> getCompiledModuleCandidates() {
     std::pair<std::string, std::string> result;
     // Should we attempt to load a swiftmodule adjacent to the swiftinterface?
-    bool shouldLoadAdjacentModule = true;
+    bool shouldLoadAdjacentModule = !ctx.IgnoreAdjacentModules;
 
     // Don't use the adjacent swiftmodule for frameworks from the public
     // Frameworks folder of the SDK.
@@ -1031,7 +1031,8 @@ class ModuleInterfaceLoaderImpl {
         ctx.SourceMgr, diagsToUse,
         astDelegate, interfacePath, moduleName, cacheDir,
         prebuiltCacheDir, backupInterfaceDir, StringRef(),
-        Opts.disableInterfaceLock, diagnosticLoc,
+        Opts.disableInterfaceLock,
+        ctx.IgnoreAdjacentModules, diagnosticLoc,
         dependencyTracker);
       // If we found an out-of-date .swiftmodule, we still want to add it as
       // a dependency of the .swiftinterface. That way if it's updated, but
@@ -1063,7 +1064,8 @@ class ModuleInterfaceLoaderImpl {
       ImplicitModuleInterfaceBuilder fallbackBuilder(
         ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath, moduleName, cacheDir,
         prebuiltCacheDir, backupInterfaceDir, StringRef(),
-        Opts.disableInterfaceLock, diagnosticLoc,
+        Opts.disableInterfaceLock,
+        ctx.IgnoreAdjacentModules, diagnosticLoc,
         dependencyTracker);
       if (rebuildInfo.sawOutOfDateModule(modulePath))
         fallbackBuilder.addExtraDependency(modulePath);
@@ -1247,7 +1249,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
     StringRef OutPath, StringRef ABIOutputPath,
     bool SerializeDependencyHashes,
     bool TrackSystemDependencies, ModuleInterfaceLoaderOptions LoaderOpts,
-    RequireOSSAModules_t RequireOSSAModules) {
+    RequireOSSAModules_t RequireOSSAModules,
+    bool silenceInterfaceDiagnostics) {
   InterfaceSubContextDelegateImpl astDelegate(
       SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, LoaderOpts,
       /*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir,
@@ -1256,7 +1259,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
   ImplicitModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath,
                                          ModuleName, CacheDir, PrebuiltCacheDir,
                                          BackupInterfaceDir, ABIOutputPath,
-                                         LoaderOpts.disableInterfaceLock);
+                                         LoaderOpts.disableInterfaceLock,
+                                         silenceInterfaceDiagnostics);
   // FIXME: We really only want to serialize 'important' dependencies here, if
   //        we want to ship the built swiftmodules to another machine.
   auto failed = builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
@@ -1274,7 +1278,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
   ImplicitModuleInterfaceBuilder backupBuilder(SourceMgr, &Diags, astDelegate, backInPath,
                                                ModuleName, CacheDir, PrebuiltCacheDir,
                                                BackupInterfaceDir, ABIOutputPath,
-                                               LoaderOpts.disableInterfaceLock);
+                                               LoaderOpts.disableInterfaceLock,
+                                               silenceInterfaceDiagnostics);
   // Ensure we can rebuild module after user changed the original interface file.
   backupBuilder.addExtraDependency(InPath);
   // FIXME: We really only want to serialize 'important' dependencies here, if

lib/FrontendTool/FrontendTool.cpp

@@ -423,6 +423,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   StringRef ABIPath = Instance.getPrimarySpecificPathsForAtMostOnePrimary()
                           .SupplementaryOutputs.ABIDescriptorOutputPath;
+  bool IgnoreAdjacentModules = Instance.hasASTContext() &&
+                               Instance.getASTContext().IgnoreAdjacentModules;
 
   // If an explicit interface build was requested, bypass the creation of a new
   // sub-instance from the interface which will build it in a separate thread,
@@ -444,7 +446,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
       Invocation.getOutputFilename(), ABIPath,
       FEOpts.SerializeModuleInterfaceDependencyHashes,
       FEOpts.shouldTrackSystemDependencies(), LoaderOpts,
-      RequireOSSAModules_t(Invocation.getSILOptions()));
+      RequireOSSAModules_t(Invocation.getSILOptions()),
+      IgnoreAdjacentModules);
 }
 
 static bool compileLLVMIR(CompilerInstance &Instance) {

lib/Index/IndexRecord.cpp

@@ -513,14 +513,40 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   if (*isUptodateOpt)
     return false;
 
-  // FIXME: Would be useful for testing if swift had clang's -Rremark system so
-  // we could output a remark here that we are going to create index data for
-  // a module file.
+  // Reload resilient modules from swiftinterface to avoid indexing
+  // internal details.
+  bool skipIndexingModule = false;
+  if (module->getResilienceStrategy() == ResilienceStrategy::Resilient &&
+      !module->isBuiltFromInterface() &&
+      !module->isStdlibModule()) {
+    module->getASTContext().setIgnoreAdjacentModules(true);
+
+    ImportPath::Module::Builder builder(module->getName());
+    ASTContext &ctx = module->getASTContext();
+    auto reloadedModule = ctx.getModule(builder.get(),
+                                        /*AllowMemoryCached=*/false);
+
+    if (reloadedModule) {
+      module = reloadedModule;
+    } else {
+      // If we can't rebuild from the swiftinterface, don't index this module.
+      skipIndexingModule = true;
+    }
+  }
+
+  if (module->getASTContext().LangOpts.EnableIndexingSystemModuleRemarks) {
+    diags.diagnose(SourceLoc(),
+                   diag::remark_indexing_system_module,
+                   filename, skipIndexingModule);
+  }
 
   // Pairs of (recordFile, groupName).
   std::vector<std::pair<std::string, std::string>> records;
 
-  if (!module->isStdlibModule()) {
+  if (skipIndexingModule) {
+    // Don't add anything to records but keep going so we still mark the module
+    // as indexed to avoid rebuilds of broken swiftinterfaces.
+  } else if (!module->isStdlibModule()) {
     std::string recordFile;
     bool failed = false;
     auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(),

lib/Sema/SourceLoader.cpp

@@ -94,7 +94,8 @@ bool SourceLoader::canImportModule(ImportPath::Module path,
 }
 
 ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc,
-                                     ImportPath::Module path) {
+                                     ImportPath::Module path,
+                                     bool AllowMemoryCache) {
   // FIXME: Swift submodules?
   if (path.size() > 1)
     return nullptr;