Frontend: teach the compiler to use a backup directory to find .swiftinterface files to compile

include/swift/AST/DiagnosticsFrontend.def

@@ -405,6 +405,8 @@ ERROR(error_option_required,none, "option '%0' is required", (StringRef))
 ERROR(error_nonexistent_output_dir,none,
       "'-output-dir' argument '%0' does not exist or is not a directory", (StringRef))
 
+REMARK(interface_file_backup_used,none,
+      "building module from '%0' failed; retrying building module from '%1'", (StringRef, StringRef))
 
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,

include/swift/AST/ModuleLoader.h

@@ -157,6 +157,8 @@ class ModuleInterfaceChecker {
                                ArrayRef<std::string> candidates,
                                StringRef outPath) = 0;
   virtual ~ModuleInterfaceChecker() = default;
+  virtual std::string getBackupPublicModuleInterfacePath(StringRef moduleName,
+                                                         StringRef interfacePath) = 0;
 };
 
 /// Abstract interface to run an action in a sub ASTContext.

include/swift/Frontend/FrontendOptions.h

@@ -81,6 +81,10 @@ class FrontendOptions {
   /// binary module has already been built for use by the compiler.
   std::string PrebuiltModuleCachePath;
 
+  /// The path to look in to find backup .swiftinterface files if those found
+  /// from SDKs are failing.
+  std::string BackupModuleInterfaceDir;
+
   /// For these modules, we should prefer using Swift interface when importing them.
   std::vector<std::string> PreferInterfaceForModules;
 

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -336,17 +336,25 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker {
   ASTContext &Ctx;
   std::string CacheDir;
   std::string PrebuiltCacheDir;
+  std::string BackupInterfaceDir;
   ModuleInterfaceLoaderOptions Opts;
   RequireOSSAModules_t RequiresOSSAModules;
 
 public:
   explicit ModuleInterfaceCheckerImpl(ASTContext &Ctx, StringRef cacheDir,
                                       StringRef prebuiltCacheDir,
+                                      StringRef BackupInterfaceDir,
                                       ModuleInterfaceLoaderOptions opts,
                                       RequireOSSAModules_t requiresOSSAModules)
       : Ctx(Ctx), CacheDir(cacheDir), PrebuiltCacheDir(prebuiltCacheDir),
+        BackupInterfaceDir(BackupInterfaceDir),
         Opts(opts), RequiresOSSAModules(requiresOSSAModules) {}
-
+  explicit ModuleInterfaceCheckerImpl(ASTContext &Ctx, StringRef cacheDir,
+                                      StringRef prebuiltCacheDir,
+                                      ModuleInterfaceLoaderOptions opts,
+                                      RequireOSSAModules_t requiresOSSAModules):
+    ModuleInterfaceCheckerImpl(Ctx, cacheDir, prebuiltCacheDir, StringRef(),
+                               opts, requiresOSSAModules) {}
   std::vector<std::string>
   getCompiledModuleCandidatesForInterface(StringRef moduleName,
                                           StringRef interfacePath) override;
@@ -359,6 +367,8 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker {
                                ArrayRef<std::string> candidates,
                                StringRef outPath) override;
   bool isCached(StringRef DepPath);
+  std::string getBackupPublicModuleInterfacePath(StringRef moduleName,
+                                                 StringRef interfacePath) override;
 };
 
 /// A ModuleLoader that runs a subordinate \c CompilerInvocation and
@@ -414,7 +424,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
       const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts,
       const ClangImporterOptions &ClangOpts, StringRef CacheDir,
-      StringRef PrebuiltCacheDir, StringRef ModuleName, StringRef InPath,
+      StringRef PrebuiltCacheDir, StringRef BackupInterfaceDir,
+      StringRef ModuleName, StringRef InPath,
       StringRef OutPath, bool SerializeDependencyHashes,
       bool TrackSystemDependencies, ModuleInterfaceLoaderOptions Opts,
       RequireOSSAModules_t RequireOSSAModules);
@@ -423,7 +434,9 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
 struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {
 private:
   SourceManager &SM;
-  DiagnosticEngine &Diags;
+public:
+  DiagnosticEngine *Diags;
+private:
   llvm::BumpPtrAllocator Allocator;
   llvm::StringSaver ArgSaver;
   std::vector<StringRef> GenericArgs;
@@ -439,7 +452,7 @@ struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {
       // Diagnose this inside the interface file, if possible.
       loc = SM.getLocFromExternalSource(interfacePath, 1, 1);
     }
-    return Diags.diagnose(loc, ID, std::move(Args)...);
+    return Diags->diagnose(loc, ID, std::move(Args)...);
   }
   void
   inheritOptionsForBuildingInterface(const SearchPathOptions &SearchPathOpts,
@@ -452,11 +465,12 @@ struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {
                                            SourceLoc diagnosticLoc);
 public:
   InterfaceSubContextDelegateImpl(
-      SourceManager &SM, DiagnosticEngine &Diags,
+      SourceManager &SM, DiagnosticEngine *Diags,
       const SearchPathOptions &searchPathOpts, const LangOptions &langOpts,
       const ClangImporterOptions &clangImporterOpts,
       ModuleInterfaceLoaderOptions LoaderOpts, bool buildModuleCacheDirIfAbsent,
       StringRef moduleCachePath, StringRef prebuiltCachePath,
+      StringRef backupModuleInterfaceDir,
       bool serializeDependencyHashes, bool trackSystemDependencies,
       RequireOSSAModules_t requireOSSAModules);
   std::error_code runInSubContext(StringRef moduleName,

include/swift/Option/FrontendOptions.td

@@ -695,6 +695,13 @@ def prebuilt_module_cache_path_EQ :
   Joined<["-"], "prebuilt-module-cache-path=">,
   Alias<prebuilt_module_cache_path>;
 
+def backup_module_interface_path :
+  Separate<["-"], "backup-module-interface-path">,
+  HelpText<"Directory of module interfaces as backups to those from SDKs">;
+def backup_module_interface_path_EQ :
+  Joined<["-"], "backup-module-interface-path=">,
+  Alias<backup_module_interface_path>;
+
 def force_public_linkage : Flag<["-"], "force-public-linkage">,
    HelpText<"Force public linkage for private symbols. Used by LLDB.">;
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -471,6 +471,9 @@ class SerializedASTFile final : public LoadedFile {
 /// Extract compiler arguments from an interface file buffer.
 bool extractCompilerFlagsFromInterface(StringRef buffer, llvm::StringSaver &ArgSaver,
                                        SmallVectorImpl<const char *> &SubArgs);
+
+/// Extract the user module version number from an interface file.
+llvm::VersionTuple extractUserModuleVersionFromInterface(StringRef moduleInterfacePath);
 } // end namespace swift
 
 #endif

lib/DependencyScan/ScanDependencies.cpp

@@ -1240,10 +1240,11 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   auto &FEOpts = instance.getInvocation().getFrontendOptions();
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   InterfaceSubContextDelegateImpl ASTDelegate(
-      ctx.SourceMgr, ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
+      ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
       ctx.ClangImporterOpts, LoaderOpts,
       /*buildModuleCacheDirIfAbsent*/ false, ModuleCachePath,
       FEOpts.PrebuiltModuleCachePath,
+      FEOpts.BackupModuleInterfaceDir,
       FEOpts.SerializeModuleInterfaceDependencyHashes,
       FEOpts.shouldTrackSystemDependencies(),
       RequireOSSAModules_t(instance.getSILOptions()));
@@ -1332,10 +1333,11 @@ swift::dependencies::performBatchModuleScan(
                         std::set<ModuleDependencyID>>
             allModules;
         InterfaceSubContextDelegateImpl ASTDelegate(
-            ctx.SourceMgr, ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
+            ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
             ctx.ClangImporterOpts, LoaderOpts,
             /*buildModuleCacheDirIfAbsent*/ false, ModuleCachePath,
             FEOpts.PrebuiltModuleCachePath,
+            FEOpts.BackupModuleInterfaceDir,
             FEOpts.SerializeModuleInterfaceDependencyHashes,
             FEOpts.shouldTrackSystemDependencies(),
             RequireOSSAModules_t(instance.getSILOptions()));
@@ -1400,10 +1402,11 @@ swift::dependencies::performBatchModulePrescan(
                         std::set<ModuleDependencyID>>
             allModules;
         InterfaceSubContextDelegateImpl ASTDelegate(
-            ctx.SourceMgr, ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
+            ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
             ctx.ClangImporterOpts, LoaderOpts,
             /*buildModuleCacheDirIfAbsent*/ false, ModuleCachePath,
             FEOpts.PrebuiltModuleCachePath,
+            FEOpts.BackupModuleInterfaceDir,
             FEOpts.SerializeModuleInterfaceDependencyHashes,
             FEOpts.shouldTrackSystemDependencies(),
             RequireOSSAModules_t(instance.getSILOptions()));

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -63,6 +63,9 @@ bool ArgsToFrontendOptionsConverter::convert(
   if (const Arg *A = Args.getLastArg(OPT_prebuilt_module_cache_path)) {
     Opts.PrebuiltModuleCachePath = A->getValue();
   }
+  if (const Arg *A = Args.getLastArg(OPT_backup_module_interface_path)) {
+    Opts.BackupModuleInterfaceDir = A->getValue();
+  }
   if (const Arg *A = Args.getLastArg(OPT_bridging_header_directory_for_print)) {
     Opts.BridgingHeaderDirForPrint = A->getValue();
   }

lib/Frontend/CompilerInvocation.cpp

@@ -129,6 +129,13 @@ void CompilerInvocation::setDefaultPrebuiltCacheIfNecessary() {
 
   FrontendOpts.PrebuiltModuleCachePath = computePrebuiltCachePath(
       SearchPathOpts.RuntimeResourcePath, LangOpts.Target, LangOpts.SDKVersion);
+  if (!FrontendOpts.PrebuiltModuleCachePath.empty())
+    return;
+  StringRef anchor = "prebuilt-modules";
+  assert(((StringRef)FrontendOpts.PrebuiltModuleCachePath).contains(anchor));
+  auto pair = ((StringRef)FrontendOpts.PrebuiltModuleCachePath).split(anchor);
+  FrontendOpts.BackupModuleInterfaceDir =
+    (llvm::Twine(pair.first) + "preferred-interfaces" + pair.second).str();
 }
 
 static void updateRuntimeLibraryPaths(SearchPathOptions &SearchPathOpts,

lib/Frontend/Frontend.cpp

@@ -530,7 +530,8 @@ bool CompilerInstance::setUpModuleLoaders() {
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   Context->addModuleInterfaceChecker(
       std::make_unique<ModuleInterfaceCheckerImpl>(
-          *Context, ModuleCachePath, FEOpts.PrebuiltModuleCachePath, LoaderOpts,
+          *Context, ModuleCachePath, FEOpts.PrebuiltModuleCachePath,
+          FEOpts.BackupModuleInterfaceDir, LoaderOpts,
           RequireOSSAModules_t(Invocation.getSILOptions())));
   // If implicit modules are disabled, we need to install an explicit module
   // loader.
@@ -571,10 +572,11 @@ bool CompilerInstance::setUpModuleLoaders() {
     auto &FEOpts = Invocation.getFrontendOptions();
     ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
     InterfaceSubContextDelegateImpl ASTDelegate(
-        Context->SourceMgr, Context->Diags, Context->SearchPathOpts,
+        Context->SourceMgr, &Context->Diags, Context->SearchPathOpts,
         Context->LangOpts, Context->ClangImporterOpts, LoaderOpts,
         /*buildModuleCacheDirIfAbsent*/ false, ModuleCachePath,
         FEOpts.PrebuiltModuleCachePath,
+        FEOpts.BackupModuleInterfaceDir,
         FEOpts.SerializeModuleInterfaceDependencyHashes,
         FEOpts.shouldTrackSystemDependencies(),
         RequireOSSAModules_t(Invocation.getSILOptions()));

lib/Frontend/ModuleInterfaceBuilder.cpp

@@ -103,7 +103,11 @@ bool ModuleInterfaceBuilder::collectDepsForSerialization(
     // dependency list -- don't serialize that.
     if (!prebuiltCachePath.empty() && DepName.startswith(prebuiltCachePath))
       continue;
-
+    // Don't serialize interface path if it's from the preferred interface dir.
+    // This ensures the prebuilt module caches generated from these interfaces are
+    // relocatable.
+    if (!backupInterfaceDir.empty() && DepName.startswith(backupInterfaceDir))
+      continue;
     if (dependencyTracker) {
       dependencyTracker->addDependency(DepName, /*isSystem*/IsSDKRelative);
     }

lib/Frontend/ModuleInterfaceBuilder.h

@@ -35,12 +35,13 @@ class DependencyTracker;
 
 class ModuleInterfaceBuilder {
   SourceManager &sourceMgr;
-  DiagnosticEngine &diags;
+  DiagnosticEngine *diags;
   InterfaceSubContextDelegate &subASTDelegate;
   const StringRef interfacePath;
   const StringRef moduleName;
   const StringRef moduleCachePath;
   const StringRef prebuiltCachePath;
+  const StringRef backupInterfaceDir;
   const bool disableInterfaceFileLock;
   const SourceLoc diagnosticLoc;
   DependencyTracker *const dependencyTracker;
@@ -50,7 +51,7 @@ class ModuleInterfaceBuilder {
   /// Emit a diagnostic tied to this declaration.
   template<typename ...ArgTypes>
   static InFlightDiagnostic diagnose(
-      DiagnosticEngine &Diags,
+      DiagnosticEngine *Diags,
       SourceManager &SM,
       StringRef InterfacePath,
       SourceLoc Loc,
@@ -60,7 +61,7 @@ class ModuleInterfaceBuilder {
       // Diagnose this inside the interface file, if possible.
       Loc = SM.getLocFromExternalSource(InterfacePath, 1, 1);
     }
-    return Diags.diagnose(Loc, ID, std::move(Args)...);
+    return Diags->diagnose(Loc, ID, std::move(Args)...);
   }
 
 private:
@@ -88,11 +89,12 @@ class ModuleInterfaceBuilder {
                                 std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
                                 ArrayRef<std::string> CandidateModules);
 public:
-  ModuleInterfaceBuilder(SourceManager &sourceMgr, DiagnosticEngine &diags,
+  ModuleInterfaceBuilder(SourceManager &sourceMgr, DiagnosticEngine *diags,
                             InterfaceSubContextDelegate &subASTDelegate,
                             StringRef interfacePath,
                             StringRef moduleName,
                             StringRef moduleCachePath,
+                            StringRef backupInterfaceDir,
                             StringRef prebuiltCachePath,
                             bool disableInterfaceFileLock = false,
                             SourceLoc diagnosticLoc = SourceLoc(),
@@ -101,6 +103,7 @@ class ModuleInterfaceBuilder {
       subASTDelegate(subASTDelegate),
       interfacePath(interfacePath), moduleName(moduleName),
       moduleCachePath(moduleCachePath), prebuiltCachePath(prebuiltCachePath),
+      backupInterfaceDir(backupInterfaceDir),
       disableInterfaceFileLock(disableInterfaceFileLock),
       diagnosticLoc(diagnosticLoc), dependencyTracker(tracker) {}