Merge pull request #66575 from cachemeifyoucan/eng/PR-caching-tune-up

Various cleanups to allow better swift caching.

Author: cachemeifyoucan

include/swift/ClangImporter/ClangImporter.h

@@ -409,6 +409,11 @@ class ClangImporter final : public ClangModuleLoader {
   /// Reads the original source file name from PCH.
   std::string getOriginalSourceFile(StringRef PCHFilename);
 
+  /// Add clang dependency file names.
+  ///
+  /// \param files The list of file to append dependencies to.
+  void addClangInvovcationDependencies(std::vector<std::string> &files);
+
   /// Makes a temporary replica of the ClangImporter's CompilerInstance, reads a
   /// module map into the replica and emits a PCM file for one of the modules it
   /// declares. Delegates to clang for everything except construction of the

include/swift/ConstExtract/ConstExtract.h

@@ -55,7 +55,7 @@ gatherConstValuesForModule(const std::unordered_set<std::string> &Protocols,
 /// Serialize a collection of \c ConstValueInfos to JSON at the
 /// provided output stream.
 bool writeAsJSONToFile(const std::vector<ConstValueTypeInfo> &ConstValueInfos,
-                       llvm::raw_fd_ostream &OS);
+                       llvm::raw_ostream &OS);
 } // namespace swift
 
 #endif

lib/AST/ModuleDependencies.cpp

@@ -446,6 +446,11 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService(
     }
   }
 
+  // Fetch some dependency files from clang importer.
+  auto clangImporter = static_cast<ClangImporter *>(
+      Instance.getASTContext().getClangModuleLoader());
+  clangImporter->addClangInvovcationDependencies(CommonDependencyFiles);
+
   auto CachingFS =
       llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
   if (!CachingFS) {

lib/ClangImporter/ClangImporter.cpp

@@ -939,6 +939,27 @@ std::string ClangImporter::getOriginalSourceFile(StringRef PCHFilename) {
       Impl.Instance->getPCHContainerReader(), Impl.Instance->getDiagnostics());
 }
 
+void ClangImporter::addClangInvovcationDependencies(
+    std::vector<std::string> &files) {
+  auto addFiles = [&files](const auto &F) {
+    files.insert(files.end(), F.begin(), F.end());
+  };
+  auto &invocation = *Impl.Invocation;
+  // FIXME: Add file dependencies that are not accounted. The long term solution
+  // is to do a dependency scanning for clang importer and use that directly.
+  SmallVector<std::string, 4> HeaderMapFileNames;
+  Impl.Instance->getPreprocessor().getHeaderSearchInfo().getHeaderMapFileNames(
+      HeaderMapFileNames);
+  addFiles(HeaderMapFileNames);
+  addFiles(invocation.getHeaderSearchOpts().VFSOverlayFiles);
+  // FIXME: Should not depend on working directory. Build system/swift driver
+  // should not pass working directory here but if that option is passed,
+  // repect that and add that into CASFS.
+  auto CWD = invocation.getFileSystemOpts().WorkingDir;
+  if (!CWD.empty())
+    files.push_back(CWD);
+}
+
 Optional<std::string>
 ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions,
                               StringRef SwiftPCHHash, bool &isExplicit) {

lib/ConstExtract/ConstExtract.cpp

@@ -804,7 +804,7 @@ void writeAttrInformation(llvm::json::OStream &JSON,
 }
 
 bool writeAsJSONToFile(const std::vector<ConstValueTypeInfo> &ConstValueInfos,
-                       llvm::raw_fd_ostream &OS) {
+                       llvm::raw_ostream &OS) {
   llvm::json::OStream JSON(OS, 2);
   JSON.array([&] {
     for (const auto &TypeInfo : ConstValueInfos) {

lib/Frontend/ArgsToFrontendOutputsConverter.cpp

@@ -587,24 +587,9 @@ createFromTypeToPathMap(const TypeToPathMap *map) {
   if (!map)
     return paths;
   const std::pair<file_types::ID, std::string &> typesAndStrings[] = {
-      {file_types::TY_ClangHeader, paths.ClangHeaderOutputPath},
-      {file_types::TY_SwiftModuleFile, paths.ModuleOutputPath},
-      {file_types::TY_SwiftModuleDocFile, paths.ModuleDocOutputPath},
-      {file_types::TY_SwiftSourceInfoFile, paths.ModuleSourceInfoOutputPath},
-      {file_types::TY_Dependencies, paths.DependenciesFilePath},
-      {file_types::TY_SwiftDeps, paths.ReferenceDependenciesFilePath},
-      {file_types::TY_SerializedDiagnostics, paths.SerializedDiagnosticsPath},
-      {file_types::TY_ModuleTrace, paths.LoadedModuleTracePath},
-      {file_types::TY_TBD, paths.TBDPath},
-      {file_types::TY_SwiftModuleInterfaceFile,
-       paths.ModuleInterfaceOutputPath},
-      {file_types::TY_SwiftModuleSummaryFile, paths.ModuleSummaryOutputPath},
-      {file_types::TY_PrivateSwiftModuleInterfaceFile,
-       paths.PrivateModuleInterfaceOutputPath},
-      {file_types::TY_YAMLOptRecord, paths.YAMLOptRecordPath},
-      {file_types::TY_BitstreamOptRecord, paths.BitstreamOptRecordPath},
-      {file_types::TY_SwiftABIDescriptor, paths.ABIDescriptorOutputPath},
-      {file_types::TY_ConstValues, paths.ConstValuesOutputPath}
+#define OUTPUT(NAME, TYPE) {file_types::TYPE, paths.NAME},
+#include "swift/Basic/SupplementaryOutputPaths.def"
+#undef OUTPUT
   };
   for (const std::pair<file_types::ID, std::string &> &typeAndString :
        typesAndStrings) {

lib/Frontend/CachedDiagnostics.cpp

@@ -108,17 +108,33 @@ struct DiagnosticSerializer {
                         ReplayFunc Fn = nullptr);
   llvm::Error serializeEmittedDiagnostics(llvm::raw_ostream &os);
 
-  static llvm::Error emitDiagnosticsFromCached(llvm::StringRef Buffer,
-                                               SourceManager &SrcMgr,
-                                               DiagnosticEngine &Diags) {
+  static llvm::Error
+  emitDiagnosticsFromCached(llvm::StringRef Buffer, SourceManager &SrcMgr,
+                            DiagnosticEngine &Diags,
+                            const FrontendInputsAndOutputs &InAndOut) {
     // Create a new DiagnosticSerializer since this cannot be shared with a
     // serialization instance.
     DiagnosticSerializer DS(SrcMgr.getFileSystem());
+    DS.addInputsToSourceMgr(InAndOut);
     return DS.doEmitFromCached(Buffer, Diags);
   }
 
   SourceManager &getSourceMgr() { return SrcMgr; }
 
+  void addInputsToSourceMgr(const FrontendInputsAndOutputs &InAndOut) {
+    // Extract all the input file names so they can be added to the source
+    // manager when replaying the diagnostics. All input files are needed even
+    // they don't contain diagnostics because FileSpecificDiagConsumer need
+    // has references to input files to find subconsumer.
+    auto addInputToSourceMgr = [&](const InputFile &Input) {
+      if (Input.getFileName() != "-")
+        SrcMgr.getExternalSourceBufferID(Input.getFileName());
+      return false;
+    };
+    InAndOut.forEachInputProducingSupplementaryOutput(addInputToSourceMgr);
+    InAndOut.forEachNonPrimaryInput(addInputToSourceMgr);
+  }
+
 private:
   // Serialization helper
   unsigned getFileIDFromBufferID(SourceManager &SM, unsigned Idx);
@@ -632,7 +648,7 @@ class CachingDiagnosticsProcessor::Implementation
 
   llvm::Error replayCachedDiagnostics(llvm::StringRef Buffer) {
     return DiagnosticSerializer::emitDiagnosticsFromCached(
-        Buffer, getDiagnosticSourceMgr(), Diags);
+        Buffer, getDiagnosticSourceMgr(), Diags, InAndOut);
   }
 
   void handleDiagnostic(SourceManager &SM,
@@ -683,18 +699,7 @@ class CachingDiagnosticsProcessor::Implementation
     if (!Serializer) {
       Serializer.reset(
           new DiagnosticSerializer(InstanceSourceMgr.getFileSystem()));
-      auto &SM = Serializer->getSourceMgr();
-      // Extract all the input file names so they can be added to the source
-      // manager when replaying the diagnostics. All input files are needed even
-      // they don't contain diagnostics because FileSpecificDiagConsumer need
-      // has references to input files to find subconsumer.
-      auto addInputToSourceMgr = [&](const InputFile &Input) {
-        if (Input.getFileName() != "-")
-          SM.getExternalSourceBufferID(Input.getFileName());
-        return false;
-      };
-      InAndOut.forEachInputProducingSupplementaryOutput(addInputToSourceMgr);
-      InAndOut.forEachNonPrimaryInput(addInputToSourceMgr);
+      Serializer->addInputsToSourceMgr(InAndOut);
     }
 
     return *Serializer;

lib/Frontend/CompileJobCacheKey.cpp

@@ -20,6 +20,8 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CAS/HierarchicalTreeBuilder.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 
 using namespace swift;
 
@@ -30,29 +32,45 @@ llvm::Expected<llvm::cas::ObjectRef> swift::createCompileJobBaseCacheKey(
 
   // TODO: Improve this list.
   static const std::vector<std::string> removeArgAndNext = {
-      "-o", "-supplementary-output-file-map", "-serialize-diagnostics-path",
-      "-num-threads", "-cas-path"};
+      "-o",
+      "-output-filelist",
+      "-supplementary-output-file-map",
+      "-index-unit-output-path",
+      "-index-unit-output-path-filelist",
+      "-serialize-diagnostics-path",
+      "-num-threads",
+      "-cas-path"};
 
   // Don't count the `-frontend` in the first location since only frontend
   // invocation can have a cache key.
   if (Args.size() > 1 && StringRef(Args.front()) == "-frontend")
     Args = Args.drop_front();
 
-  bool SkipNext = false;
-  for (StringRef Arg : Args) {
-    if (SkipNext) {
-      SkipNext = false;
-      continue;
-    }
+  for (unsigned I = 0, IE =Args.size(); I < IE; ++I) {
+    StringRef Arg = Args[I];
     if (llvm::is_contained(removeArgAndNext, Arg)) {
-      SkipNext = true;
+      ++I;
       continue;
     }
     // FIXME: Use a heuristic to remove all the flags that affect output paths.
     // Those should not affect compile cache key.
     if (Arg.startswith("-emit-")) {
       if (Arg.endswith("-path"))
-        SkipNext = true;
+        ++I;
+      continue;
+    }
+    // Handle -file-list option. Need to drop the option but adds the file
+    // content instead.
+    // FIXME: will be nice if the same list of files gets the same key no matter
+    // going through command-line or filelist.
+    if (Arg == "-filelist" || Arg == "-primary-filelist") {
+      auto FileList = llvm::MemoryBuffer::getFile(Args[++I]);
+      if (!FileList)
+        return llvm::errorCodeToError(FileList.getError());
+      CommandLine.append(Arg);
+      CommandLine.push_back(0);
+      CommandLine.append((*FileList)->getBuffer());
+      CommandLine.push_back(0);
       continue;
     }
     CommandLine.append(Arg);

lib/Frontend/ModuleInterfaceBuilder.cpp

@@ -145,8 +145,11 @@ bool ExplicitModuleInterfaceBuilder::collectDepsForSerialization(
       continue;
 
     auto Status = fs.status(DepName);
-    if (!Status)
+    if (!Status) {
+      Instance.getDiags().diagnose(SourceLoc(), diag::cannot_open_file, DepName,
+                                   Status.getError().message());
       return true;
+    }
 
     /// Lazily load the dependency buffer if we need it. If we're not
     /// dealing with a hash-based dependencies, and if the dependency is
@@ -155,11 +158,14 @@ bool ExplicitModuleInterfaceBuilder::collectDepsForSerialization(
     auto getDepBuf = [&]() -> llvm::MemoryBuffer * {
       if (DepBuf)
         return DepBuf.get();
-      if (auto Buf = fs.getBufferForFile(DepName, /*FileSize=*/-1,
-                                         /*RequiresNullTerminator=*/false)) {
+      auto Buf = fs.getBufferForFile(DepName, /*FileSize=*/-1,
+                                     /*RequiresNullTerminator=*/false);
+      if (Buf) {
         DepBuf = std::move(Buf.get());
         return DepBuf.get();
       }
+      Instance.getDiags().diagnose(SourceLoc(), diag::cannot_open_file, DepName,
+                                   Buf.getError().message());
       return nullptr;
     };
 
@@ -287,11 +293,11 @@ std::error_code ExplicitModuleInterfaceBuilder::buildSwiftModuleFromInterface(
   SerializationOpts.ABIDescriptorPath = ABIDescriptorPath.str();
   SmallVector<FileDependency, 16> Deps;
   bool SerializeHashes = FEOpts.SerializeModuleInterfaceDependencyHashes;
-  if (collectDepsForSerialization(Deps, InterfacePath, SerializeHashes)) {
-    return std::make_error_code(std::errc::not_supported);
-  }
-  if (ShouldSerializeDeps)
+  if (ShouldSerializeDeps) {
+    if (collectDepsForSerialization(Deps, InterfacePath, SerializeHashes))
+      return std::make_error_code(std::errc::not_supported);
     SerializationOpts.Dependencies = Deps;
+  }
   SerializationOpts.IsOSSA = SILOpts.EnableOSSAModules;
 
   SILMod->setSerializeSILAction([&]() {

lib/FrontendTool/FrontendTool.cpp

@@ -726,10 +726,12 @@ static bool emitConstValuesForWholeModuleIfNeeded(
     return true;
   auto ConstValues = gatherConstValuesForModule(Protocols,
                                                 Instance.getMainModule());
-  std::error_code EC;
-  llvm::raw_fd_ostream OS(ConstValuesFilePath, EC, llvm::sys::fs::OF_None);
-  writeAsJSONToFile(ConstValues, OS);
-  return false;
+
+  return withOutputPath(Instance.getDiags(), Instance.getOutputBackend(),
+                        ConstValuesFilePath, [&](llvm::raw_ostream &OS) {
+                          writeAsJSONToFile(ConstValues, OS);
+                          return false;
+                        });
 }
 
 static void emitConstValuesForAllPrimaryInputsIfNeeded(
@@ -755,9 +757,11 @@ static void emitConstValuesForAllPrimaryInputsIfNeeded(
       continue;
 
     auto ConstValues = gatherConstValuesForPrimary(Protocols, SF);
-    std::error_code EC;
-    llvm::raw_fd_ostream OS(ConstValuesFilePath, EC, llvm::sys::fs::OF_None);
-    writeAsJSONToFile(ConstValues, OS);
+    withOutputPath(Instance.getDiags(), Instance.getOutputBackend(),
+                   ConstValuesFilePath, [&](llvm::raw_ostream &OS) {
+                     writeAsJSONToFile(ConstValues, OS);
+                     return false;
+                   });
   }
 }
 
@@ -772,9 +776,12 @@ static bool writeModuleSemanticInfoIfNeeded(CompilerInstance &Instance) {
   auto ModuleSemanticPath = frontendOpts.InputsAndOutputs
     .getPrimarySpecificPathsForAtMostOnePrimary().SupplementaryOutputs
     .ModuleSemanticInfoOutputPath;
-  llvm::raw_fd_ostream OS(ModuleSemanticPath, EC, llvm::sys::fs::OF_None);
-  OS << "{}\n";
-  return false;
+
+  return withOutputPath(Instance.getDiags(), Instance.getOutputBackend(),
+                        ModuleSemanticPath, [&](llvm::raw_ostream &OS) {
+                          OS << "{}\n";
+                          return false;
+                        });
 }
 
 static bool writeTBDIfNeeded(CompilerInstance &Instance) {
@@ -799,7 +806,8 @@ static bool writeTBDIfNeeded(CompilerInstance &Instance) {
 
   const std::string &TBDPath = Invocation.getTBDPathForWholeModule();
 
-  return writeTBD(Instance.getMainModule(), TBDPath, tbdOpts);
+  return writeTBD(Instance.getMainModule(), TBDPath,
+                  Instance.getOutputBackend(), tbdOpts);
 }
 
 static bool performCompileStepsPostSILGen(CompilerInstance &Instance,

lib/FrontendTool/TBD.cpp

@@ -15,6 +15,7 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/AST/FileSystem.h"
 #include "swift/AST/FileUnit.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/TBDGenRequests.h"
@@ -29,6 +30,7 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include <vector>
 
 using namespace swift;
@@ -42,18 +44,13 @@ static std::vector<StringRef> sortSymbols(llvm::StringSet<> &symbols) {
 }
 
 bool swift::writeTBD(ModuleDecl *M, StringRef OutputFilename,
+                     llvm::vfs::OutputBackend &Backend,
                      const TBDGenOptions &Opts) {
-  std::error_code EC;
-  llvm::raw_fd_ostream OS(OutputFilename, EC, llvm::sys::fs::OF_None);
-  if (EC) {
-    M->getASTContext().Diags.diagnose(SourceLoc(), diag::error_opening_output,
-                                      OutputFilename, EC.message());
-    return true;
-  }
-
-  writeTBDFile(M, OS, Opts);
-
-  return false;
+  return withOutputPath(M->getDiags(), Backend, OutputFilename,
+                        [&](raw_ostream &OS) -> bool {
+                          writeTBDFile(M, OS, Opts);
+                          return false;
+                        });
 }
 
 static bool validateSymbols(DiagnosticEngine &diags,

lib/FrontendTool/TBD.h

@@ -18,6 +18,9 @@
 namespace llvm {
 class StringRef;
 class Module;
+namespace vfs {
+class OutputBackend;
+}
 }
 namespace swift {
 class ModuleDecl;
@@ -26,7 +29,7 @@ class FrontendOptions;
 struct TBDGenOptions;
 
 bool writeTBD(ModuleDecl *M, StringRef OutputFilename,
-              const TBDGenOptions &Opts);
+              llvm::vfs::OutputBackend &Backend, const TBDGenOptions &Opts);
 bool validateTBD(ModuleDecl *M,
                  const llvm::Module &IRModule,
                  const TBDGenOptions &opts,