Virtualize swift-frontend outputs

Using a virutal output backend to capture all the outputs from
swift-frontend invocation. This allows future virtualization of swift
compiler outputs.

Author: cachemeifyoucan

include/swift/AST/ASTContext.h

@@ -44,6 +44,7 @@
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include <functional>
 #include <memory>
 #include <utility>
@@ -227,6 +228,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
 
 public:
@@ -244,6 +246,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
   ~ASTContext();
 
@@ -277,6 +280,9 @@ class ASTContext final {
   /// Diags - The diagnostics engine.
   DiagnosticEngine &Diags;
 
+  /// OutputBackend for writing outputs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> Backend;
+
   /// If the shared pointer is not a \c nullptr and the pointee is \c true,
   /// all operations working on this ASTContext should be aborted at the next
   /// possible opportunity.
@@ -1450,6 +1456,18 @@ class ASTContext final {
   /// search within the library.
   void *getAddressOfSymbol(const char *name, void *libraryHandleHint = nullptr);
 
+  /// Get the output backend. The output backend needs to be initialized via
+  /// constructor or `setOutputBackend`.
+  llvm::vfs::OutputBackend &getOutputBackend() const {
+    assert(Backend && "OutputBackend is not setup");
+    return *Backend;
+  }
+  /// Set output backend for virtualized outputs.
+  void setOutputBackend(
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutputBackend) {
+    Backend = std::move(OutputBackend);
+  }
+
 private:
   friend Decl;
 

include/swift/AST/AbstractSourceFileDepGraphFactory.h

@@ -16,6 +16,7 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DeclContext.h"
 #include "swift/AST/FineGrainedDependencies.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 
 namespace swift {
 class DiagnosticEngine;
@@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory {
 
   DiagnosticEngine &diags;
 
+  /// OutputBackend.
+  llvm::vfs::OutputBackend &backend;
+
   /// Graph under construction
   SourceFileDepGraph g;
 
@@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory {
                                     StringRef swiftDeps,
                                     Fingerprint fileFingerprint,
                                     bool emitDotFileAfterConstruction,
-                                    DiagnosticEngine &diags);
+                                    DiagnosticEngine &diags,
+                                    llvm::vfs::OutputBackend &outputBackend);
 
   virtual ~AbstractSourceFileDepGraphFactory() = default;
 

include/swift/AST/FileSystem.h

@@ -16,29 +16,38 @@
 #include "swift/Basic/FileSystem.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsCommon.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
 
 namespace swift {
-
-/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any
-/// filesystem errors and asserts the output path is nonempty.
+/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file
+/// system errors during output creation.
 ///
 /// \returns true if there were any errors, either from the filesystem
 /// operations or from \p action returning true.
 inline bool
-withOutputFile(DiagnosticEngine &diags, StringRef outputPath,
-               llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
+withOutputBackend(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend,
+                  StringRef outputPath,
+                  llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
   assert(!outputPath.empty());
+  llvm::vfs::OutputConfig config;
+  config.setOverwriteOnlyDifferent();
+
+  auto outputFile = Backend.createFile(outputPath, config);
+  if (!outputFile) {
+    diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath,
+                   toString(outputFile.takeError()));
+    return false;
+  }
 
-  bool actionFailed = false;
-  std::error_code EC = swift::atomicallyWritingToFile(
-      outputPath,
-      [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); });
-  if (EC) {
+  bool result = action(*outputFile);
+  // If there is an error, discard output. Otherwise keep the output file.
+  if (auto error = result ? outputFile->discard() : outputFile->keep()) {
     diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath,
-                   EC.message());
-    return true;
+                   toString(std::move(error)));
+    return false;
   }
-  return actionFailed;
+  return result;
 }
 } // end namespace swift
 

include/swift/AST/FineGrainedDependencies.h

@@ -24,6 +24,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -357,8 +358,9 @@ class BiIndexedTwoStageMap {
 /// \Note The returned graph should not be escaped from the callback.
 bool withReferenceDependencies(
     llvm::PointerUnion<const ModuleDecl *, const SourceFile *> MSF,
-    const DependencyTracker &depTracker, StringRef outputPath,
-    bool alsoEmitDotFile, llvm::function_ref<bool(SourceFileDepGraph &&)>);
+    const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend,
+    StringRef outputPath, bool alsoEmitDotFile,
+    llvm::function_ref<bool(SourceFileDepGraph &&)>);
 
 //==============================================================================
 // MARK: Enums
@@ -895,7 +897,8 @@ class SourceFileDepGraph {
 
   bool verifySequenceNumber() const;
 
-  void emitDotFile(StringRef outputPath, DiagnosticEngine &diags);
+  void emitDotFile(llvm::vfs::OutputBackend &outputBackend,
+                   StringRef outputPath, DiagnosticEngine &diags);
 
   void addNode(SourceFileDepGraphNode *n) {
     n->setSequenceNumber(allNodes.size());

include/swift/AST/FineGrainedDependencyFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
+                                           llvm::vfs::OutputBackend &backend,
                                            llvm::StringRef path,
                                            const SourceFileDepGraph &g);
 

include/swift/AST/ModuleLoader.h

@@ -32,6 +32,9 @@
 
 namespace llvm {
 class FileCollectorBase;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace clang {
@@ -156,6 +159,7 @@ class ModuleInterfaceChecker {
   virtual bool tryEmitForwardingModule(StringRef moduleName,
                                StringRef interfacePath,
                                ArrayRef<std::string> candidates,
+                               llvm::vfs::OutputBackend &backend,
                                StringRef outPath) = 0;
   virtual ~ModuleInterfaceChecker() = default;
 };

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs{
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeInterModuleDependenciesCache(DiagnosticEngine &diags,
+                                       llvm::vfs::OutputBackend &backend,
                                        llvm::StringRef path,
                                        const SwiftDependencyScanningService &cache);
 

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -20,11 +20,14 @@
 #include "swift/Basic/OptionSet.h"
 #include "swift/Driver/Job.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -201,6 +204,9 @@ class ModuleDepGraph {
                           std::vector<const ModuleDepGraphNode *>>
       dependencyPathsToJobs;
 
+  /// VirtualOutputBackend for emitting graphs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> backend;
+
   /// For helping with performance tuning, may be null:
   UnifiedStatsReporter *stats;
 
@@ -306,6 +312,10 @@ class ModuleDepGraph {
                 : None),
         stats(stats) {
     assert(verify() && "ModuleDepGraph should be fine when created");
+
+    // Create a OnDiskOutputBackend for emitting graphs. Currently, this is
+    // only used by driver so the backend is not shared with a CompilerInstance.
+    backend = llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();
   }
 
   /// For unit tests.

include/swift/Frontend/Frontend.h

@@ -47,6 +47,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 
 #include <memory>
 
@@ -449,6 +450,9 @@ class CompilerInstance {
   /// instance has completed its setup, this will be null.
   std::unique_ptr<UnifiedStatsReporter> Stats;
 
+  /// Virtual OutputBackend.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> TheOutputBackend = nullptr;
+
   mutable ModuleDecl *MainModule = nullptr;
   SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr;
   MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr;
@@ -499,6 +503,12 @@ class CompilerInstance {
     SourceMgr.setFileSystem(FS);
   }
 
+  llvm::vfs::OutputBackend &getOutputBackend() { return *TheOutputBackend; }
+  void
+  setOutputBackend(llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> Backend) {
+    TheOutputBackend = std::move(Backend);
+  }
+
   ASTContext &getASTContext() { return *Context; }
   const ASTContext &getASTContext() const { return *Context; }
 
@@ -609,6 +619,7 @@ class CompilerInstance {
   bool setUpASTContextIfNeeded();
   void setupStatsReporter();
   void setupDependencyTrackerIfNeeded();
+  void setupOutputBackend();
 
   /// \return false if successful, true on error.
   bool setupDiagnosticVerifierIfNeeded();

include/swift/Frontend/FrontendOptions.h

@@ -424,6 +424,9 @@ class FrontendOptions {
   /// \return true if the given action requires input files to be provided.
   static bool doesActionRequireInputs(ActionType action);
 
+  /// \return true if the given action writes any output files.
+  static bool doesActionRequireOutputBackend(ActionType action);
+
   /// \return true if the given action requires input files to be provided.
   static bool doesActionPerformEndOfPipelineActions(ActionType action);
 

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -380,6 +380,7 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker {
   bool tryEmitForwardingModule(StringRef moduleName,
                                StringRef interfacePath,
                                ArrayRef<std::string> candidates,
+                               llvm::vfs::OutputBackend &backend,
                                StringRef outPath) override;
   bool isCached(StringRef DepPath);
 };

include/swift/Subsystems.h

@@ -35,6 +35,9 @@ namespace llvm {
   class Module;
   class TargetOptions;
   class TargetMachine;
+  namespace vfs {
+    class OutputBackend;
+  }
 }
 
 namespace swift {
@@ -275,13 +278,15 @@ namespace swift {
   /// \param Module LLVM module to code gen, required.
   /// \param TargetMachine target of code gen, required.
   /// \param OutputFilename Filename for output.
+  /// \param Backend OutputBackend for writing output.
   bool performLLVM(const IRGenOptions &Opts,
                    DiagnosticEngine &Diags,
                    llvm::sys::Mutex *DiagMutex,
                    llvm::GlobalVariable *HashGlobal,
                    llvm::Module *Module,
                    llvm::TargetMachine *TargetMachine,
                    StringRef OutputFilename,
+                   llvm::vfs::OutputBackend &Backend,
                    UnifiedStatsReporter *Stats);
 
   /// Dump YAML describing all fixed-size types imported from the given module.

lib/AST/ASTContext.cpp

@@ -68,6 +68,8 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include <algorithm>
 #include <queue>
 #include <memory>
@@ -621,6 +623,7 @@ ASTContext *ASTContext::get(
     ClangImporterOptions &ClangImporterOpts,
     symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
     SourceManager &SourceMgr, DiagnosticEngine &Diags,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutputBackend,
     std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback) {
   // If more than two data structures are concatentated, then the aggregate
   // size math needs to become more complicated due to per-struct alignment
@@ -632,9 +635,10 @@ ASTContext *ASTContext::get(
   impl = reinterpret_cast<void *>(
       llvm::alignAddr(impl, llvm::Align(alignof(Implementation))));
   new (impl) Implementation();
-  return new (mem) ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts,
-                              ClangImporterOpts, SymbolGraphOpts, SourceMgr,
-                              Diags, PreModuleImportCallback);
+  return new (mem)
+      ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts,
+                 ClangImporterOpts, SymbolGraphOpts, SourceMgr, Diags,
+                 std::move(OutputBackend), PreModuleImportCallback);
 }
 
 ASTContext::ASTContext(
@@ -643,17 +647,19 @@ ASTContext::ASTContext(
     ClangImporterOptions &ClangImporterOpts,
     symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
     SourceManager &SourceMgr, DiagnosticEngine &Diags,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutputBackend,
     std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback)
     : LangOpts(langOpts), TypeCheckerOpts(typecheckOpts), SILOpts(silOpts),
       SearchPathOpts(SearchPathOpts), ClangImporterOpts(ClangImporterOpts),
       SymbolGraphOpts(SymbolGraphOpts), SourceMgr(SourceMgr), Diags(Diags),
-      evaluator(Diags, langOpts), TheBuiltinModule(createBuiltinModule(*this)),
+      Backend(std::move(OutputBackend)), evaluator(Diags, langOpts),
+      TheBuiltinModule(createBuiltinModule(*this)),
       StdlibModuleName(getIdentifier(STDLIB_NAME)),
       SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)),
       PreModuleImportCallback(PreModuleImportCallback),
       TheErrorType(new (*this, AllocationArena::Permanent) ErrorType(
           *this, Type(), RecursiveTypeProperties::HasError)),
-      TheUnresolvedType(new (*this, AllocationArena::Permanent)
+      TheUnresolvedType(new(*this, AllocationArena::Permanent)
                             UnresolvedType(*this)),
       TheEmptyTupleType(TupleType::get(ArrayRef<TupleTypeElt>(), *this)),
       TheEmptyPackType(PackType::get(*this, {})),
@@ -692,6 +698,10 @@ ASTContext::ASTContext(
   registerAccessRequestFunctions(evaluator);
   registerNameLookupRequestFunctions(evaluator);
 
+  // Provide a default OnDiskOutputBackend if user didn't supply one.
+  if (!Backend)
+    Backend = llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();
+
   loadCompilerPlugins();
 }