Virtualize swift compiler outputs (#63206)

Using a virutal output backend to capture all the outputs from
swift-frontend invocation. This allows redirecting and/or mirroring
compiler outputs to multiple location using different OutputBackend.

As an example usage for the virtual outputs, teach swift compiler to
check its output determinism by running the compiler invocation
twice and compare the hash of all its outputs.

Virtual output will be used to enable caching in the future.

Author: cachemeifyoucan

include/swift/APIDigester/ModuleAnalyzerNodes.h

@@ -784,8 +784,8 @@ class SwiftDeclCollector: public VisibleDeclConsumer {
   void deSerialize(StringRef Filename);
 
   // Serialize the content of all roots to a given file using JSON format.
-  void serialize(StringRef Filename);
-  static void serialize(StringRef Filename, SDKNode *Root, PayLoad otherInfo);
+  void serialize(llvm::raw_ostream &os);
+  static void serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad otherInfo);
 
   // After collecting decls, either from imported modules or from a previously
   // serialized JSON file, using this function to get the root of the SDK.
@@ -835,14 +835,15 @@ SDKNodeRoot *getSDKNodeRoot(SDKContext &SDKCtx,
 
 SDKNodeRoot *getEmptySDKNodeRoot(SDKContext &SDKCtx);
 
-void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, StringRef OutputFile);
-void dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile);
+void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, llvm::raw_ostream &os);
+void dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os);
 
 int dumpSDKContent(const CompilerInvocation &InitInvoke,
                    const llvm::StringSet<> &ModuleNames,
-                   StringRef OutputFile, CheckerOptions Opts);
+                   llvm::raw_ostream &os, CheckerOptions Opts);
 
-void dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, bool ABI, bool Empty);
+void dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI,
+                       bool Empty);
 
 /// Mostly for testing purposes, this function de-serializes the SDK dump in
 /// dumpPath and re-serialize them to OutputPath. If the tool performs correctly,

include/swift/AST/ASTContext.h

@@ -44,6 +44,7 @@
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include <functional>
 #include <memory>
 #include <utility>
@@ -231,6 +232,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
 
 public:
@@ -248,6 +250,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
   ~ASTContext();
 
@@ -281,6 +284,9 @@ class ASTContext final {
   /// Diags - The diagnostics engine.
   DiagnosticEngine &Diags;
 
+  /// OutputBackend for writing outputs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutputBackend;
+
   /// If the shared pointer is not a \c nullptr and the pointee is \c true,
   /// all operations working on this ASTContext should be aborted at the next
   /// possible opportunity.
@@ -1518,6 +1524,18 @@ class ASTContext final {
 
   const llvm::StringSet<> &getLoadedPluginLibraryPaths() const;
 
+  /// Get the output backend. The output backend needs to be initialized via
+  /// constructor or `setOutputBackend`.
+  llvm::vfs::OutputBackend &getOutputBackend() const {
+    assert(OutputBackend && "OutputBackend is not setup");
+    return *OutputBackend;
+  }
+  /// Set output backend for virtualized outputs.
+  void setOutputBackend(
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend) {
+    OutputBackend = std::move(OutBackend);
+  }
+
 private:
   friend Decl;
 

include/swift/AST/AbstractSourceFileDepGraphFactory.h

@@ -16,6 +16,7 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DeclContext.h"
 #include "swift/AST/FineGrainedDependencies.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 
 namespace swift {
 class DiagnosticEngine;
@@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory {
 
   DiagnosticEngine &diags;
 
+  /// OutputBackend.
+  llvm::vfs::OutputBackend &backend;
+
   /// Graph under construction
   SourceFileDepGraph g;
 
@@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory {
                                     StringRef swiftDeps,
                                     Fingerprint fileFingerprint,
                                     bool emitDotFileAfterConstruction,
-                                    DiagnosticEngine &diags);
+                                    DiagnosticEngine &diags,
+                                    llvm::vfs::OutputBackend &outputBackend);
 
   virtual ~AbstractSourceFileDepGraphFactory() = default;
 

include/swift/AST/DiagnosticsCommon.def

@@ -29,6 +29,9 @@ ERROR(not_implemented,none,
 ERROR(error_opening_output,none,
       "error opening '%0' for output: %1", (StringRef, StringRef))
 
+ERROR(error_closing_output,none,
+      "error closing '%0' for output: %1", (StringRef, StringRef))
+
 ERROR(cannot_find_group_info_file,none,
       "cannot find group info file at path: '%0'", (StringRef))
 

include/swift/AST/DiagnosticsFrontend.def

@@ -475,6 +475,16 @@ REMARK(interface_file_backup_used,none,
 
 WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef))
 
+// Output deterministic check
+ERROR(error_nondeterministic_output,none,
+      "output file '%0' is not deterministic: hash value '%1' vs '%2' between two compilations",
+      (StringRef, StringRef, StringRef))
+ERROR(error_output_missing,none,
+      "output file '%0' is missing from %select{first|second}1 compilation for deterministic check",
+      (StringRef, /*SecondRun=*/bool))
+REMARK(matching_output_produced,none,
+      "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/AST/FileSystem.h

@@ -16,29 +16,42 @@
 #include "swift/Basic/FileSystem.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsCommon.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
 
 namespace swift {
-
-/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any
-/// filesystem errors and asserts the output path is nonempty.
+/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file
+/// system errors during output creation.
 ///
 /// \returns true if there were any errors, either from the filesystem
 /// operations or from \p action returning true.
 inline bool
-withOutputFile(DiagnosticEngine &diags, StringRef outputPath,
+withOutputPath(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend,
+               StringRef outputPath,
                llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
   assert(!outputPath.empty());
+  llvm::vfs::OutputConfig config;
+  config.setAtomicWrite().setOnlyIfDifferent();
 
-  bool actionFailed = false;
-  std::error_code EC = swift::atomicallyWritingToFile(
-      outputPath,
-      [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); });
-  if (EC) {
+  auto outputFile = Backend.createFile(outputPath, config);
+  if (!outputFile) {
     diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath,
-                   EC.message());
+                   toString(outputFile.takeError()));
+    return true;
+  }
+
+  bool failed = action(*outputFile);
+  // If there is an error, discard output. Otherwise keep the output file.
+  if (auto error = failed ? outputFile->discard() : outputFile->keep()) {
+    // Don't diagnose discard error.
+    if (failed)
+      consumeError(std::move(error));
+    else
+      diags.diagnose(SourceLoc(), diag::error_closing_output, outputPath,
+                     toString(std::move(error)));
     return true;
   }
-  return actionFailed;
+  return failed;
 }
 } // end namespace swift
 

include/swift/AST/FineGrainedDependencies.h

@@ -24,6 +24,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -357,8 +358,9 @@ class BiIndexedTwoStageMap {
 /// \Note The returned graph should not be escaped from the callback.
 bool withReferenceDependencies(
     llvm::PointerUnion<const ModuleDecl *, const SourceFile *> MSF,
-    const DependencyTracker &depTracker, StringRef outputPath,
-    bool alsoEmitDotFile, llvm::function_ref<bool(SourceFileDepGraph &&)>);
+    const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend,
+    StringRef outputPath, bool alsoEmitDotFile,
+    llvm::function_ref<bool(SourceFileDepGraph &&)>);
 
 //==============================================================================
 // MARK: Enums
@@ -895,7 +897,8 @@ class SourceFileDepGraph {
 
   bool verifySequenceNumber() const;
 
-  void emitDotFile(StringRef outputPath, DiagnosticEngine &diags);
+  void emitDotFile(llvm::vfs::OutputBackend &outputBackend,
+                   StringRef outputPath, DiagnosticEngine &diags);
 
   void addNode(SourceFileDepGraphNode *n) {
     n->setSequenceNumber(allNodes.size());

include/swift/AST/FineGrainedDependencyFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
+                                           llvm::vfs::OutputBackend &backend,
                                            llvm::StringRef path,
                                            const SourceFileDepGraph &g);
 

include/swift/AST/IRGenOptions.h

@@ -318,6 +318,9 @@ class IRGenOptions {
   /// Print the LLVM inline tree at the end of the LLVM pass pipeline.
   unsigned PrintInlineTree : 1;
 
+  /// Always recompile the output even if the module hash might match.
+  unsigned AlwaysCompile : 1;
+
   /// Whether we should embed the bitcode file.
   IRGenEmbedMode EmbedMode : 2;
 
@@ -481,7 +484,7 @@ class IRGenOptions {
         DisableLLVMOptzns(false), DisableSwiftSpecificLLVMOptzns(false),
         Playground(false),
         EmitStackPromotionChecks(false), UseSingleModuleLLVMEmission(false),
-        FunctionSections(false), PrintInlineTree(false),
+        FunctionSections(false), PrintInlineTree(false), AlwaysCompile(false),
         EmbedMode(IRGenEmbedMode::None), LLVMLTOKind(IRGenLLVMLTOKind::None),
         SwiftAsyncFramePointer(SwiftAsyncFramePointerKind::Auto),
         HasValueNamesSetting(false), ValueNames(false),

include/swift/AST/ModuleLoader.h

@@ -32,6 +32,9 @@
 
 namespace llvm {
 class FileCollectorBase;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace clang {
@@ -156,6 +159,7 @@ class ModuleInterfaceChecker {
   virtual bool tryEmitForwardingModule(StringRef moduleName,
                                StringRef interfacePath,
                                ArrayRef<std::string> candidates,
+                               llvm::vfs::OutputBackend &backend,
                                StringRef outPath) = 0;
   virtual ~ModuleInterfaceChecker() = default;
 };

include/swift/ClangImporter/ClangImporter.h

@@ -28,6 +28,7 @@ namespace llvm {
   template<typename Fn> class function_ref;
   namespace vfs {
     class FileSystem;
+    class OutputBackend;
   }
 }
 
@@ -394,8 +395,7 @@ class ClangImporter final : public ClangModuleLoader {
   /// replica.
   ///
   /// \sa clang::GeneratePCHAction
-  bool emitBridgingPCH(StringRef headerPath,
-                       StringRef outputPCHPath);
+  bool emitBridgingPCH(StringRef headerPath, StringRef outputPCHPath);
 
   /// Returns true if a clang CompilerInstance can successfully read in a PCH,
   /// assuming it exists, with the current options. This can be used to find out

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs{
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeInterModuleDependenciesCache(DiagnosticEngine &diags,
+                                       llvm::vfs::OutputBackend &backend,
                                        llvm::StringRef path,
                                        const SwiftDependencyScanningService &cache);
 

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -20,11 +20,14 @@
 #include "swift/Basic/OptionSet.h"
 #include "swift/Driver/Job.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -201,6 +204,9 @@ class ModuleDepGraph {
                           std::vector<const ModuleDepGraphNode *>>
       dependencyPathsToJobs;
 
+  /// VirtualOutputBackend for emitting graphs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> backend;
+
   /// For helping with performance tuning, may be null:
   UnifiedStatsReporter *stats;
 
@@ -306,6 +312,10 @@ class ModuleDepGraph {
                 : None),
         stats(stats) {
     assert(verify() && "ModuleDepGraph should be fine when created");
+
+    // Create a OnDiskOutputBackend for emitting graphs. Currently, this is
+    // only used by driver so the backend is not shared with a CompilerInstance.
+    backend = llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();
   }
 
   /// For unit tests.