Add an option to verify the deterministic output of swift compiler

Add option to test swift compiler determinism by running the job twice
and compare the output produced.

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -449,6 +449,16 @@ REMARK(interface_file_backup_used,none,
 
 WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef))
 
+// Output deterministic check
+ERROR(error_nondeterministic_output,none,
+      "output file '%0' is not deterministic: hash value '%1' vs '%2'",
+      (StringRef, StringRef, StringRef))
+ERROR(error_output_missing,none,
+      "output file '%0' is missing from %select{first|second}1 run",
+      (StringRef, /*SecondRun=*/bool))
+REMARK(matching_output_produced,none,
+      "produced matching output file '%0': hash '%1'", (StringRef, StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/Frontend/Frontend.h

@@ -45,6 +45,8 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Option/ArgList.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/HashingOutputBackend.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualOutputBackend.h"
@@ -453,6 +455,10 @@ class CompilerInstance {
   /// Virtual OutputBackend.
   llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> TheOutputBackend = nullptr;
 
+  /// The verification output backend.
+  using HashBackendTy = llvm::vfs::HashingOutputBackend<llvm::BLAKE3>;
+  llvm::IntrusiveRefCntPtr<HashBackendTy> HashBackend;
+
   mutable ModuleDecl *MainModule = nullptr;
   SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr;
   MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr;
@@ -508,6 +514,8 @@ class CompilerInstance {
   setOutputBackend(llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> Backend) {
     TheOutputBackend = std::move(Backend);
   }
+  using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr<HashBackendTy>;
+  HashingBackendPtrTy getHashingBackend() { return HashBackend; }
 
   ASTContext &getASTContext() { return *Context; }
   const ASTContext &getASTContext() const { return *Context; }

include/swift/Frontend/FrontendOptions.h

@@ -492,6 +492,9 @@ class FrontendOptions {
   /// to encode the actual paths into the .swiftmodule file.
   PathObfuscator serializedPathObfuscator;
 
+  /// Whether to run the job twice to check determinism.
+  bool DeterministicCheck = false;
+
   /// Avoid printing actual module content into the ABI descriptor file.
   /// This should only be used as a workaround when emitting ABI descriptor files
   /// crashes the compiler.

include/swift/Option/FrontendOptions.td

@@ -1135,6 +1135,10 @@ def enable_emit_generic_class_ro_t_list :
   HelpText<"Enable emission of a section with references to class_ro_t of "
            "generic class patterns">;
 
+def enable_swift_deterministic_check :
+  Flag<["-"], "enable-swift-deterministic-check">,
+  HelpText<"Check swift compiler output determinisim by run it twice">;
+
 def experimental_spi_only_imports :
   Flag<["-"], "experimental-spi-only-imports">,
   HelpText<"Enable use of @_spiOnly imports">;

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -338,6 +338,7 @@ bool ArgsToFrontendOptionsConverter::convert(
     Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second);
   }
   Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor);
+  Opts.DeterministicCheck = Args.hasArg(OPT_enable_swift_deterministic_check);
   return false;
 }
 

lib/Frontend/Frontend.cpp

@@ -401,6 +401,18 @@ void CompilerInstance::setupOutputBackend() {
           Invocation.getFrontendOptions().RequestedAction))
     return;
 
+  // Setup verification backend.
+  // Create a mirroring outputbackend to produce hash for output files.
+  // We cannot skip disk here since swift compiler is expecting to read back
+  // some output file in later stages.
+  if (Invocation.getFrontendOptions().DeterministicCheck) {
+    HashBackend = llvm::makeIntrusiveRefCnt<HashBackendTy>();
+    TheOutputBackend = llvm::vfs::makeMirroringOutputBackend(
+        llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>(),
+        HashBackend);
+    return;
+  }
+
   // Default to on disk output backend.
   TheOutputBackend =
       llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();

lib/FrontendTool/FrontendTool.cpp

@@ -71,7 +71,10 @@
 
 #include "clang/Lex/Preprocessor.h"
 
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -82,6 +85,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/FileSystem.h"
 
@@ -2338,6 +2342,23 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     PDC.setSuppressOutput(true);
   }
 
+  CompilerInstance::HashingBackendPtrTy HashBackend = nullptr;
+  if (Invocation.getFrontendOptions().DeterministicCheck) {
+    // Setup a verfication instance to run.
+    std::unique_ptr<CompilerInstance> VerifyInstance =
+        std::make_unique<CompilerInstance>();
+    std::string InstanceSetupError;
+    // This should not fail because it passed already.
+    (void)VerifyInstance->setup(Invocation, InstanceSetupError);
+
+    // Run the first time without observer and discard return value;
+    int ReturnValueTest = 0;
+    (void)performCompile(*VerifyInstance, ReturnValueTest,
+                         /*observer*/ nullptr);
+    // Get the hashing output backend and free the compiler instance.
+    HashBackend = VerifyInstance->getHashingBackend();
+  }
+
   int ReturnValue = 0;
   bool HadError = performCompile(*Instance, ReturnValue, observer);
 
@@ -2352,6 +2373,44 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     }
   }
 
+  if (Invocation.getFrontendOptions().DeterministicCheck) {
+    // Collect all output files.
+    auto ReHashBackend = Instance->getHashingBackend();
+    std::set<std::string> AllOutputs;
+    llvm::for_each(HashBackend->outputFiles(), [&](StringRef F) {
+      AllOutputs.insert(F.str());
+    });
+    llvm::for_each(ReHashBackend->outputFiles(), [&](StringRef F) {
+      AllOutputs.insert(F.str());
+    });
+
+    DiagnosticEngine &diags = Instance->getDiags();
+    for (auto &Filename : AllOutputs) {
+      auto O1 = HashBackend->getHashValueForFile(Filename);
+      if (!O1) {
+        diags.diagnose(SourceLoc(), diag::error_output_missing, Filename,
+                       /*SecondRun=*/false);
+        HadError = true;
+        continue;
+      }
+      auto O2 = ReHashBackend->getHashValueForFile(Filename);
+      if (!O2) {
+        diags.diagnose(SourceLoc(), diag::error_output_missing, Filename,
+                       /*SecondRun=*/true);
+        HadError = true;
+        continue;
+      }
+      if (*O1 != *O2) {
+        diags.diagnose(SourceLoc(), diag::error_nondeterministic_output,
+                       Filename, *O1, *O2);
+        HadError = true;
+        continue;
+      }
+      diags.diagnose(SourceLoc(), diag::matching_output_produced, Filename,
+                     *O1);
+    }
+  }
+
   auto r = finishDiagProcessing(HadError ? 1 : ReturnValue, verifierEnabled);
   if (auto *StatsReporter = Instance->getStatsReporter())
     StatsReporter->noteCurrentProcessExitStatus(r);

test/Frontend/output_determinism_check.swift

@@ -0,0 +1,12 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT
+// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT
+
+/// object files are "not" deterministic because the second run going to match the mod hash and skip code generation.
+// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT
+
+// MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule'
+// SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib'
+// OBJECT_OUTPUT: error: output file '{{.*}}{{/|\\}}test.o' is missing from second run
+
+public var x = 1