Add an option to verify the deterministic output of swift compiler

Add option to test swift compiler determinism by running the job twice
and compare the output produced.

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -449,6 +449,16 @@ REMARK(interface_file_backup_used,none,
 
 WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef))
 
+// Output deterministic check
+ERROR(error_nondeterministic_output,none,
+      "output file '%0' is not deterministic: hash value '%1' vs '%2'",
+      (StringRef, StringRef, StringRef))
+ERROR(error_output_missing,none,
+      "output file '%0' is missing from %select{first|second}1 run",
+      (StringRef, /*SecondRun=*/bool))
+REMARK(matching_output_produced,none,
+      "produced matching output file '%0': hash '%1'", (StringRef, StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/Frontend/FrontendOptions.h

@@ -489,6 +489,9 @@ class FrontendOptions {
   /// to encode the actual paths into the .swiftmodule file.
   PathObfuscator serializedPathObfuscator;
 
+  /// Whether to run the job twice to check determinism.
+  bool DeterministicCheck = false;
+
   /// Avoid printing actual module content into the ABI descriptor file.
   /// This should only be used as a workaround when emitting ABI descriptor files
   /// crashes the compiler.

include/swift/Option/FrontendOptions.td

@@ -1135,6 +1135,10 @@ def enable_emit_generic_class_ro_t_list :
   HelpText<"Enable emission of a section with references to class_ro_t of "
            "generic class patterns">;
 
+def enable_swift_deterministic_check :
+  Flag<["-"], "enable-swift-deterministic-check">,
+  HelpText<"Check swift compiler output determinisim by run it twice">;
+
 def experimental_spi_only_imports :
   Flag<["-"], "experimental-spi-only-imports">,
   HelpText<"Enable use of @_spiOnly imports">;

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -338,6 +338,7 @@ bool ArgsToFrontendOptionsConverter::convert(
     Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second);
   }
   Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor);
+  Opts.DeterministicCheck = Args.hasArg(OPT_enable_swift_deterministic_check);
   return false;
 }
 

lib/FrontendTool/FrontendTool.cpp

@@ -71,17 +71,23 @@
 
 #include "clang/Lex/Preprocessor.h"
 
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Option/OptTable.h"
+#include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/HashingOutputBackend.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/FileSystem.h"
 
@@ -2345,6 +2351,38 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     PDC.setSuppressOutput(true);
   }
 
+  using HashBackendTy = llvm::vfs::HashingOutputBackend<llvm::BLAKE3>;
+  llvm::IntrusiveRefCntPtr<HashBackendTy> HashBackend, ReHashBackend;
+  int ReturnValueTest = 0;
+  if (Invocation.getFrontendOptions().DeterministicCheck) {
+    std::unique_ptr<CompilerInstance> VerifyInstance =
+        std::make_unique<CompilerInstance>();
+    std::string InstanceSetupError;
+    // This should not fail because it passed already.
+    (void)VerifyInstance->setup(Invocation, InstanceSetupError);
+
+    // Create a mirroring outputbackend to produce hash for output files.
+    // We cannot skip disk here since swift compiler is expecting to read back
+    // some output file in later stages.
+    // TODO: Implement diagnostics determinism check. We didn't setup any diag
+    // consumer currently.
+    HashBackend = llvm::makeIntrusiveRefCnt<HashBackendTy>();
+    auto VerifyBackend = llvm::vfs::makeMirroringOutputBackend(
+        llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>(),
+        HashBackend);
+    VerifyInstance->setOutputBackend(VerifyBackend);
+    // Run the first time without observer and discard return value;
+    (void)performCompile(*VerifyInstance, ReturnValueTest,
+                         /*observer*/ nullptr);
+
+    // Mirror the output to disk on second run.
+    ReHashBackend = llvm::makeIntrusiveRefCnt<HashBackendTy>();
+    auto FinalBackend = llvm::vfs::makeMirroringOutputBackend(
+        llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>(),
+        ReHashBackend);
+    Instance->setOutputBackend(FinalBackend);
+  }
+
   int ReturnValue = 0;
   bool HadError = performCompile(*Instance, ReturnValue, observer);
 
@@ -2359,6 +2397,43 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     }
   }
 
+  if (Invocation.getFrontendOptions().DeterministicCheck) {
+    // Collect all output files.
+    std::set<std::string> AllOutputs;
+    llvm::for_each(HashBackend->outputFiles(), [&](StringRef F) {
+      AllOutputs.insert(F.str());
+    });
+    llvm::for_each(ReHashBackend->outputFiles(), [&](StringRef F) {
+      AllOutputs.insert(F.str());
+    });
+
+    DiagnosticEngine &diags = Instance->getDiags();
+    for (auto &Filename : AllOutputs) {
+      auto O1 = HashBackend->getHashValueForFile(Filename);
+      if (!O1) {
+        diags.diagnose(SourceLoc(), diag::error_output_missing, Filename,
+                       /*SecondRun=*/false);
+        HadError = true;
+        continue;
+      }
+      auto O2 = ReHashBackend->getHashValueForFile(Filename);
+      if (!O2) {
+        diags.diagnose(SourceLoc(), diag::error_output_missing, Filename,
+                       /*SecondRun=*/true);
+        HadError = true;
+        continue;
+      }
+      if (*O1 != *O2) {
+        diags.diagnose(SourceLoc(), diag::error_nondeterministic_output,
+                       Filename, *O1, *O2);
+        HadError = true;
+        continue;
+      }
+      diags.diagnose(SourceLoc(), diag::matching_output_produced, Filename,
+                     *O1);
+    }
+  }
+
   auto r = finishDiagProcessing(HadError ? 1 : ReturnValue, verifierEnabled);
   if (auto *StatsReporter = Instance->getStatsReporter())
     StatsReporter->noteCurrentProcessExitStatus(r);

test/Frontend/output_determinism_check.swift

@@ -0,0 +1,10 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT
+
+/// object files are "not" deterministic because the second run going to match the mod hash and skip code generation.
+// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-swift-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT
+
+// MODULE_OUTPUT: remark: produced matching output file
+// OBJECT_OUTPUT: error: output file '{{.*}}{{/|\\}}test.o' is missing from second run
+
+public var x = 1