Mirror outputs into CAS using CASOutputBackend

When `-enable-cas` option is used, mirror the output into CAS so it can
be fetched from CAS with the cache key in the future.

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -485,6 +485,10 @@ ERROR(error_output_missing,none,
 REMARK(matching_output_produced,none,
       "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef))
 
+// Caching related diagnostics
+ERROR(error_caching_no_cas_fs, none,
+    "caching is enabled without -cas-fs option, input is not immutable", ())
+
 // CAS related diagnostics
 ERROR(error_create_cas, none, "failed to create CAS '%0' (%1)", (StringRef, StringRef))
 ERROR(error_invalid_cas_id, none, "invalid CASID '%0' (%1)", (StringRef, StringRef))

include/swift/Frontend/CachingUtils.h

@@ -0,0 +1,50 @@
+//===--- CachingUtils.h -----------------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_FRONTEND_CACHINGUTILS_H
+#define SWIFT_FRONTEND_CACHINGUTILS_H
+
+#include "swift/Frontend/FrontendInputsAndOutputs.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include <memory>
+
+namespace swift {
+
+/// Create a swift caching output backend that stores the output from
+/// compiler into a CAS.
+llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend>
+createSwiftCachingOutputBackend(
+    llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
+    llvm::cas::ObjectRef BaseKey,
+    const FrontendInputsAndOutputs &InputsAndOutputs);
+
+/// Load the cached compile result from cache.
+std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
+    llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
+    DiagnosticEngine &Diag, llvm::StringRef CacheKey,
+    llvm::StringRef Filename = "");
+
+/// Store compiler output.
+llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
+                                      llvm::cas::ActionCache &Cache,
+                                      StringRef Path, StringRef Bytes,
+                                      llvm::cas::ObjectRef BaseKey,
+                                      StringRef CorrespondingInput,
+                                      file_types::ID OutputKind);
+
+}
+
+#endif

include/swift/Frontend/Frontend.h

@@ -633,6 +633,9 @@ class CompilerInstance {
   /// i.e. if it can be found.
   bool canImportCxxShim() const;
 
+  /// Whether this compiler instance supports caching.
+  bool supportCaching() const;
+
   /// Gets the SourceFile which is the primary input for this CompilerInstance.
   /// \returns the primary SourceFile, or nullptr if there is no primary input;
   /// if there are _multiple_ primary inputs, fails with an assertion.

include/swift/Frontend/FrontendOptions.h

@@ -445,6 +445,9 @@ class FrontendOptions {
   /// \return true if the given action requires input files to be provided.
   static bool doesActionPerformEndOfPipelineActions(ActionType action);
 
+  /// \return true if the given action supports caching.
+  static bool supportCompilationCaching(ActionType action);
+
   /// Return a hash code of any components from these options that should
   /// contribute to a Swift Bridging PCH hash.
   llvm::hash_code getPCHHashComponents() const {

include/swift/Option/Options.td

@@ -1674,7 +1674,7 @@ def use_interface_for_module: Separate<["-", "--"], "use-interface-for-module">,
 def cache_tool_action: JoinedOrSeparate<["-"], "cache-tool-action">,
   Flags<[NoDriverOption, SwiftCacheToolOption]>,
   HelpText<"Swift Cache Tool Action Kind">,
-  MetaVarName<"<print-base-key|print-output-keys>">;
+  MetaVarName<"<print-base-key|print-output-keys|validate-outputs>">;
 
 // ONLY SUPPORTED IN NEW DRIVER
 
@@ -1822,6 +1822,10 @@ def cas_path: Separate<["-"], "cas-path">,
   Flags<[FrontendOption, NewDriverOnlyOption, SwiftCacheToolOption]>,
   HelpText<"Path to CAS">, MetaVarName<"<path>">;
 
+def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-for-testing">,
+  Flags<[FrontendOption, HelpHidden, NoDriverOption]>,
+  HelpText<"Allow compilation caching with unstable inputs for testing purpose">;
+
 // END ONLY SUPPORTED IN NEW DRIVER
 
 def enable_cas: Flag<["-"], "enable-cas">,

lib/DriverTool/swift_cache_tool_main.cpp

@@ -18,6 +18,7 @@
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/LLVMInitialize.h"
 #include "swift/Basic/Version.h"
+#include "swift/Frontend/CachingUtils.h"
 #include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
@@ -27,6 +28,7 @@
 #include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include <memory>
 
 using namespace swift;
@@ -38,7 +40,15 @@ namespace {
 enum class SwiftCacheToolAction {
   Invalid,
   PrintBaseKey,
-  PrintOutputKeys
+  PrintOutputKeys,
+  ValidateOutputs
+};
+
+struct OutputEntry {
+  std::string InputPath;
+  std::string OutputPath;
+  std::string OutputKind;
+  std::string CacheKey;
 };
 
 class SwiftCacheToolInvocation {
@@ -48,6 +58,7 @@ class SwiftCacheToolInvocation {
   PrintingDiagnosticConsumer PDC;
   std::string MainExecutablePath;
   std::string CASPath;
+  std::vector<std::string> Inputs;
   std::vector<std::string> FrontendArgs;
   SwiftCacheToolAction ActionKind = SwiftCacheToolAction::Invalid;
 
@@ -84,17 +95,19 @@ class SwiftCacheToolInvocation {
     CASPath =
         ParsedArgs.getLastArgValue(OPT_cas_path, getDefaultOnDiskCASPath());
 
+    Inputs = ParsedArgs.getAllArgValues(OPT_INPUT);
     FrontendArgs = ParsedArgs.getAllArgValues(OPT__DASH_DASH);
     if (auto *A = ParsedArgs.getLastArg(OPT_cache_tool_action))
       ActionKind =
           llvm::StringSwitch<SwiftCacheToolAction>(A->getValue())
               .Case("print-base-key", SwiftCacheToolAction::PrintBaseKey)
               .Case("print-output-keys", SwiftCacheToolAction::PrintOutputKeys)
+              .Case("validate-outputs", SwiftCacheToolAction::ValidateOutputs)
               .Default(SwiftCacheToolAction::Invalid);
 
     if (ActionKind == SwiftCacheToolAction::Invalid) {
       llvm::errs() << "Invalid option specified for -cache-tool-action: "
-                   << "use print-base-key|print-output-keys\n";
+                   << "use print-base-key|print-output-keys|validate-outputs\n";
       return 1;
     }
 
@@ -107,6 +120,8 @@ class SwiftCacheToolInvocation {
       return printBaseKey();
     case SwiftCacheToolAction::PrintOutputKeys:
       return printOutputKeys();
+    case SwiftCacheToolAction::ValidateOutputs:
+      return validateOutputs();
     case SwiftCacheToolAction::Invalid:
       llvm::report_fatal_error("invalid action");
     }
@@ -186,6 +201,7 @@ class SwiftCacheToolInvocation {
   }
 
   int printOutputKeys();
+  int validateOutputs();
 };
 
 } // end anonymous namespace
@@ -199,12 +215,6 @@ int SwiftCacheToolInvocation::printOutputKeys() {
   if (!BaseKey)
     return 1;
 
-  struct OutputEntry {
-    std::string InputPath;
-    std::string OutputPath;
-    std::string OutputKind;
-    std::string CacheKey;
-  };
   std::vector<OutputEntry> OutputKeys;
   bool hasError = false;
   auto addOutputKey = [&](StringRef InputPath, file_types::ID OutputKind,
@@ -260,6 +270,55 @@ int SwiftCacheToolInvocation::printOutputKeys() {
   return 0;
 }
 
+int SwiftCacheToolInvocation::validateOutputs() {
+  auto DB = llvm::cas::createOnDiskUnifiedCASDatabases(CASPath);
+  if (!DB)
+    report_fatal_error(DB.takeError());
+
+  PrintingDiagnosticConsumer PDC;
+  Instance.getDiags().addConsumer(PDC);
+
+  auto validateCacheKeysFromFile = [&](const std::string &Path) {
+    auto JSONContent = llvm::MemoryBuffer::getFile(Path);
+    if (!JSONContent) {
+      llvm::errs() << "failed to read " << Path << ": "
+                   << JSONContent.getError().message() << "\n";
+      return true;
+    }
+    auto JSONValue = llvm::json::parse((*JSONContent)->getBuffer());
+    if (!JSONValue) {
+      llvm::errs() << "failed to parse " << Path << ": "
+                   << toString(JSONValue.takeError()) << "\n";
+      return true;
+    }
+
+    auto Keys = JSONValue->getAsArray();
+    if (!Keys) {
+      llvm::errs() << "invalid keys format in " << Path << "\n";
+      return true;
+    }
+
+    for (const auto& Entry : *Keys) {
+      if (auto *Obj = Entry.getAsObject()) {
+        if (auto Key = Obj->getString("CacheKey")) {
+          if (auto Buffer = loadCachedCompileResultFromCacheKey(
+                  *DB->first, *DB->second, Instance.getDiags(), *Key))
+            continue;
+          llvm::errs() << "failed to find output for cache key " << *Key
+                       << "\n";
+          return true;
+        }
+      }
+      llvm::errs() << "can't read cache key from " << Path << "\n";
+      return true;
+    }
+
+    return false;
+  };
+
+  return llvm::any_of(Inputs, validateCacheKeysFromFile);
+}
+
 int swift_cache_tool_main(ArrayRef<const char *> Args, const char *Argv0,
                           void *MainAddr) {
   INITIALIZE_LLVM();

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -355,6 +355,13 @@ bool ArgsToFrontendOptionsConverter::convert(
   Opts.CASPath =
       Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
   Opts.CASFSRootID = Args.getLastArgValue(OPT_cas_fs);
+  if (Opts.EnableCAS && Opts.CASFSRootID.empty() &&
+      FrontendOptions::supportCompilationCaching(Opts.RequestedAction)) {
+    if (!Args.hasArg(OPT_allow_unstable_cache_key_for_testing)) {
+        Diags.diagnose(SourceLoc(), diag::error_caching_no_cas_fs);
+        return true;
+    }
+  }
 
   return false;
 }

lib/Frontend/CMakeLists.txt

@@ -3,6 +3,7 @@ add_swift_host_library(swiftFrontend STATIC
   ArgsToFrontendInputsConverter.cpp
   ArgsToFrontendOptionsConverter.cpp
   ArgsToFrontendOutputsConverter.cpp
+  CachingUtils.cpp
   CompileJobCacheKey.cpp
   CompilerInvocation.cpp
   DependencyVerifier.cpp