[SwiftScan] Add SwiftScan APIs to replay the cache output

Add new C APIs to libSwiftScan for cache querying and cache replays.

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -25,7 +25,7 @@
 /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions.
 /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 5
+#define SWIFTSCAN_VERSION_MINOR 6
 
 SWIFTSCAN_BEGIN_DECLS
 
@@ -441,6 +441,51 @@ typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t;
 /// ActionCache.
 typedef struct swiftscan_cas_s *swiftscan_cas_t;
 
+/// Opaque container for a cached compilation.
+typedef struct swiftscan_cached_compilation_s *swiftscan_cached_compilation_t;
+
+/// Opaque type for a cache replay instance.
+typedef struct swiftscan_cache_replay_instance_s
+    *swiftscan_cache_replay_instance_t;
+
+/// Opaque type for a cancellation token for async cache operations.
+typedef struct swiftscan_cache_cancellation_token_s
+    *swiftscan_cache_cancellation_token_t;
+
+/// Enum types for output types for cache key computation.
+/// List should contain all the output file types, including supplemententary
+/// outputs, except diagnostics outputs, which are covered by cached diagnostic
+/// entry.
+typedef enum {
+  SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_HEADER = 6,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFT_SOURCE_INFO = 7,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFT_MODULE_DOC = 8,
+  SWIFTSCAN_OUTPUT_TYPE_DEPENDENCIES = 9,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFT_DEPS = 10,
+  SWIFTSCAN_OUTPUT_TYPE_MODULE_TRACE = 11,
+  SWIFTSCAN_OUTPUT_TYPE_TBD = 12,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFT_MODULE_SUMMARY = 13,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFT_ABI_DESCRIPTOR = 14,
+  SWIFTSCAN_OUTPUT_TYPE_CONST_VALUE = 15,
+  SWIFTSCAN_OUTPUT_TYPE_MODULE_SEMANTIC_INFO = 16,
+  SWIFTSCAN_OUTPUT_TYPE_YAML_OPT_RECORD = 17,
+  SWIFTSCAN_OUTPUT_TYPE_BITSTREAM_OPT_RECORD = 18,
+  SWIFTSCAN_OUTPUT_TYPE_CACHED_DIAGNOSTICS = 19
+} swiftscan_output_kind_t;
+
+/// Enum types for cache result lookup or replay.
+typedef enum {
+  SWIFTSCAN_CACHE_RESULT_SUCCESS = 0,
+  SWIFTSCAN_CACHE_RESULT_NOT_FOUND = 1,
+  SWIFTSCAN_CACHE_RESULT_ERROR = 2,
+} swiftscan_cache_result_t;
+
 /// Create a \c CASOptions for creating CAS inside scanner specified.
 SWIFTSCAN_PUBLIC swiftscan_cas_options_t swiftscan_cas_options_create(void);
 
@@ -492,6 +537,129 @@ SWIFTSCAN_PUBLIC swiftscan_string_ref_t
 swiftscan_cache_compute_key(swiftscan_cas_t cas, int argc, const char **argv,
                             const char *input, swiftscan_string_ref_t *error);
 
+/// Query the result of the compilation using the output cache key. \c globally
+/// suggests if the lookup should check remote cache if such operation exists.
+/// Returns the CachedCompilation of the result if found, or nullptr if output
+/// is not found or an error occurs. When an error occurs, the error message is
+/// returned via \c error parameter and its caller needs to free the message
+/// using `swiftscan_string_dispose`. The returned CachedCompilation needs to be
+/// freed via `swiftscan_cached_compilation_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_cached_compilation_t
+swiftscan_cache_query(swiftscan_cas_t cas, const char *key, bool globally,
+                      swiftscan_string_ref_t *error);
+
+/// Async version of `swiftscan_cache_query` where result is returned via
+/// callback. Both cache_result enum and CachedCompilation will be provided to
+/// callback. \c ctx is an opaque value that passed to the callback and \c
+/// swiftscan_cache_cancellation_token_t will return an token that can be used
+/// to cancel the async operation. The token needs to be freed by caller using
+/// `swiftscan_cache_cancellation_token_dispose`.
+SWIFTSCAN_PUBLIC void swiftscan_cache_query_async(
+    swiftscan_cas_t cas, const char *key, bool globally, void *ctx,
+    void (*callback)(void *ctx, swiftscan_cache_result_t,
+                     swiftscan_cached_compilation_t,
+                     swiftscan_string_ref_t error),
+    swiftscan_cache_cancellation_token_t *);
+
+/// Dispose a CachedCompilation.
+SWIFTSCAN_PUBLIC void
+    swiftscan_cached_compilation_dispose(swiftscan_cached_compilation_t);
+
+/// Download and materialized the CASObject referenced by the CASID in the local
+/// CAS if needed from a remote CAS.
+/// If the return value is SWIFTSCAN_CACHE_RESULT_ERROR, the error message is
+/// returned via \c error parameter and its caller needs to free the message
+/// using `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_cache_result_t
+swiftscan_cache_load_object(swiftscan_cas_t cas, swiftscan_cached_compilation_t,
+                            swiftscan_string_ref_t *error);
+
+/// Async version of `swiftscan_cache_load_object` where result is returned via
+/// callback. \c ctx is an opaque value that passed to the callback and
+/// \c swiftscan_cache_cancellation_token_t will return an token that can be
+/// used to cancel the async operation. The token needs to be freed by caller
+/// using `swiftscan_cache_cancellation_token_dispose`.
+SWIFTSCAN_PUBLIC void swiftscan_cache_load_object_async(
+    swiftscan_cas_t cas, swiftscan_cached_compilation_t, void *ctx,
+    void (*callback)(void *ctx, swiftscan_cache_result_t,
+                     swiftscan_string_ref_t error),
+    swiftscan_cache_cancellation_token_t *);
+
+/// Check if CachedCompilation is materialized locally and can be accessed
+/// without downloading.
+SWIFTSCAN_PUBLIC bool
+    swiftscan_cache_compilation_is_loaded(swiftscan_cached_compilation_t);
+
+/// Query the number of outputs from a cached compilation. The cached
+/// compilation needs to be loaded, otherwise will return error. The error
+/// message is returned via \c error parameter and its caller needs to free the
+/// message using `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC unsigned
+swiftscan_cache_get_num_cached_outputs(swiftscan_cached_compilation_t,
+                                       swiftscan_string_ref_t *error);
+
+/// Query if the kind of otuput exists in a cached compilation. The cached
+/// compilation needs to be loaded, otherwise will return error. The error
+/// message is returned via \c error parameter and its caller needs to free the
+/// message using `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC bool
+swiftscan_cache_compilation_has_output_kind(swiftscan_cached_compilation_t,
+                                            swiftscan_output_kind_t,
+                                            swiftscan_string_ref_t *error);
+
+/// Make the cache compilation available globally. \c callback will be called
+/// on completion.
+/// \c swiftscan_cache_cancellation_token_t will return an token that can be
+/// used to cancel the async operation. The token needs to be freed by caller
+/// using `swiftscan_cache_cancellation_token_dispose`.
+SWIFTSCAN_PUBLIC void swiftscan_cache_make_global_async(
+    swiftscan_cas_t cas, const char *key, void *ctx,
+    void (*callback)(void *ctx, swiftscan_string_ref_t error),
+    swiftscan_cache_cancellation_token_t *);
+
+/// Cancel the async cache action that is associated with token.
+SWIFTSCAN_PUBLIC void
+    swiftscan_cache_action_cancel(swiftscan_cache_cancellation_token_t);
+
+/// Dispose the cancellation token.
+SWIFTSCAN_PUBLIC void swiftscan_cache_cancellation_token_dispose(
+    swiftscan_cache_cancellation_token_t);
+
+/// Create a swift cached compilation replay instance with its command-line
+/// invocation. Return nullptr when errors occurs and the error message is
+/// returned via \c error parameter and its caller needs to free the message
+/// using `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_cache_replay_instance_t
+swiftscan_cache_create_replay_instance(int argc, const char **argv,
+                                       swiftscan_string_ref_t *error);
+
+/// Dispose swift cached compilation replay instance.
+SWIFTSCAN_PUBLIC void
+    swiftscan_cache_replay_instance_dispose(swiftscan_cache_replay_instance_t);
+
+/// Replay the cached compilation using cached compliation replay instance.
+/// If the return value is SWIFTSCAN_CACHE_RESULT_ERROR, the error message is
+/// returned via \c error parameter and its caller needs to free the message
+/// using `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_cache_result_t swiftscan_cache_replay_compilation(
+    swiftscan_cas_t cas, swiftscan_cache_replay_instance_t,
+    swiftscan_cached_compilation_t, swiftscan_string_ref_t *error);
+
+/// Async replay the cached compilation. Callback function will be provided
+/// by the cache replay result, and its stdout, stderr, and any error message.
+/// \c ctx is an opaque value that passed to the callback and
+/// \c swiftscan_cache_cancellation_token_t will return an token that can be
+/// used to cancel the async operation. The token needs to be freed by caller
+/// using `swiftscan_cache_cancellation_token_dispose`.
+SWIFTSCAN_PUBLIC void swiftscan_cache_replay_compilation_async(
+    swiftscan_cas_t cas, swiftscan_cache_replay_instance_t,
+    swiftscan_cached_compilation_t, void *ctx,
+    void (*callback)(void *ctx, swiftscan_cache_result_t,
+                     swiftscan_string_ref_t std_out,
+                     swiftscan_string_ref_t std_err,
+                     swiftscan_string_ref_t error),
+    swiftscan_cache_cancellation_token_t *);
+
 //===----------------------------------------------------------------------===//
 
 SWIFTSCAN_END_DECLS

lib/DriverTool/swift_cache_tool_main.cpp

@@ -50,9 +50,8 @@ enum class SwiftCacheToolAction {
 
 struct OutputEntry {
   std::string InputPath;
-  std::string OutputPath;
-  std::string OutputKind;
   std::string CacheKey;
+  std::vector<std::pair<std::string, std::string>> Outputs;
 };
 
 enum ID {
@@ -285,44 +284,41 @@ int SwiftCacheToolInvocation::printOutputKeys() {
 
   std::vector<OutputEntry> OutputKeys;
   bool hasError = false;
-  auto addOutputKey = [&](StringRef InputPath, file_types::ID OutputKind,
-                          StringRef OutputPath) {
+  auto addFromInputFile = [&](const InputFile &Input) {
+    auto InputPath = Input.getFileName();
     auto OutputKey =
         createCompileJobCacheKeyForOutput(CAS, *BaseKey, InputPath);
     if (!OutputKey) {
-      llvm::errs() << "cannot create cache key for " << OutputPath << ": "
+      llvm::errs() << "cannot create cache key for " << InputPath << ": "
                    << toString(OutputKey.takeError()) << "\n";
       hasError = true;
     }
     OutputKeys.emplace_back(
-        OutputEntry{InputPath.str(), OutputPath.str(),
-                    file_types::getTypeName(OutputKind).str(),
-                    CAS.getID(*OutputKey).toString()});
-  };
-  auto addFromInputFile = [&](const InputFile &Input) {
-    auto InputPath = Input.getFileName();
+        OutputEntry{InputPath, CAS.getID(*OutputKey).toString(), {}});
+    auto &Outputs = OutputKeys.back().Outputs;
     if (!Input.outputFilename().empty())
-      addOutputKey(InputPath,
-                   Invocation.getFrontendOptions()
-                       .InputsAndOutputs.getPrincipalOutputType(),
-                   Input.outputFilename());
+      Outputs.emplace_back(file_types::getTypeName(
+                               Invocation.getFrontendOptions()
+                                   .InputsAndOutputs.getPrincipalOutputType()),
+                           Input.outputFilename());
     Input.getPrimarySpecificPaths()
         .SupplementaryOutputs.forEachSetOutputAndType(
             [&](const std::string &File, file_types::ID ID) {
               // Dont print serialized diagnostics.
               if (file_types::isProducedFromDiagnostics(ID))
                 return;
-
-              addOutputKey(InputPath, ID, File);
+              Outputs.emplace_back(file_types::getTypeName(ID), File);
             });
   };
   llvm::for_each(
       Invocation.getFrontendOptions().InputsAndOutputs.getAllInputs(),
       addFromInputFile);
 
   // Add diagnostics file.
-  addOutputKey("<cached-diagnostics>", file_types::ID::TY_CachedDiagnostics,
-               "<cached-diagnostics>");
+  if (!OutputKeys.empty())
+    OutputKeys.front().Outputs.emplace_back(
+        file_types::getTypeName(file_types::ID::TY_CachedDiagnostics),
+        "<cached-diagnostics>");
 
   if (hasError)
     return 1;
@@ -331,10 +327,16 @@ int SwiftCacheToolInvocation::printOutputKeys() {
   Out.array([&] {
     for (const auto &E : OutputKeys) {
       Out.object([&] {
-        Out.attribute("OutputPath", E.OutputPath);
-        Out.attribute("OutputKind", E.OutputKind);
         Out.attribute("Input", E.InputPath);
         Out.attribute("CacheKey", E.CacheKey);
+        Out.attributeArray("Outputs", [&] {
+          for (const auto &OutEntry : E.Outputs) {
+            Out.object([&] {
+              Out.attribute("Kind", OutEntry.first);
+              Out.attribute("Path", OutEntry.second);
+            });
+          }
+        });
       });
     }
   });

lib/Frontend/CASOutputBackends.cpp

@@ -157,6 +157,11 @@ void SwiftCASOutputBackend::Implementation::initBackend(
             });
   };
   llvm::for_each(InputsAndOutputs.getAllInputs(), addInput);
+
+  // FIXME: The current diagnostics is associated with the first input file.
+  OutputToInputMap.insert(
+      {"<cached-diagnostics>",
+       {InputsAndOutputs.getAllInputs()[0], file_types::TY_CachedDiagnostics}});
 }
 
 Error SwiftCASOutputBackend::Implementation::storeImpl(

lib/Frontend/CachedDiagnostics.cpp

@@ -772,10 +772,13 @@ CachingDiagnosticsProcessor::CachingDiagnosticsProcessor(
     }
 
     StringRef Content = Compression.empty() ? Output : toStringRef(Compression);
-    // Store CachedDiagnostics in the CAS/Cache. There is no real associated
-    // inputs.
+    // Store CachedDiagnostics in the CAS/Cache.
+    // FIXME: Currently, it is associated with first input file.
     auto Err = Instance.getCASOutputBackend().storeCachedDiagnostics(
-        "<cached-diagnostics>", Content);
+        Instance.getInvocation()
+            .getFrontendOptions()
+            .InputsAndOutputs.getFilenameOfFirstInput(),
+        Content);
 
     if (Err) {
       Instance.getDiags().diagnose(SourceLoc(), diag::error_cas,

lib/Frontend/CachingUtils.cpp

@@ -209,19 +209,15 @@ bool replayCachedCompilerOutputs(
               Outputs.try_emplace(ID, File);
             });
 
-    // Nothing to replay.
-    if (Outputs.empty())
-      return;
+    // Add cached diagnostic entry for lookup. Output path doesn't matter here.
+    Outputs.try_emplace(file_types::ID::TY_CachedDiagnostics,
+                        "<cached-diagnostics>");
 
     return replayOutputsForInputFile(InputPath, Outputs);
   };
 
   llvm::for_each(InputsAndOutputs.getAllInputs(), replayOutputFromInput);
 
-  replayOutputsForInputFile(
-      "<cached-diagnostics>",
-      {{file_types::ID::TY_CachedDiagnostics, "<cached-diagnostics>"}});
-
   if (!CanReplayAllOutput)
     return false;
 

lib/FrontendTool/FrontendTool.cpp

@@ -2107,9 +2107,8 @@ class PrettyStackTraceFrontend : public llvm::PrettyStackTraceEntry {
   };
 };
 
-int swift::performFrontend(ArrayRef<const char *> Args,
-                           const char *Argv0, void *MainAddr,
-                           FrontendObserver *observer) {
+int swift::performFrontend(ArrayRef<const char *> Args, const char *Argv0,
+                           void *MainAddr, FrontendObserver *observer) {
   INITIALIZE_LLVM();
   llvm::setBugReportMsg(SWIFT_CRASH_BUG_REPORT_MESSAGE "\n");
   llvm::EnablePrettyStackTraceOnSigInfoForThisThread();

test/CAS/Inputs/ExtractOutputKey.py

@@ -10,8 +10,9 @@
 
 
 with open(input_json, 'r') as file:
-    outputs = json.load(file)
-    for output in outputs:
-        if output['OutputPath'] != output_path:
-            continue
-        print(output['CacheKey'])
+    entries = json.load(file)
+    for entry in entries:
+        for output in entry["Outputs"]:
+            if output['Path'] != output_path:
+                continue
+            print(entry['CacheKey'])