Merge pull request #7304 from akyrtzi/akyrtzi/pr/major-libclang-cas-queries

[stable/20230725][CAS/libclang] Add libclang APIs for handling and replaying cached compilations

Author: akyrtzi

clang/include/clang-c/CAS.h

@@ -20,6 +20,7 @@
 #ifndef LLVM_CLANG_C_CAS_H
 #define LLVM_CLANG_C_CAS_H
 
+#include "clang-c/CXErrorCode.h"
 #include "clang-c/CXString.h"
 #include "clang-c/Platform.h"
 
@@ -53,6 +54,23 @@ typedef struct CXOpaqueCASObjectStore *CXCASObjectStore;
  */
 typedef struct CXOpaqueCASActionCache *CXCASActionCache;
 
+typedef struct CXOpaqueCASObject *CXCASObject;
+
+/**
+ * Result of \c clang_experimental_cas_getCachedCompilation.
+ */
+typedef struct CXOpaqueCASCachedCompilation *CXCASCachedCompilation;
+
+/**
+ * Result of \c clang_experimental_cas_replayCompilation.
+ */
+typedef struct CXOpaqueCASReplayResult *CXCASReplayResult;
+
+/**
+ * Used for cancelling asynchronous actions.
+ */
+typedef struct CXOpaqueCASCancellationToken *CXCASCancellationToken;
+
 /**
  * Create a \c CXCASOptions object.
  */
@@ -99,6 +117,181 @@ clang_experimental_cas_Databases_create(CXCASOptions Opts, CXString *Error);
  */
 CINDEX_LINKAGE void clang_experimental_cas_Databases_dispose(CXCASDatabases);
 
+/**
+ * Loads an object using its printed \p CASID.
+ *
+ * \param CASID The printed CASID string for the object.
+ * \param[out] OutError The error object to pass back to client (if any).
+ * If non-null the object must be disposed using \c clang_Error_dispose.
+ *
+ * \returns The resulting object, or null if the object was not found or an
+ * error occurred. The object should be disposed using
+ * \c clang_experimental_cas_CASObject_dispose.
+ */
+CINDEX_LINKAGE CXCASObject clang_experimental_cas_loadObjectByString(
+    CXCASDatabases, const char *CASID, CXError *OutError);
+
+/**
+ * Asynchronous version of \c clang_experimental_cas_loadObjectByString.
+ *
+ * \param CASID The printed CASID string for the object.
+ * \param Ctx opaque value to pass to the callback.
+ * \param Callback receives a \c CXCASObject, or \c CXError if an error occurred
+ * or both NULL if the object was not found or the call was cancelled.
+ * The objects should be disposed with
+ * \c clang_experimental_cas_CASObject_dispose or \c clang_Error_dispose.
+ * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that
+ * can be used to cancel the call using
+ * \c clang_experimental_cas_CancellationToken_cancel. The object should be
+ * disposed using \c clang_experimental_cas_CancellationToken_dispose.
+ */
+CINDEX_LINKAGE void clang_experimental_cas_loadObjectByString_async(
+    CXCASDatabases, const char *CASID, void *Ctx,
+    void (*Callback)(void *Ctx, CXCASObject, CXError),
+    CXCASCancellationToken *OutToken);
+
+/**
+ * Dispose of a \c CXCASObject object.
+ */
+CINDEX_LINKAGE void clang_experimental_cas_CASObject_dispose(CXCASObject);
+
+/**
+ * Looks up a cache key and returns the associated set of compilation output IDs
+ *
+ * \param CacheKey The printed compilation cache key string.
+ * \param Globally if true it is a hint to the underlying CAS implementation
+ * that the lookup is profitable to be done on a distributed caching level, not
+ * just locally.
+ * \param[out] OutError The error object to pass back to client (if any).
+ * If non-null the object must be disposed using \c clang_Error_dispose.
+ *
+ * \returns The resulting object, or null if the cache key was not found or an
+ * error occurred. The object should be disposed using
+ * \c clang_experimental_cas_CachedCompilation_dispose.
+ */
+CINDEX_LINKAGE CXCASCachedCompilation
+clang_experimental_cas_getCachedCompilation(CXCASDatabases,
+                                            const char *CacheKey, bool Globally,
+                                            CXError *OutError);
+
+/**
+ * Asynchronous version of \c clang_experimental_cas_getCachedCompilation.
+ *
+ * \param CacheKey The printed compilation cache key string.
+ * \param Globally if true it is a hint to the underlying CAS implementation
+ * that the lookup is profitable to be done on a distributed caching level, not
+ * just locally.
+ * \param Ctx opaque value to pass to the callback.
+ * \param Callback receives a \c CXCASCachedCompilation, or \c CXError if an
+ * error occurred or both NULL if the object was not found or the call was
+ * cancelled. The objects should be disposed with
+ * \c clang_experimental_cas_CachedCompilation_dispose or \c clang_Error_dispose
+ * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that
+ * can be used to cancel the call using
+ * \c clang_experimental_cas_CancellationToken_cancel. The object should be
+ * disposed using \c clang_experimental_cas_CancellationToken_dispose.
+ */
+CINDEX_LINKAGE void clang_experimental_cas_getCachedCompilation_async(
+    CXCASDatabases, const char *CacheKey, bool Globally, void *Ctx,
+    void (*Callback)(void *Ctx, CXCASCachedCompilation, CXError),
+    CXCASCancellationToken *OutToken);
+
+/**
+ * Dispose of a \c CXCASCachedCompilation object.
+ */
+CINDEX_LINKAGE void
+    clang_experimental_cas_CachedCompilation_dispose(CXCASCachedCompilation);
+
+/**
+ * \returns number of compilation outputs.
+ */
+CINDEX_LINKAGE size_t clang_experimental_cas_CachedCompilation_getNumOutputs(
+    CXCASCachedCompilation);
+
+/**
+ * \returns the compilation output name given the index via \p OutputIdx.
+ */
+CINDEX_LINKAGE CXString clang_experimental_cas_CachedCompilation_getOutputName(
+    CXCASCachedCompilation, size_t OutputIdx);
+
+/**
+ * \returns the compilation output printed CASID given the index via
+ * \p OutputIdx.
+ */
+CINDEX_LINKAGE CXString
+clang_experimental_cas_CachedCompilation_getOutputCASIDString(
+    CXCASCachedCompilation, size_t OutputIdx);
+
+/**
+ * \returns whether the compilation output data exist in the local CAS given the
+ * index via \p OutputIdx.
+ */
+CINDEX_LINKAGE bool
+clang_experimental_cas_CachedCompilation_isOutputMaterialized(
+    CXCASCachedCompilation, size_t OutputIdx);
+
+/**
+ * If distributed caching is available it uploads the compilation outputs and
+ * the association of key <-> outputs to the distributed cache.
+ * This allows separating the task of computing the compilation outputs and
+ * storing them in the local cache, from the task of "uploading" them.
+ *
+ * \param Ctx opaque value to pass to the callback.
+ * \param Callback receives a \c CXError if an error occurred. The error will be
+ * NULL if the call was successful or cancelled. The error should be disposed
+ * via \c clang_Error_dispose.
+ * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that
+ * can be used to cancel the call using
+ * \c clang_experimental_cas_CancellationToken_cancel. The object should be
+ * disposed using \c clang_experimental_cas_CancellationToken_dispose.
+ */
+CINDEX_LINKAGE void clang_experimental_cas_CachedCompilation_makeGlobal(
+    CXCASCachedCompilation, void *Ctx, void (*Callback)(void *Ctx, CXError),
+    CXCASCancellationToken *OutToken);
+
+/**
+ * Replays a cached compilation by writing the cached outputs to the filesystem
+ * and/or stderr based on the given compilation arguments.
+ *
+ * \param argc number of compilation arguments.
+ * \param argv array of compilation arguments.
+ * \param WorkingDirectory working directory to use, can be NULL.
+ * \param reserved for future use, caller must pass NULL.
+ * \param[out] OutError The error object to pass back to client (if any).
+ * If non-null the object must be disposed using \c clang_Error_dispose.
+ *
+ * \returns a \c CXCASReplayResult object or NULL if an error occurred or a
+ * compilation output was not found in the CAS. The object should be disposed
+ * via \c clang_experimental_cas_ReplayResult_dispose.
+ */
+CINDEX_LINKAGE CXCASReplayResult clang_experimental_cas_replayCompilation(
+    CXCASCachedCompilation, int argc, const char *const *argv,
+    const char *WorkingDirectory, void *reserved, CXError *OutError);
+
+/**
+ * Dispose of a \c CXCASReplayResult object.
+ */
+CINDEX_LINKAGE void
+    clang_experimental_cas_ReplayResult_dispose(CXCASReplayResult);
+
+/**
+ * Get the diagnostic text of a replayed cached compilation.
+ */
+CINDEX_LINKAGE
+CXString clang_experimental_cas_ReplayResult_getStderr(CXCASReplayResult);
+
+/**
+ * Cancel an asynchronous CAS-related action.
+ */
+CINDEX_LINKAGE void
+    clang_experimental_cas_CancellationToken_cancel(CXCASCancellationToken);
+
+/**
+ * Dispose of a \c CXCASCancellationToken object.
+ */
+CINDEX_LINKAGE void
+    clang_experimental_cas_CancellationToken_dispose(CXCASCancellationToken);
+
 /**
  * Dispose of a \c CXCASObjectStore object.
  */

clang/include/clang-c/CXErrorCode.h

@@ -74,6 +74,26 @@ enum CXErrorCode {
   CXError_RefactoringNameInvalid = 7
 };
 
+/**
+ * Represents an error with error code and description string.
+ */
+typedef struct CXOpaqueError *CXError;
+
+/**
+ * \returns the error code.
+ */
+CINDEX_LINKAGE enum CXErrorCode clang_Error_getCode(CXError);
+
+/**
+ * \returns the error description string.
+ */
+CINDEX_LINKAGE const char *clang_Error_getDescription(CXError);
+
+/**
+ * Dispose of a \c CXError object.
+ */
+CINDEX_LINKAGE void clang_Error_dispose(CXError);
+
 LLVM_CLANG_C_EXTERN_C_END
 
 #endif

clang/include/clang-c/Dependencies.h

@@ -210,6 +210,14 @@ CINDEX_LINKAGE void
 clang_experimental_DependencyScannerServiceOptions_setCASDatabases(
     CXDependencyScannerServiceOptions Opts, CXCASDatabases);
 
+/**
+ * Specify the specific CAS options for the scanner to use for the produced
+ * compiler arguments.
+ */
+CINDEX_LINKAGE void
+clang_experimental_DependencyScannerServiceOptions_setCASOptions(
+    CXDependencyScannerServiceOptions Opts, CXCASOptions);
+
 /**
  * Specify a \c CXCASObjectStore in the given options. If an object store and
  * action cache are available, the scanner will produce cached commands.

clang/include/clang/Frontend/CompileJobCache.h

@@ -14,6 +14,7 @@
 namespace clang {
 
 class CompilerInstance;
+class CompilerInvocation;
 class DiagnosticsEngine;
 
 // Manage caching and replay of compile jobs.
@@ -80,6 +81,13 @@ class CompileJobCache {
   /// \returns true if finished successfully.
   bool finishComputedResult(CompilerInstance &Clang, bool Success);
 
+  static llvm::Expected<std::optional<int>> replayCachedResult(
+      std::shared_ptr<CompilerInvocation> Invok,
+      const llvm::cas::CASID &CacheKey,
+      cas::CompileJobCacheResult &CachedResult, SmallVectorImpl<char> &DiagText,
+      bool WriteOutputAsCASID = false, bool UseCASBackend = false,
+      std::optional<llvm::cas::CASID> *MCOutputID = nullptr);
+
   class CachingOutputs;
 
 private:

clang/include/clang/Frontend/CompileJobCacheResult.h

@@ -55,6 +55,8 @@ class CompileJobCacheResult : public ObjectProxy {
 
   size_t getNumOutputs() const;
 
+  Output getOutput(size_t I) const;
+
   /// Retrieves a specific output specified by \p Kind, if it exists.
   std::optional<Output> getOutput(OutputKind Kind) const;