[libclang][deps] Add initial support for compilation caching

benlangmuir · benlangmuir · commit c11f7ffd8916 · 2022-12-01T16:23:38.000-08:00
Adds API to allow specifying the CAS + ActionCache to use in a DependencyScannerService, and test that this can be used to get cached module compilations. (cherry picked from commit 1305fe6)
diff --git a/clang/include/clang-c/CAS.h b/clang/include/clang-c/CAS.h
@@ -0,0 +1,89 @@
+/*==-- clang-c/CAS.h - CAS Interface ------------------------------*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header provides interfaces for creating and working with CAS and      *|
+|* ActionCache interfaces.                                                    *|
+|*                                                                            *|
+|* An example of its usage is available in c-index-test/core_main.cpp.        *|
+|*                                                                            *|
+|* EXPERIMENTAL: These interfaces are experimental and will change. If you    *|
+|* use these be prepared for them to change without notice on any commit.     *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_CLANG_C_CAS_H
+#define LLVM_CLANG_C_CAS_H
+
+#include "clang-c/CXString.h"
+#include "clang-c/Platform.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \defgroup CAS CAS and ActionCache interface.
+ * @{
+ */
+
+/**
+ * Content-addressable storage for objects.
+ */
+typedef struct CXOpaqueCASObjectStore *CXCASObjectStore;
+
+/**
+ * A cache from a key describing an action to the result of doing it.
+ */
+typedef struct CXOpaqueCASActionCache *CXCASActionCache;
+
+/**
+ * Dispose of a \c CXCASObjectStore object.
+ */
+CINDEX_LINKAGE void
+clang_experimental_cas_ObjectStore_dispose(CXCASObjectStore CAS);
+
+/**
+ * Dispose of a \c CXCASActionCache object.
+ */
+CINDEX_LINKAGE void
+clang_experimental_cas_ActionCache_dispose(CXCASActionCache Cache);
+
+/**
+ * Gets or creates a persistent on-disk CAS object store at \p Path.
+ *
+ * \param Path The path to locate the object store.
+ * \param[out] Error The error string to pass back to client (if any).
+ *
+ * \returns The resulting object store, or null if there was an error.
+ */
+CINDEX_LINKAGE CXCASObjectStore
+clang_experimental_cas_OnDiskObjectStore_create(
+    const char *Path, CXString *Error);
+
+/**
+ * Gets or creates a persistent on-disk action cache at \p Path.
+ *
+ * \param Path The path to locate the object store.
+ * \param[out] Error The error string to pass back to client (if any).
+ *
+ * \returns The resulting object store, or null if there was an error.
+ */
+CINDEX_LINKAGE CXCASActionCache
+clang_experimental_cas_OnDiskActionCache_create(
+    const char *Path, CXString *Error);
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // LLVM_CLANG_C_CAS_H
diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h
@@ -21,6 +21,7 @@
 #define LLVM_CLANG_C_DEPENDENCIES_H
 
 #include "clang-c/BuildSystem.h"
+#include "clang-c/CAS.h"
 #include "clang-c/CXDiagnostic.h"
 #include "clang-c/CXErrorCode.h"
 #include "clang-c/CXString.h"
@@ -201,6 +202,22 @@ CINDEX_LINKAGE void
 clang_experimental_DependencyScannerServiceOptions_setDependencyMode(
     CXDependencyScannerServiceOptions Opts, CXDependencyMode Mode);
 
+/**
+ * Specify a \c CXCASObjectStore in the given options. If an object store and
+ * action cache are available, the scanner will produce cached commands.
+ */
+CINDEX_LINKAGE void
+clang_experimental_DependencyScannerServiceOptions_setObjectStore(
+    CXDependencyScannerServiceOptions Opts, CXCASObjectStore CAS);
+
+/**
+ * Specify a \c CXCASActionCache in the given options. If an object store and
+ * action cache are available, the scanner will produce cached commands.
+ */
+CINDEX_LINKAGE void
+clang_experimental_DependencyScannerServiceOptions_setActionCache(
+    CXDependencyScannerServiceOptions Opts, CXCASActionCache Cache);
+
 /**
  * See \c clang_experimental_DependencyScannerService_create_v1.
  */
diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m
@@ -0,0 +1,57 @@
+// RUN: rm -rf %t.mcp %t
+// RUN: echo %S > %t.result
+//
+// RUN: c-index-test core --scan-deps %S -output-dir=%t \
+// RUN:     -cas-path %t/cas -action-cache-path %t/cache \
+// RUN:  -- %clang -c -I %S/Inputs/module \
+// RUN:     -fmodules -fmodules-cache-path=%t.mcp \
+// RUN:     -o FoE.o -x objective-c %s >> %t.result
+// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DOUTPUTS=%/t
+
+// RUN: c-index-test core --scan-deps %S -output-dir=%t \
+// RUN:  -- %clang -c -I %S/Inputs/module \
+// RUN:     -fmodules -fmodules-cache-path=%t.mcp \
+// RUN:     -o FoE.o -x objective-c %s | FileCheck %s -check-prefix=NO_CAS
+// NO_CAS-NOT: fcas
+// NO_CAS-NOT: faction-cache
+// NO_CAS-NOT: fcache-compile-job
+
+@import ModA;
+
+// CHECK: [[PREFIX:.*]]
+// CHECK-NEXT: modules:
+// CHECK-NEXT:   module:
+// CHECK-NEXT:     name: ModA
+// CHECK-NEXT:     context-hash: [[HASH_MOD_A:[A-Z0-9]+]]
+// CHECK-NEXT:     module-map-path: [[PREFIX]]/Inputs/module/module.modulemap
+// CHECK-NEXT:     module-deps:
+// CHECK-NEXT:     file-deps:
+// CHECK-NEXT:       [[PREFIX]]/Inputs/module/ModA.h
+// CHECK-NEXT:       [[PREFIX]]/Inputs/module/SubModA.h
+// CHECK-NEXT:       [[PREFIX]]/Inputs/module/SubSubModA.h
+// CHECK-NEXT:       [[PREFIX]]/Inputs/module/module.modulemap
+// CHECK-NEXT:     build-args:
+// CHECK-SAME:       -cc1
+// CHECK-SAME:       -fcas-path
+// CHECK-SAME:       -faction-cache-path
+// CHECK-SAME:       -fcas-fs llvmcas://{{[[:xdigit:]]+}}
+// CHECK-SAME:       -fcache-compile-job
+// CHECK-SAME:       -emit-module
+// CHECK-SAME:       -fmodule-name=ModA
+// CHECK-SAME:       -fno-implicit-modules
+
+// CHECK-NEXT: dependencies:
+// CHECK-NEXT:   command 0:
+// CHECK-NEXT:     context-hash: [[HASH_TU:[A-Z0-9]+]]
+// CHECK-NEXT:     module-deps:
+// CHECK-NEXT:       ModA:[[HASH_MOD_A]]
+// CHECK-NEXT:     file-deps:
+// CHECK-NEXT:       [[PREFIX]]/scan-deps-cas.m
+// CHECK-NEXT:     build-args:
+// CHECK-SAME:       -cc1
+// CHECK-SAME:       -fcas-path
+// CHECK-SAME:       -faction-cache-path
+// CHECK-SAME:       -fcas-fs llvmcas://{{[[:xdigit:]]+}}
+// CHECK-SAME:       -fcache-compile-job
+// CHECK-SAME:       -fmodule-file-cache-key=[[PCM:.*ModA_.*pcm]]=llvmcas://{{[[:xdigit:]]+}}
+// CHECK-SAME:       -fmodule-file={{(ModA=)?}}[[PCM]]
diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp
@@ -139,6 +139,11 @@ static cl::list<std::string> DependencyTargets(
 static cl::opt<bool> DeprecatedDriverCommand(
     "deprecated-driver-command",
     cl::desc("use a single driver command to build the tu (deprecated)"));
+static llvm::cl::opt<std::string>
+    CASPath("cas-path", llvm::cl::desc("Path for on-disk CAS."));
+static llvm::cl::opt<std::string>
+    CachePath("action-cache-path",
+              llvm::cl::desc("Path for on-disk action cache."));
 }
 } // anonymous namespace
 
@@ -677,6 +682,8 @@ static void printSymbolNameAndUSR(const clang::Module *Mod, raw_ostream &OS) {
 static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
                     bool SerializeDiags, bool DependencyFile,
                     ArrayRef<std::string> DepTargets, std::string OutputPath,
+                    Optional<std::string> CASPath,
+                    Optional<std::string> CachePath,
                     Optional<std::string> ModuleName = None) {
   CXDependencyScannerServiceOptions Opts =
       clang_experimental_DependencyScannerServiceOptions_create();
@@ -686,6 +693,36 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
   clang_experimental_DependencyScannerServiceOptions_setDependencyMode(
       Opts, CXDependencyMode_Full);
 
+  CXString Error;
+  if (CASPath) {
+    CXCASObjectStore CAS = clang_experimental_cas_OnDiskObjectStore_create(
+        CASPath->c_str(), &Error);
+    auto CleanupCache = llvm::make_scope_exit(
+        [&] { clang_experimental_cas_ObjectStore_dispose(CAS); });
+    if (!CAS) {
+      llvm::errs() << "error: failed to create ObjectStore\n";
+      llvm::errs() << clang_getCString(Error) << "\n";
+      clang_disposeString(Error);
+      return 1;
+    }
+    clang_experimental_DependencyScannerServiceOptions_setObjectStore(Opts,
+                                                                      CAS);
+    if (CachePath) {
+      CXCASActionCache Cache = clang_experimental_cas_OnDiskActionCache_create(
+          CachePath->c_str(), &Error);
+      auto CleanupCache = llvm::make_scope_exit(
+          [&] { clang_experimental_cas_ActionCache_dispose(Cache); });
+      if (!Cache) {
+        llvm::errs() << "error: failed to create ActionCache\n";
+        llvm::errs() << clang_getCString(Error) << "\n";
+        clang_disposeString(Error);
+        return 1;
+      }
+      clang_experimental_DependencyScannerServiceOptions_setActionCache(Opts,
+                                                                        Cache);
+    }
+  }
+
   CXDependencyScannerService Service =
       clang_experimental_DependencyScannerService_create_v1(Opts);
   CXDependencyScannerWorker Worker =
@@ -1129,15 +1166,21 @@ int indextest_core_main(int argc, const char **argv) {
     }
     return aggregateDataAsJSON(storePath, PathRemapper, OS);
   }
-  
+
+  Optional<std::string> CASPath =
+      options::CASPath.empty() ? None : Optional<std::string>(options::CASPath);
+  Optional<std::string> CachePath =
+      options::CachePath.empty() ? None
+                                 : Optional<std::string>(options::CachePath);
+
   if (options::Action == ActionType::ScanDeps) {
     if (options::InputFiles.empty()) {
       errs() << "error: missing working directory\n";
       return 1;
     }
     return scanDeps(CompArgs, options::InputFiles[0], options::SerializeDiags,
                     options::DependencyFile, options::DependencyTargets,
-                    options::OutputDir);
+                    options::OutputDir, CASPath, CachePath);
   }
 
   if (options::Action == ActionType::ScanDepsByModuleName) {
@@ -1152,7 +1195,8 @@ int indextest_core_main(int argc, const char **argv) {
     }
     return scanDeps(CompArgs, options::InputFiles[0], options::SerializeDiags,
                     options::DependencyFile, options::DependencyTargets,
-                    options::OutputDir, options::ModuleName);
+                    options::OutputDir, CASPath, CachePath,
+                    options::ModuleName);
   }
 
   if (options::Action == ActionType::WatchDir) {
diff --git a/clang/tools/libclang/CASUtils.h b/clang/tools/libclang/CASUtils.h
@@ -0,0 +1,38 @@
+//===- CASUtils.h - libclang CAS utilities --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_LIBCLANG_CASUTILS_H
+#define LLVM_CLANG_TOOLS_LIBCLANG_CASUTILS_H
+
+#include "clang-c/CAS.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/CBindingWrapping.h"
+#include <string>
+
+namespace clang {
+namespace cas {
+
+struct WrappedObjectStore {
+  std::shared_ptr<ObjectStore> CAS;
+  std::string CASPath;
+};
+
+struct WrappedActionCache {
+  std::shared_ptr<ActionCache> Cache;
+  std::string CachePath;
+};
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedObjectStore, CXCASObjectStore)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedActionCache, CXCASActionCache)
+
+} // namespace cas
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_LIBCLANG_CASUTILS_H
diff --git a/clang/tools/libclang/CCAS.cpp b/clang/tools/libclang/CCAS.cpp
@@ -0,0 +1,51 @@
+//===- CCAS.cpp -----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang-c/CAS.h"
+
+#include "CASUtils.h"
+#include "CXString.h"
+
+#include "clang/Basic/LLVM.h"
+#include "clang/CAS/CASOptions.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+
+using namespace clang;
+using namespace clang::cas;
+
+void clang_experimental_cas_ObjectStore_dispose(CXCASObjectStore CAS) {
+  delete unwrap(CAS);
+}
+void clang_experimental_cas_ActionCache_dispose(CXCASActionCache Cache) {
+  delete unwrap(Cache);
+}
+
+CXCASObjectStore
+clang_experimental_cas_OnDiskObjectStore_create(const char *Path,
+                                                CXString *Error) {
+  auto CAS = llvm::cas::createOnDiskCAS(Path);
+  if (!CAS) {
+    if (Error)
+      *Error = cxstring::createDup(llvm::toString(CAS.takeError()));
+    return nullptr;
+  }
+  return wrap(new WrappedObjectStore{std::move(*CAS), Path});
+}
+
+CXCASActionCache
+clang_experimental_cas_OnDiskActionCache_create(const char *Path,
+                                                CXString *Error) {
+  auto Cache = llvm::cas::createOnDiskActionCache(Path);
+  if (!Cache) {
+    if (Error)
+      *Error = cxstring::createDup(llvm::toString(Cache.takeError()));
+    return nullptr;
+  }
+  return wrap(new WrappedActionCache{std::move(*Cache), Path});
+}
diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp
diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt
diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map
