[clang] Port libclang-replay-job test to clang-scan-deps

This test was unintentionally using the native filesystem instead of the
cas include-tree filesystem. Move to using clang-scan-deps to calculate
the cc1 instead of trying to hand write it.

Author: benlangmuir

clang/lib/Frontend/CompileJobCache.cpp

@@ -535,6 +535,11 @@ Expected<std::optional<int>> CompileJobCache::replayCachedResult(
       new TextDiagnosticPrinter(DiagOS, &Clang.getDiagnosticOpts()));
   Clang.setVerboseOutputStream(DiagOS);
 
+  // FIXME: we should create an include-tree filesystem based on the cache key
+  // to guarantee that the filesystem used during diagnostic replay will match
+  // the cached diagnostics. Currently we rely on the invocation having a
+  // matching -fcas-include-tree option.
+
   auto FinishDiagnosticClient =
       llvm::make_scope_exit([&]() { Clang.getDiagnosticClient().finish(); });
 

clang/test/CAS/libclang-replay-job.c

@@ -1,19 +1,20 @@
 // REQUIRES: shell
 
 // RUN: rm -rf %t && mkdir -p %t
-// RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid
-//
-// RUN: %clang -cc1 -triple x86_64-apple-macos11 \
-// RUN:   -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
-// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
-// RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
-// RUN:   -dependency-file %t/t1.d -MT deps -emit-obj -o %t/output1.o %s
-// RUN: %clang -cc1 -triple x86_64-apple-macos11 \
-// RUN:   -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
-// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN: split-file %s %t
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -format experimental-include-tree-full \
+// RUN:   -cas-path %t/cas -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
-// RUN:   -serialize-diagnostic-file %t/t1.dia -dependency-file %t/t1.d -MT deps \
-// RUN:   -Rcompile-job-cache-hit -emit-obj -o %t/output1.o %s 2> %t/output1.txt
+// RUN:   > %t/deps.json
+
+// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/cc1.rsp
+
+// RUN: (cd %t; %clang @%t/cc1.rsp)
+// RUN: (cd %t; %clang @%t/cc1.rsp -Rcompile-job-cache-hit \
+// RUN:   -serialize-diagnostic-file %t/t1.dia 2> %t/output1.txt)
 
 // Verify the warning was recorded and we compare populated .dia files.
 // RUN: c-index-test -read-diagnostics %t/t1.dia 2>&1 | FileCheck %s --check-prefix=DIAGS
@@ -25,37 +26,54 @@
 
 // Delete the "local" cache and use the "upstream" one to re-materialize the outputs locally.
 // RUN: rm -rf %t/cas
+
+// Re-run the scan to populate the include-tree in the cas
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -format experimental-include-tree-full \
+// RUN:   -cas-path %t/cas -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
+// RUN:   > %t/deps2.json
+// RUN: diff -u %t/deps.json %t/deps2.json
+
+
 // RUN: c-index-test core -materialize-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging
 
 // RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option no-logging \
-// RUN: -- -cc1 \
+// RUN:   -working-dir %t \
+// RUN: -- @%t/cc1.rsp \
 // RUN:   -serialize-diagnostic-file %t/t2.dia -Rcompile-job-cache-hit \
-// RUN:   -dependency-file %t/t2.d -MT deps \
-// RUN:   -o %t/output2.o 2> %t/output2.txt
+// RUN:   -dependency-file %t/t2.d -o %t/output2.o 2> %t/output2.txt
 
 // RUN: diff %t/output1.o %t/output2.o
 // RUN: diff -u %t/output1.txt %t/output2.txt
 // RUN: diff %t/t1.dia %t/t2.dia
 // RUN: diff -u %t/t1.d %t/t2.d
 
-// Check with `-working-dir` flag.
+// Check with different `-working-dir` flag.
 // RUN: mkdir -p %t/a/b
 // RUN: cd %t/a
 // RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option no-logging \
 // RUN:   -working-dir %t/a/b \
-// RUN: -- -cc1 %t/a/b \
+// RUN: -- @%t/cc1.rsp \
 // RUN:   -serialize-diagnostic-file rel.dia -Rcompile-job-cache-hit \
-// RUN:   -dependency-file rel.d -MT deps \
-// RUN:   -o reloutput.o
+// RUN:   -dependency-file rel.d -o reloutput.o
 
 // RUN: diff %t/output1.o %t/a/b/reloutput.o
 // RUN: diff -u %t/t1.d %t/a/b/rel.d
 // FIXME: Get clang's `-working-directory` to affect relative path for serialized diagnostics.
 
+//--- cdb.json.template
+[{
+  "directory": "DIR",
+  "command": "clang -c DIR/main.c -target x86_64-apple-macos11 -MD -MF t1.d -MT deps -o output1.o",
+  "file": "DIR/main.c"
+}]
+
+//--- main.c
 #warning some warning

clang/tools/clang-scan-deps/ClangScanDeps.cpp

@@ -96,7 +96,7 @@ static bool Verbose;
 static bool PrintTiming;
 static std::vector<const char *> CommandLine;
 static bool EmitCASCompDB;
-static std::string OnDiskCASPath;
+static CASOptions CASOpts;
 static bool InMemoryCAS;
 static std::string PrefixMapToolchain;
 static std::string PrefixMapSDK;
@@ -220,7 +220,14 @@ static void ParseArgs(int argc, char **argv) {
   InMemoryCAS = Args.hasArg(OPT_in_memory_cas);
 
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_cas_path_EQ))
-    OnDiskCASPath = A->getValue();
+    CASOpts.CASPath = A->getValue();
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_fcas_plugin_path_EQ))
+    CASOpts.PluginPath = A->getValue();
+  for (const llvm::opt::Arg *A : Args.filtered(OPT_fcas_plugin_option_EQ)) {
+    auto [L, R] = StringRef(A->getValue()).split('=');
+    CASOpts.PluginOptions.emplace_back(std::string(L), std::string(R));
+  }
+
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_prefix_map_toolchain_EQ))
     PrefixMapToolchain = A->getValue();
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_prefix_map_sdk_EQ))
@@ -556,7 +563,7 @@ static bool outputFormatRequiresCAS() {
 }
 
 static bool useCAS() {
-  return InMemoryCAS || !OnDiskCASPath.empty() || outputFormatRequiresCAS();
+  return InMemoryCAS || !CASOpts.CASPath.empty() || outputFormatRequiresCAS();
 }
 
 static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
@@ -1132,17 +1139,12 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
   DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions());
   Diags.setClient(DiagsConsumer.get(), /*ShouldOwnClient=*/false);
 
-  CASOptions CASOpts;
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> Cache;
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
   if (useCAS()) {
-    if (!InMemoryCAS) {
-      if (!OnDiskCASPath.empty())
-        CASOpts.CASPath = OnDiskCASPath;
-      else
-        CASOpts.ensurePersistentCAS();
-    }
+    if (!InMemoryCAS)
+      CASOpts.ensurePersistentCAS();
 
     std::tie(CAS, Cache) = CASOpts.getOrCreateDatabases(Diags);
     if (!CAS)

clang/tools/clang-scan-deps/Opts.td

@@ -35,6 +35,9 @@ def emit_cas_compdb : F<"emit-cas-compdb", "Emit compilation DB with updated cla
 
 defm cas_path : Eq<"cas-path", "Path for on-disk CAS.">;
 
+defm fcas_plugin_path : Eq<"fcas-plugin-path", "Path to a shared library implementing the LLVM CAS plugin API">;
+defm fcas_plugin_option : Eq<"fcas-plugin-option", "Option to pass to the CAS plugin">;
+
 def in_memory_cas : F<"in-memory-cas", "Use an in-memory CAS instead of on-disk.">;
 
 defm prefix_map_toolchain : Eq<"prefix-map-toolchain", "Path to remap toolchain path to.">;

clang/tools/libclang/CCAS.cpp

@@ -14,6 +14,7 @@
 
 #include "clang/Basic/LLVM.h"
 #include "clang/CAS/CASOptions.h"
+#include "clang/Driver/Driver.h"
 #include "clang/Frontend/CompileJobCache.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
@@ -415,10 +416,22 @@ CXCASReplayResult clang_experimental_cas_replayCompilation(
   auto *DiagsPrinter = new TextDiagnosticPrinter(DiagOS, DiagOpts.get());
   DiagnosticsEngine Diags(DiagID, DiagOpts.get(), DiagsPrinter);
 
+  SmallVector<const char *, 256> Args(argv, argv + argc);
+  llvm::BumpPtrAllocator Alloc;
+  bool CLMode = driver::IsClangCL(
+      driver::getDriverMode(argv[0], ArrayRef(Args).slice(1)));
+
+  if (llvm::Error E = driver::expandResponseFiles(Args, CLMode, Alloc)) {
+    Diags.Report(diag::err_drv_expand_response_file)
+        << llvm::toString(std::move(E));
+    if (OutError)
+      *OutError = cxerror::create(DiagOS.str());
+    return nullptr;
+  }
+
   auto Invok = std::make_shared<CompilerInvocation>();
-  ArrayRef Argv(argv, argc);
-  bool Success = CompilerInvocation::CreateFromArgs(*Invok, Argv.drop_front(),
-                                                    Diags, Argv.front());
+  bool Success = CompilerInvocation::CreateFromArgs(*Invok, ArrayRef(Args).drop_front(),
+                                                    Diags, Args.front());
   if (!Success) {
     if (OutError)
       *OutError = cxerror::create(DiagOS.str());