[clang] Port libclang-replay-job test to clang-scan-deps

This test was unintentionally using the native filesystem instead of the
cas include-tree filesystem. Move to using clang-scan-deps to calculate
the cc1 instead of trying to hand write it.

(cherry picked from commit a838e29)

Author: benlangmuir

clang/lib/Frontend/CompileJobCache.cpp

@@ -500,6 +500,11 @@ Expected<std::optional<int>> CompileJobCache::replayCachedResult(
       new TextDiagnosticPrinter(DiagOS, &Clang.getDiagnosticOpts()));
   Clang.setVerboseOutputStream(DiagOS);
 
+  // FIXME: we should create an include-tree filesystem based on the cache key
+  // to guarantee that the filesystem used during diagnostic replay will match
+  // the cached diagnostics. Currently we rely on the invocation having a
+  // matching -fcas-include-tree option.
+
   auto FinishDiagnosticClient =
       llvm::make_scope_exit([&]() { Clang.getDiagnosticClient().finish(); });
 

clang/test/CAS/libclang-replay-job.c

@@ -1,19 +1,20 @@
 // REQUIRES: shell
 
 // RUN: rm -rf %t && mkdir -p %t
-// RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid
-//
-// RUN: %clang -cc1 -triple x86_64-apple-macos11 \
-// RUN:   -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
-// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
-// RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
-// RUN:   -dependency-file %t/t1.d -MT deps -emit-obj -o %t/output1.o %s
-// RUN: %clang -cc1 -triple x86_64-apple-macos11 \
-// RUN:   -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
-// RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN: split-file %s %t
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -format experimental-include-tree-full \
+// RUN:   -cas-path %t/cas -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
-// RUN:   -serialize-diagnostic-file %t/t1.dia -dependency-file %t/t1.d -MT deps \
-// RUN:   -Rcompile-job-cache-hit -emit-obj -o %t/output1.o %s 2> %t/output1.txt
+// RUN:   > %t/deps.json
+
+// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/cc1.rsp
+
+// RUN: (cd %t; %clang @%t/cc1.rsp)
+// RUN: (cd %t; %clang @%t/cc1.rsp -Rcompile-job-cache-hit \
+// RUN:   -serialize-diagnostic-file %t/t1.dia 2> %t/output1.txt)
 
 // Verify the warning was recorded and we compare populated .dia files.
 // RUN: c-index-test -read-diagnostics %t/t1.dia 2>&1 | FileCheck %s --check-prefix=DIAGS
@@ -25,37 +26,54 @@
 
 // Delete the "local" cache and use the "upstream" one to re-materialize the outputs locally.
 // RUN: rm -rf %t/cas
+
+// Re-run the scan to populate the include-tree in the cas
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -format experimental-include-tree-full \
+// RUN:   -cas-path %t/cas -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
+// RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \
+// RUN:   > %t/deps2.json
+// RUN: diff -u %t/deps.json %t/deps2.json
+
+
 // RUN: c-index-test core -materialize-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging
 
 // RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option no-logging \
-// RUN: -- -cc1 \
+// RUN:   -working-dir %t \
+// RUN: -- @%t/cc1.rsp \
 // RUN:   -serialize-diagnostic-file %t/t2.dia -Rcompile-job-cache-hit \
-// RUN:   -dependency-file %t/t2.d -MT deps \
-// RUN:   -o %t/output2.o 2> %t/output2.txt
+// RUN:   -dependency-file %t/t2.d -o %t/output2.o 2> %t/output2.txt
 
 // RUN: diff %t/output1.o %t/output2.o
 // RUN: diff -u %t/output1.txt %t/output2.txt
 // RUN: diff %t/t1.dia %t/t2.dia
 // RUN: diff -u %t/t1.d %t/t2.d
 
-// Check with `-working-dir` flag.
+// Check with different `-working-dir` flag.
 // RUN: mkdir -p %t/a/b
 // RUN: cd %t/a
 // RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \
 // RUN:   -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
 // RUN:   -fcas-plugin-option no-logging \
 // RUN:   -working-dir %t/a/b \
-// RUN: -- -cc1 %t/a/b \
+// RUN: -- @%t/cc1.rsp \
 // RUN:   -serialize-diagnostic-file rel.dia -Rcompile-job-cache-hit \
-// RUN:   -dependency-file rel.d -MT deps \
-// RUN:   -o reloutput.o
+// RUN:   -dependency-file rel.d -o reloutput.o
 
 // RUN: diff %t/output1.o %t/a/b/reloutput.o
 // RUN: diff -u %t/t1.d %t/a/b/rel.d
 // FIXME: Get clang's `-working-directory` to affect relative path for serialized diagnostics.
 
+//--- cdb.json.template
+[{
+  "directory": "DIR",
+  "command": "clang -c DIR/main.c -target x86_64-apple-macos11 -MD -MF t1.d -MT deps -o output1.o",
+  "file": "DIR/main.c"
+}]
+
+//--- main.c
 #warning some warning

clang/tools/clang-scan-deps/ClangScanDeps.cpp

@@ -225,6 +225,12 @@ llvm::cl::opt<bool> EmitCASCompDB(
 llvm::cl::opt<std::string>
     OnDiskCASPath("cas-path", llvm::cl::desc("Path for on-disk CAS."),
                   llvm::cl::cat(DependencyScannerCategory));
+llvm::cl::opt<std::string>
+    CASPluginPath("fcas-plugin-path", llvm::cl::desc("Path to a shared library implementing the LLVM CAS plugin API"),
+                  llvm::cl::cat(DependencyScannerCategory));
+llvm::cl::list<std::string>
+    CASPluginOptions("fcas-plugin-option", llvm::cl::desc("Option to pass to the CAS plugin"),
+                  llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::opt<bool> InMemoryCAS(
     "in-memory-cas", llvm::cl::desc("Use an in-memory CAS instead of on-disk."),
@@ -898,16 +904,19 @@ int main(int argc, const char **argv) {
   Diags.setClient(DiagsConsumer.get(), /*ShouldOwnClient=*/false);
 
   CASOptions CASOpts;
+  CASOpts.CASPath = OnDiskCASPath;
+  CASOpts.PluginPath = CASPluginPath;
+  for (StringRef Arg : CASPluginOptions) {
+    auto [L, R] = Arg.split('=');
+    CASOpts.PluginOptions.emplace_back(std::string(L), std::string(R));
+  }
+
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> Cache;
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
   if (useCAS()) {
-    if (!InMemoryCAS) {
-      if (!OnDiskCASPath.empty())
-        CASOpts.CASPath = OnDiskCASPath;
-      else
-        CASOpts.ensurePersistentCAS();
-    }
+    if (!InMemoryCAS)
+      CASOpts.ensurePersistentCAS();
 
     std::tie(CAS, Cache) = CASOpts.getOrCreateDatabases(Diags);
     if (!CAS)

clang/tools/libclang/CCAS.cpp

@@ -14,6 +14,7 @@
 
 #include "clang/Basic/LLVM.h"
 #include "clang/CAS/CASOptions.h"
+#include "clang/Driver/Driver.h"
 #include "clang/Frontend/CompileJobCache.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
@@ -419,10 +420,22 @@ CXCASReplayResult clang_experimental_cas_replayCompilation(
   auto *DiagsPrinter = new TextDiagnosticPrinter(DiagOS, DiagOpts.get());
   DiagnosticsEngine Diags(DiagID, DiagOpts.get(), DiagsPrinter);
 
+  SmallVector<const char *, 256> Args(argv, argv + argc);
+  llvm::BumpPtrAllocator Alloc;
+  bool CLMode = driver::IsClangCL(
+      driver::getDriverMode(argv[0], ArrayRef(Args).slice(1)));
+
+  if (llvm::Error E = driver::expandResponseFiles(Args, CLMode, Alloc)) {
+    Diags.Report(diag::err_drv_expand_response_file)
+        << llvm::toString(std::move(E));
+    if (OutError)
+      *OutError = cxerror::create(DiagOS.str());
+    return nullptr;
+  }
+
   auto Invok = std::make_shared<CompilerInvocation>();
-  ArrayRef Argv(argv, argc);
-  bool Success = CompilerInvocation::CreateFromArgs(*Invok, Argv.drop_front(),
-                                                    Diags, Argv.front());
+  bool Success = CompilerInvocation::CreateFromArgs(*Invok, ArrayRef(Args).drop_front(),
+                                                    Diags, Args.front());
   if (!Success) {
     if (OutError)
       *OutError = cxerror::create(DiagOS.str());