Merge commit '5a4109d44085' from swift/release/6.0 into stable/20230725

Author: git apple-llvm automerger

clang/include/clang/Frontend/CompileJobCache.h

@@ -92,6 +92,10 @@ class CompileJobCache {
   class CachingOutputs;
 
 private:
+  /// \returns true if the output from the compilation is not supported for
+  /// caching.
+  Expected<bool>
+  maybeIngestNonVirtualOutputFromFileSystem(CompilerInstance &Clang);
   int reportCachingBackendError(DiagnosticsEngine &Diag, llvm::Error &&E);
 
   bool CacheCompileJob = false;

clang/lib/Frontend/CompileJobCache.cpp

@@ -51,6 +51,11 @@ class CompileJobCache::CachingOutputs {
 
   void stopDiagnosticsCapture();
 
+  /// This is for an output file that was written directly on the file system.
+  /// It's a workaround until all compilation consumers adopt
+  /// \c llvm::vfs::OutputBackend.
+  virtual Error addNonVirtualOutputFile(StringRef FilePath) = 0;
+
   /// Finish writing outputs from a computed result, after a cache miss.
   /// If SkipCache is true, it should not insert the ResultCacheKey into
   /// Cache for future uses.
@@ -105,6 +110,8 @@ class ObjectStoreCachingOutputs : public CompileJobCache::CachingOutputs {
 
   bool prepareOutputCollection() override;
 
+  Error addNonVirtualOutputFile(StringRef FilePath) override;
+
   Error finishComputedResult(const llvm::cas::CASID &ResultCacheKey,
                              bool SkipCache) override;
 
@@ -143,6 +150,11 @@ class CollectingOutputBackend : public llvm::vfs::ProxyOutputBackend {
     KindMaps.push_back({Saver.save(Kind), Saver.save(Path)});
   }
 
+  void addOutput(StringRef Path) {
+    StringRef Name = tryRemapPath(Path);
+    OutputNames.push_back(Name.str());
+  }
+
 private:
   llvm::BumpPtrAllocator Alloc;
   llvm::StringSaver Saver{Alloc};
@@ -166,8 +178,7 @@ class CollectingOutputBackend : public llvm::vfs::ProxyOutputBackend {
   Expected<std::unique_ptr<llvm::vfs::OutputFileImpl>>
   createFileImpl(StringRef Path,
                  std::optional<llvm::vfs::OutputConfig> Config) override {
-    StringRef Name = tryRemapPath(Path);
-    OutputNames.push_back(Name.str());
+    addOutput(Path);
     return ProxyOutputBackend::createFileImpl(Path, std::move(Config));
   }
 
@@ -198,6 +209,8 @@ class RemoteCachingOutputs : public CompileJobCache::CachingOutputs {
 
   bool prepareOutputCollection() override;
 
+  Error addNonVirtualOutputFile(StringRef FilePath) override;
+
   Error finishComputedResult(const llvm::cas::CASID &ResultCacheKey,
                              bool SkipCache) override;
 
@@ -514,9 +527,18 @@ bool CompileJobCache::finishComputedResult(CompilerInstance &Clang,
 
   DiagnosticsEngine &Diags = Clang.getDiagnostics();
 
+  bool UnsupportedOutput;
+  if (Error E = maybeIngestNonVirtualOutputFromFileSystem(Clang).moveInto(
+          UnsupportedOutput)) {
+    reportCachingBackendError(Diags, std::move(E));
+    return false;
+  }
+
   // Check if we encounter any source that would generate non-reproducible
   // outputs.
-  bool SkipCache = Clang.hasPreprocessor() && Clang.isSourceNonReproducible();
+  bool SkipCache =
+      (Clang.hasPreprocessor() && Clang.isSourceNonReproducible()) ||
+      UnsupportedOutput;
   if (SkipCache) {
     switch (Clang.getPreprocessorOpts().CachingDiagOption) {
     case CachingDiagKind::None:
@@ -538,6 +560,31 @@ bool CompileJobCache::finishComputedResult(CompilerInstance &Clang,
   return true;
 }
 
+Expected<bool> CompileJobCache::maybeIngestNonVirtualOutputFromFileSystem(
+    CompilerInstance &Clang) {
+  // FIXME: All consumers should adopt \c llvm::vfs::OutputBackend and this
+  // function should go away.
+
+  const auto &FrontendOpts = Clang.getFrontendOpts();
+  if (FrontendOpts.ProgramAction == frontend::RunAnalysis) {
+    StringRef OutputPath = FrontendOpts.OutputFile;
+    if (OutputPath.empty())
+      return false;
+    if (llvm::sys::fs::is_directory(OutputPath)) {
+      // FIXME: A directory is produced for the 'html' output of the analyzer,
+      // support it for caching purposes.
+      Clang.getDiagnostics().Report(diag::warn_clang_cache_disabled_caching)
+          << "analyzer output is not supported";
+      return true;
+    }
+    if (Error E = CacheBackend->addNonVirtualOutputFile(OutputPath))
+      return std::move(E);
+    return false;
+  }
+
+  return false;
+}
+
 Expected<std::optional<int>> CompileJobCache::replayCachedResult(
     std::shared_ptr<CompilerInvocation> Invok, StringRef WorkingDir,
     const llvm::cas::CASID &CacheKey, cas::CompileJobCacheResult &CachedResult,
@@ -633,6 +680,20 @@ Expected<llvm::cas::ObjectRef> ObjectStoreCachingOutputs::writeOutputs(
   return CachedResultBuilder.build(*CAS);
 }
 
+Error ObjectStoreCachingOutputs::addNonVirtualOutputFile(StringRef FilePath) {
+  auto F = llvm::sys::fs::openNativeFileForRead(FilePath);
+  if (!F)
+    return F.takeError();
+  auto CloseOnExit =
+      llvm::make_scope_exit([&F]() { llvm::sys::fs::closeFile(*F); });
+
+  std::optional<llvm::cas::ObjectRef> ObjRef;
+  if (Error E = CAS->storeFromOpenFile(*F).moveInto(ObjRef))
+    return E;
+  CASOutputs->addObject(FilePath, *ObjRef);
+  return Error::success();
+}
+
 Error ObjectStoreCachingOutputs::finishComputedResult(
     const llvm::cas::CASID &ResultCacheKey, bool SkipCache) {
   Expected<llvm::cas::ObjectRef> Result = writeOutputs(ResultCacheKey);
@@ -1017,6 +1078,11 @@ RemoteCachingOutputs::saveOutputs(const llvm::cas::CASID &ResultCacheKey) {
   return std::move(CompResult);
 }
 
+Error RemoteCachingOutputs::addNonVirtualOutputFile(StringRef FilePath) {
+  CollectingOutputs->addOutput(FilePath);
+  return Error::success();
+}
+
 Error RemoteCachingOutputs::finishComputedResult(
     const llvm::cas::CASID &ResultCacheKey, bool SkipCache) {
   if (SkipCache)

clang/test/CAS/analyze-action-remote-service.c

@@ -0,0 +1,40 @@
+// REQUIRES: remote-cache-service
+
+// Need a short path for the unix domain socket (and unique for this test file).
+// RUN: rm -f %{remote-cache-dir}/%basename_t
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %clang_cc1 -cc1 -triple x86_64-apple-macosx12 -analyze -analyzer-checker=deadcode -analyzer-output plist %s -o %t/regular.plist 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-DIAG
+// RUN: FileCheck %s --input-file=%t/regular.plist --check-prefix=CHECK-PLIST
+
+// CHECK-DIAG: Value stored to 'v' during its initialization is never read
+// CHECK-PLIST: Value stored to 'v' during its initialization is never read
+
+// RUN: llvm-remote-cache-test -socket-path=%{remote-cache-dir}/%basename_t -cache-path=%t/cache -- env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache \
+// RUN:   %clang -target x86_64-apple-macosx12 --analyze --analyzer-output plist %s -o %t/cached.plist
+// RUN: rm %t/cached.plist
+// RUN: llvm-remote-cache-test -socket-path=%{remote-cache-dir}/%basename_t -cache-path=%t/cache -- env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache \
+// RUN:   %clang -target x86_64-apple-macosx12 --analyze --analyzer-output plist %s -o %t/cached.plist -Rcompile-job-cache \
+// RUN:   2>&1 | FileCheck %s --check-prefix=CHECK-HIT
+
+// CHECK-HIT: remark: compile job cache hit
+// CHECK-HIT: Value stored to 'v' during its initialization is never read
+
+// RUN: diff -u %t/regular.plist %t/cached.plist
+
+// Check cache is skipped for analyzer html output.
+// RUN: llvm-remote-cache-test -socket-path=%{remote-cache-dir}/%basename_t -cache-path=%t/cache -- env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache \
+// RUN:   %clang -target x86_64-apple-macosx12 --analyze --analyzer-output html %s -o %t/analysis -Rcompile-job-cache -Wclang-cache \
+// RUN:   2>&1 | FileCheck %s --check-prefix=CHECK-HTML
+// RUN: llvm-remote-cache-test -socket-path=%{remote-cache-dir}/%basename_t -cache-path=%t/cache -- env LLVM_CACHE_CAS_PATH=%t/cas %clang-cache \
+// RUN:   %clang -target x86_64-apple-macosx12 --analyze --analyzer-output html %s -o %t/analysis -Rcompile-job-cache -Wclang-cache \
+// RUN:   2>&1 | FileCheck %s --check-prefix=CHECK-HTML
+
+// CHECK-HTML: remark: compile job cache miss
+// CHECK-HTML: remark: compile job cache skipped
+// FIXME: `analyse` action passes `-w` for `-cc1` args and the "caching disabled" warning doesn't show up.
+
+void foo(int *p) {
+  int v = p[0];
+}

clang/test/CAS/analyze-action.c

@@ -0,0 +1,37 @@
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %clang_cc1 -cc1 -triple x86_64-apple-macosx12 -analyze -analyzer-checker=deadcode -analyzer-output plist %s -o %t/regular.plist 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-DIAG
+// RUN: FileCheck %s --input-file=%t/regular.plist --check-prefix=CHECK-PLIST
+
+// CHECK-DIAG: Value stored to 'v' during its initialization is never read
+// CHECK-PLIST: Value stored to 'v' during its initialization is never read
+
+// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t.rsp -cc1-args \
+// RUN:   -cc1 -triple x86_64-apple-macosx12 -fcas-path %t/cas -analyze -analyzer-checker=deadcode -analyzer-output plist %s -o %t/cached.plist
+// RUN: %clang @%t/t.rsp
+
+// RUN: rm %t/cached.plist
+// RUN: %clang @%t/t.rsp -Rcompile-job-cache 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-HIT
+
+// CHECK-HIT: remark: compile job cache hit
+// CHECK-HIT: Value stored to 'v' during its initialization is never read
+
+// RUN: diff -u %t/regular.plist %t/cached.plist
+
+// Check cache is skipped for analyzer html output.
+// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t2.rsp -cc1-args \
+// RUN:   -cc1 -triple x86_64-apple-macosx12 -fcas-path %t/cas -analyze -analyzer-checker=deadcode -analyzer-output html %s -o %t/analysis
+// RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-HTML
+// RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-HTML
+
+// CHECK-HTML: remark: compile job cache miss
+// CHECK-HTML: warning: caching disabled because analyzer output is not supported
+// CHECK-HTML: remark: compile job cache skipped
+
+void foo(int *p) {
+  int v = p[0];
+}

llvm/include/llvm/CAS/CASOutputBackend.h

@@ -33,7 +33,7 @@ class CASOutputBackend : public vfs::OutputBackend {
   SmallVector<OutputFile> takeOutputs() { return std::move(Outputs); }
 
   /// Add a CAS object to the path in the output backend.
-  Error addObject(StringRef Path, const CASID &Object);
+  void addObject(StringRef Path, ObjectRef Object);
 
 private:
   Expected<std::unique_ptr<vfs::OutputFileImpl>>

llvm/lib/CAS/CASOutputBackend.cpp

@@ -60,8 +60,12 @@ CASOutputBackend::createFileImpl(StringRef ResolvedPath,
         if (Error E =
                 CAS.storeFromString(std::nullopt, Bytes).moveInto(BytesRef))
           return E;
-        // FIXME: Should there be a lock taken before modifying Outputs?
-        Outputs.push_back({std::string(Path), *BytesRef});
+        addObject(Path, *BytesRef);
         return Error::success();
       });
 }
+
+void CASOutputBackend::addObject(StringRef Path, ObjectRef Object) {
+  // FIXME: Should there be a lock taken before modifying Outputs?
+  Outputs.push_back({std::string(Path), Object});
+}