Merge pull request #7143 from benlangmuir/cp-cas

[stable/20221013] Cherry-pick CAS changes

Author: benlangmuir

clang-tools-extra/clangd/Compiler.cpp

@@ -73,7 +73,7 @@ void disableUnsupportedOptions(CompilerInvocation &CI) {
   // Always default to raw container format as clangd doesn't registry any other
   // and clang dies when faced with unknown formats.
   CI.getHeaderSearchOpts().ModuleFormat =
-      PCHContainerOperations().getRawReader().getFormat().str();
+      PCHContainerOperations().getRawReader().getFormats().front().str();
 
   CI.getFrontendOpts().Plugins.clear();
   CI.getFrontendOpts().AddPluginActions.clear();

clang/include/clang/Basic/DiagnosticIDs.h

@@ -160,6 +160,10 @@ class DiagnosticMapping {
     Result.Severity = Bits & 0x7;
     return Result;
   }
+
+  bool operator==(DiagnosticMapping Other) const {
+    return serialize() == Other.serialize();
+  }
 };
 
 /// Used for handling and querying diagnostic IDs.
@@ -209,6 +213,9 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   /// default.
   static bool isDefaultMappingAsError(unsigned DiagID);
 
+  /// Get the default mapping for this diagnostic.
+  static DiagnosticMapping getDefaultMapping(unsigned DiagID);
+
   /// Determine whether the given built-in diagnostic ID is a Note.
   static bool isBuiltinNote(unsigned DiagID);
 

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -764,7 +764,8 @@ def warn_pp_date_time : Warning<
 
 def warn_pp_encounter_nonreproducible: Warning<
   "encountered non-reproducible token, caching will be skipped">,
-  InGroup<DiagGroup<"reproducible-caching">>, DefaultIgnore;
+  InGroup<DiagGroup<"reproducible-caching">>,
+  DefaultWarnNoWerror, ShowInSystemHeader, ShowInSystemMacro;
 def err_pp_encounter_nonreproducible: Error<
   "encountered non-reproducible token, caching failed">;
 

clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h

@@ -32,7 +32,7 @@ class ObjectFilePCHContainerWriter : public PCHContainerWriter {
 /// A PCHContainerReader implementation that uses LLVM to
 /// wraps Clang modules inside a COFF, ELF, or Mach-O container.
 class ObjectFilePCHContainerReader : public PCHContainerReader {
-  StringRef getFormat() const override { return "obj"; }
+  ArrayRef<StringRef> getFormats() const override;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;

clang/include/clang/Driver/Options.td

@@ -6650,6 +6650,11 @@ defm cache_disable_replay : BoolFOption<"cache-disable-replay",
     PosFlag<SetTrue, [], "Disable replaying a cached compile job">,
     NegFlag<SetFalse>>;
 
+defm modules_skip_diagnostic_options : BoolFOption<"modules-skip-diagnostic-options",
+    HeaderSearchOpts<"ModulesSkipDiagnosticOptions">, DefaultFalse,
+    PosFlag<SetTrue, [], "Disable writing diagnostic options">,
+    NegFlag<SetFalse>>;
+
 def fcompilation_caching_service_path : Separate<["-"], "fcompilation-caching-service-path">,
     Group<f_Group>, MetaVarName<"<pathname>">,
     HelpText<"Specify the socket path for connecting to a remote caching service">,

clang/include/clang/Frontend/CompileJobCache.h

@@ -0,0 +1,100 @@
+//===- CompileJobCache.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_FRONTEND_COMPILEJOBCACHE_H
+#define LLVM_CLANG_FRONTEND_COMPILEJOBCACHE_H
+
+#include "clang/Frontend/CompileJobCacheResult.h"
+
+namespace clang {
+
+class CompilerInstance;
+class DiagnosticsEngine;
+
+// Manage caching and replay of compile jobs.
+//
+// The high-level model is:
+//
+//  1. Extract options from the CompilerInvocation:
+//       - that can be simulated and
+//       - that don't affect the compile job's result.
+//  2. Canonicalize the options extracted in (1).
+//  3. Compute the result of the compile job using the canonicalized
+//     CompilerInvocation, with hooks installed to redirect outputs and
+//     enable live-streaming of a running compile job to stdout or stderr.
+//       - Compute a cache key.
+//       - Check the cache, and run the compile job if there's a cache miss.
+//       - Store the result of the compile job in the cache.
+//  4. Replay the compile job, using the options extracted in (1).
+//
+// An example (albeit not yet implemented) is handling options controlling
+// output of diagnostics. The CompilerInvocation can be canonicalized to
+// serialize the diagnostics to a virtual path (<output>.diag or something).
+//
+//   - On a cache miss, the compile job runs, and the diagnostics are
+//     serialized and stored in the cache per the canonicalized options
+//     from (2).
+//   - Either way, the diagnostics are replayed according to the options
+//     extracted from (1) during (4).
+//
+// The above will produce the correct output for diagnostics, but the experience
+// will be degraded in the common command-line case (emitting to stderr)
+// because the diagnostics will not be streamed live. This can be improved:
+//
+//   - Change (3) to accept a hook: a DiagnosticsConsumer that diagnostics
+//     are mirrored to (in addition to canonicalized options from (2)).
+//   - If diagnostics would be live-streamed, send in a diagnostics consumer
+//     that matches (1). Otherwise, send in an IgnoringDiagnosticsConsumer.
+//   - In step (4), only skip replaying the diagnostics if they were already
+//     handled.
+class CompileJobCache {
+public:
+  CompileJobCache();
+  ~CompileJobCache();
+
+  using OutputKind = clang::cas::CompileJobCacheResult::OutputKind;
+
+  StringRef getPathForOutputKind(OutputKind Kind);
+
+  /// Canonicalize \p Clang.
+  ///
+  /// \returns status if should exit immediately, otherwise None.
+  ///
+  /// TODO: Refactor \a cc1_main() so that instead this canonicalizes the
+  /// CompilerInvocation before Clang gets access to command-line arguments, to
+  /// control what might leak.
+  std::optional<int> initialize(CompilerInstance &Clang);
+
+  /// Try looking up a cached result and replaying it.
+  ///
+  /// \returns status if should exit immediately, otherwise None.
+  std::optional<int> tryReplayCachedResult(CompilerInstance &Clang);
+
+  /// Finish writing outputs from a computed result, after a cache miss.
+  ///
+  /// \returns true if finished successfully.
+  bool finishComputedResult(CompilerInstance &Clang, bool Success);
+
+  class CachingOutputs;
+
+private:
+  int reportCachingBackendError(DiagnosticsEngine &Diag, llvm::Error &&E);
+
+  bool CacheCompileJob = false;
+  bool DisableCachedCompileJobReplay = false;
+
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> Cache;
+  llvm::Optional<llvm::cas::CASID> ResultCacheKey;
+
+  std::unique_ptr<CachingOutputs> CacheBackend;
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_FRONTEND_COMPILEJOBCACHE_H

clang/include/clang/Frontend/CompilerInstance.h

@@ -25,6 +25,7 @@
 #include "llvm/CAS/CASOutputBackend.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <cassert>
 #include <list>
@@ -94,6 +95,9 @@ class CompilerInstance : public ModuleLoader {
   /// The \c ActionCache key for this compilation, if caching is enabled.
   Optional<cas::CASID> CompileJobCacheKey;
 
+  /// The prefix mapper; empty by default.
+  llvm::PrefixMapper PrefixMapper;
+
   /// The file manager.
   IntrusiveRefCntPtr<FileManager> FileMgr;
 
@@ -362,6 +366,10 @@ class CompilerInstance : public ModuleLoader {
   }
   bool isSourceNonReproducible() const;
 
+  llvm::PrefixMapper &getPrefixMapper() { return PrefixMapper; }
+
+  void setPrefixMapper(llvm::PrefixMapper PM) { PrefixMapper = std::move(PM); }
+
   /// }
   /// @name Diagnostics Engine
   /// {

clang/include/clang/Frontend/FrontendOptions.h

@@ -375,6 +375,10 @@ class FrontendOptions {
   /// caching of compilation outputs. This is used for testing purposes.
   unsigned DisableCachedCompileJobReplay : 1;
 
+  /// Keep the diagnostic client open for receiving diagnostics after the source
+  /// files have been processed.
+  unsigned MayEmitDiagnosticsAfterProcessingSourceFiles : 1;
+
   /// Output (and read) PCM files regardless of compiler errors.
   unsigned AllowPCMWithCompilerErrors : 1;
 
@@ -570,7 +574,9 @@ class FrontendOptions {
         ASTDumpLookups(false), BuildingImplicitModule(false),
         BuildingImplicitModuleUsesLock(true), ModulesEmbedAllFiles(false),
         IncludeTimestamps(true), UseTemporary(true), CacheCompileJob(false),
-        DisableCachedCompileJobReplay(false), AllowPCMWithCompilerErrors(false),
+        DisableCachedCompileJobReplay(false),
+        MayEmitDiagnosticsAfterProcessingSourceFiles(false),
+        AllowPCMWithCompilerErrors(false),
         ModulesShareFileManager(true), TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file

clang/include/clang/Lex/DependencyDirectivesScanner.h

@@ -68,6 +68,7 @@ enum DirectiveKind : uint8_t {
   pp_pragma_push_macro,
   pp_pragma_pop_macro,
   pp_pragma_include_alias,
+  pp_pragma_system_header,
   pp_include_next,
   pp_if,
   pp_ifdef,

clang/include/clang/Lex/HeaderSearch.h

@@ -653,9 +653,13 @@ class HeaderSearch {
 
   /// Retrieve all the modules corresponding to the given file.
   ///
+  /// \param AllowCreation Whether to allow inference of a new submodule, or to
+  ///        only return existing known modules.
+  ///
   /// \ref findModuleForHeader should typically be used instead of this.
   ArrayRef<ModuleMap::KnownHeader>
-  findAllModulesForHeader(const FileEntry *File) const;
+  findAllModulesForHeader(const FileEntry *File,
+                          bool AllowCreation = true) const;
 
   /// Read the contents of the given module map file.
   ///

clang/include/clang/Lex/HeaderSearchOptions.h

@@ -216,6 +216,9 @@ class HeaderSearchOptions {
 
   unsigned ModulesValidateDiagnosticOptions : 1;
 
+  /// Whether to entirely skip writing diagnostic options.
+  unsigned ModulesSkipDiagnosticOptions : 1;
+
   unsigned ModulesHashContent : 1;
 
   /// Whether we should include all things that could impact the module in the
@@ -234,7 +237,8 @@ class HeaderSearchOptions {
         ModulesValidateOncePerBuildSession(false),
         ModulesValidateSystemHeaders(false),
         ValidateASTInputFilesContent(false), UseDebugInfo(false),
-        ModulesValidateDiagnosticOptions(true), ModulesHashContent(false),
+        ModulesValidateDiagnosticOptions(true),
+        ModulesSkipDiagnosticOptions(false), ModulesHashContent(false),
         ModulesStrictContextHash(false) {}
 
   /// AddPath - Add the \p Path path to the specified \p Group list.

clang/include/clang/Lex/ModuleMap.h

@@ -458,9 +458,13 @@ class ModuleMap {
   /// and does not consult the external source. (Those checks are the
   /// responsibility of \ref HeaderSearch.)
   ///
+  /// \param AllowCreation Whether to allow inference of a new submodule, or to
+  ///        only return existing known modules.
+  ///
   /// Typically, \ref findModuleForHeader should be used instead, as it picks
   /// the preferred module for the header.
-  ArrayRef<KnownHeader> findAllModulesForHeader(const FileEntry *File);
+  ArrayRef<KnownHeader> findAllModulesForHeader(const FileEntry *File,
+                                                bool AllowCreation = true);
 
   /// Like \ref findAllModulesForHeader, but do not attempt to infer module
   /// ownership from umbrella headers if we've not already done so.

clang/include/clang/Lex/Pragma.h

@@ -123,6 +123,13 @@ class PragmaNamespace : public PragmaHandler {
   PragmaNamespace *getIfNamespace() override { return this; }
 };
 
+/// Destringize a \c _Pragma("") string according to C11 6.10.9.1:
+/// "The string literal is destringized by deleting any encoding prefix,
+/// deleting the leading and trailing double-quotes, replacing each escape
+/// sequence \" by a double-quote, and replacing each escape sequence \\ by a
+/// single backslash."
+void prepare_PragmaString(SmallVectorImpl<char> &StrVal);
+
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_PRAGMA_H

clang/include/clang/Serialization/ASTReader.h

@@ -961,8 +961,14 @@ class ASTReader
 
 private:
   /// A list of modules that were imported by precompiled headers or
-  /// any other non-module AST file.
-  SmallVector<ImportedSubmodule, 2> ImportedModules;
+  /// any other non-module AST file and have not yet been made visible. If a
+  /// module is made visible in the ASTReader, it will be transfered to
+  /// \c PendingImportedModulesSema.
+  SmallVector<ImportedSubmodule, 2> PendingImportedModules;
+
+  /// A list of modules that were imported by precompiled headers or
+  /// any other non-module AST file and have not yet been made visible for Sema.
+  SmallVector<ImportedSubmodule, 2> PendingImportedModulesSema;
   //@}
 
   /// The system include root to be used when loading the

clang/include/clang/Serialization/PCHContainerOperations.h

@@ -56,7 +56,7 @@ class PCHContainerReader {
 public:
   virtual ~PCHContainerReader() = 0;
   /// Equivalent to the format passed to -fmodule-format=
-  virtual llvm::StringRef getFormat() const = 0;
+  virtual llvm::ArrayRef<llvm::StringRef> getFormats() const = 0;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   virtual llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const = 0;
@@ -78,16 +78,17 @@ class RawPCHContainerWriter : public PCHContainerWriter {
 
 /// Implements read operations for a raw pass-through PCH container.
 class RawPCHContainerReader : public PCHContainerReader {
-  llvm::StringRef getFormat() const override { return "raw"; }
-
+  llvm::ArrayRef<llvm::StringRef> getFormats() const override;
   /// Simply returns the buffer contained in Buffer.
   llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
 };
 
 /// A registry of PCHContainerWriter and -Reader objects for different formats.
 class PCHContainerOperations {
   llvm::StringMap<std::unique_ptr<PCHContainerWriter>> Writers;
-  llvm::StringMap<std::unique_ptr<PCHContainerReader>> Readers;
+  llvm::StringMap<PCHContainerReader *> Readers;
+  llvm::SmallVector<std::unique_ptr<PCHContainerReader>> OwnedReaders;
+
 public:
   /// Automatically registers a RawPCHContainerWriter and
   /// RawPCHContainerReader.
@@ -96,13 +97,17 @@ class PCHContainerOperations {
     Writers[Writer->getFormat()] = std::move(Writer);
   }
   void registerReader(std::unique_ptr<PCHContainerReader> Reader) {
-    Readers[Reader->getFormat()] = std::move(Reader);
+    assert(!Reader->getFormats().empty() &&
+           "PCHContainerReader must handle >=1 format");
+    for (llvm::StringRef Fmt : Reader->getFormats())
+      Readers[Fmt] = Reader.get();
+    OwnedReaders.push_back(std::move(Reader));
   }
   const PCHContainerWriter *getWriterOrNull(llvm::StringRef Format) {
     return Writers[Format].get();
   }
   const PCHContainerReader *getReaderOrNull(llvm::StringRef Format) {
-    return Readers[Format].get();
+    return Readers[Format];
   }
   const PCHContainerReader &getRawReader() {
     return *getReaderOrNull("raw");

clang/lib/Basic/Diagnostic.cpp

@@ -184,6 +184,18 @@ void DiagnosticsEngine::ReportDelayed() {
   Report(ID) << DelayedDiagArg1 << DelayedDiagArg2 << DelayedDiagArg3;
 }
 
+DiagnosticMapping &
+DiagnosticsEngine::DiagState::getOrAddMapping(diag::kind Diag) {
+  std::pair<iterator, bool> Result =
+      DiagMap.insert(std::make_pair(Diag, DiagnosticMapping()));
+
+  // Initialize the entry if we added it.
+  if (Result.second)
+    Result.first->second = DiagnosticIDs::getDefaultMapping(Diag);
+
+  return Result.first->second;
+}
+
 void DiagnosticsEngine::DiagStateMap::appendFirst(DiagState *State) {
   assert(Files.empty() && "not first");
   FirstDiagState = CurDiagState = State;