Merge from 'master' to 'sycl-web' (#5)

  CONFLICT (content): Merge conflict in clang/lib/Basic/IdentifierTable.cpp

Author: bader

clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp

@@ -456,7 +456,7 @@ void ProTypeMemberInitCheck::checkMissingMemberInitializer(
     // Don't suggest fixes for bitfields because in-class initialization is not
     // possible until C++2a.
     if (F->getType()->isEnumeralType() ||
-        (!getLangOpts().CPlusPlus2a && F->isBitField()))
+        (!getLangOpts().CPlusPlus20 && F->isBitField()))
       return;
     if (!F->getParent()->isUnion() || UnionsSeen.insert(F->getParent()).second)
       FieldsToFix.insert(F);

clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp

@@ -217,7 +217,7 @@ void UseDefaultMemberInitCheck::registerMatchers(MatchFinder *Finder) {
           isDefaultConstructor(), unless(isInstantiated()),
           forEachConstructorInitializer(
               cxxCtorInitializer(
-                  forField(unless(anyOf(getLangOpts().CPlusPlus2a
+                  forField(unless(anyOf(getLangOpts().CPlusPlus20
                                             ? unless(anything())
                                             : isBitField(),
                                         hasInClassInitializer(anything()),

clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp

@@ -96,7 +96,7 @@ void SIMDIntrinsicsCheck::registerMatchers(MatchFinder *Finder) {
   // If Std is not specified, infer it from the language options.
   // libcxx implementation backports it to C++11 std::experimental::simd.
   if (Std.empty())
-    Std = getLangOpts().CPlusPlus2a ? "std" : "std::experimental";
+    Std = getLangOpts().CPlusPlus20 ? "std" : "std::experimental";
 
   Finder->addMatcher(callExpr(callee(functionDecl(
                                   matchesName("^::(_mm_|_mm256_|_mm512_|vec_)"),

clang-tools-extra/clangd/ClangdServer.cpp

@@ -280,11 +280,8 @@ void ClangdServer::signatureHelp(PathRef File, Position Pos,
                              Pos, FS, Index));
   };
 
-  // Unlike code completion, we wait for an up-to-date preamble here.
-  // Signature help is often triggered after code completion. If the code
-  // completion inserted a header to make the symbol available, then using
-  // the old preamble would yield useless results.
-  WorkScheduler.runWithPreamble("SignatureHelp", File, TUScheduler::Consistent,
+  // Unlike code completion, we wait for a preamble here.
+  WorkScheduler.runWithPreamble("SignatureHelp", File, TUScheduler::Stale,
                                 std::move(Action));
 }
 

clang-tools-extra/clangd/CodeComplete.cpp

@@ -1023,6 +1023,7 @@ struct SemaCompleteInput {
   PathRef FileName;
   const tooling::CompileCommand &Command;
   const PreambleData &Preamble;
+  const PreamblePatch &Patch;
   llvm::StringRef Contents;
   size_t Offset;
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS;
@@ -1060,7 +1061,6 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   ParseInput.CompileCommand = Input.Command;
   ParseInput.FS = VFS;
   ParseInput.Contents = std::string(Input.Contents);
-  ParseInput.Opts = ParseOptions();
 
   IgnoreDiagnostics IgnoreDiags;
   auto CI = buildCompilerInvocation(ParseInput, IgnoreDiags);
@@ -1096,6 +1096,7 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   PreambleBounds PreambleRegion =
       ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
   bool CompletingInPreamble = PreambleRegion.Size > Input.Offset;
+  Input.Patch.apply(*CI);
   // NOTE: we must call BeginSourceFile after prepareCompilerInstance. Otherwise
   // the remapped buffers do not get freed.
   auto Clang = prepareCompilerInstance(
@@ -1754,8 +1755,10 @@ codeComplete(PathRef FileName, const tooling::CompileCommand &Command,
       SpecFuzzyFind, Opts);
   return (!Preamble || Opts.RunParser == CodeCompleteOptions::NeverParse)
              ? std::move(Flow).runWithoutSema(Contents, *Offset, VFS)
-             : std::move(Flow).run(
-                   {FileName, Command, *Preamble, Contents, *Offset, VFS});
+             : std::move(Flow).run({FileName, Command, *Preamble,
+                                    // We want to serve code completions with
+                                    // low latency, so don't bother patching.
+                                    PreamblePatch(), Contents, *Offset, VFS});
 }
 
 SignatureHelp signatureHelp(PathRef FileName,
@@ -1775,10 +1778,15 @@ SignatureHelp signatureHelp(PathRef FileName,
   Options.IncludeMacros = false;
   Options.IncludeCodePatterns = false;
   Options.IncludeBriefComments = false;
-  IncludeStructure PreambleInclusions; // Unused for signatureHelp
+
+  ParseInputs PI;
+  PI.CompileCommand = Command;
+  PI.Contents = Contents.str();
+  PI.FS = std::move(VFS);
+  auto PP = PreamblePatch::create(FileName, PI, Preamble);
   semaCodeComplete(
       std::make_unique<SignatureHelpCollector>(Options, Index, Result), Options,
-      {FileName, Command, Preamble, Contents, *Offset, std::move(VFS)});
+      {FileName, Command, Preamble, PP, Contents, *Offset, std::move(PI.FS)});
   return Result;
 }
 

clang-tools-extra/clangd/Diagnostics.cpp

@@ -556,10 +556,23 @@ void StoreDiags::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
     if (!InsideMainFile)
       return false;
 
+    // Copy as we may modify the ranges.
+    auto FixIts = Info.getFixItHints().vec();
     llvm::SmallVector<TextEdit, 1> Edits;
-    for (auto &FixIt : Info.getFixItHints()) {
-      // Follow clang's behavior, don't apply FixIt to the code in macros,
-      // we are less certain it is the right fix.
+    for (auto &FixIt : FixIts) {
+      // Allow fixits within a single macro-arg expansion to be applied.
+      // This can be incorrect if the argument is expanded multiple times in
+      // different contexts. Hopefully this is rare!
+      if (FixIt.RemoveRange.getBegin().isMacroID() &&
+          FixIt.RemoveRange.getEnd().isMacroID() &&
+          SM.getFileID(FixIt.RemoveRange.getBegin()) ==
+              SM.getFileID(FixIt.RemoveRange.getEnd())) {
+        FixIt.RemoveRange = CharSourceRange(
+            {SM.getTopMacroCallerLoc(FixIt.RemoveRange.getBegin()),
+             SM.getTopMacroCallerLoc(FixIt.RemoveRange.getEnd())},
+            FixIt.RemoveRange.isTokenRange());
+      }
+      // Otherwise, follow clang's behavior: no fixits in macros.
       if (FixIt.RemoveRange.getBegin().isMacroID() ||
           FixIt.RemoveRange.getEnd().isMacroID())
         return false;
@@ -570,8 +583,8 @@ void StoreDiags::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
 
     llvm::SmallString<64> Message;
     // If requested and possible, create a message like "change 'foo' to 'bar'".
-    if (SyntheticMessage && Info.getNumFixItHints() == 1) {
-      const auto &FixIt = Info.getFixItHint(0);
+    if (SyntheticMessage && FixIts.size() == 1) {
+      const auto &FixIt = FixIts.front();
       bool Invalid = false;
       llvm::StringRef Remove =
           Lexer::getSourceText(FixIt.RemoveRange, SM, *LangOpts, &Invalid);

clang-tools-extra/clangd/Headers.cpp

@@ -28,7 +28,7 @@ class RecordHeaders : public PPCallbacks {
 
   // Record existing #includes - both written and resolved paths. Only #includes
   // in the main file are collected.
-  void InclusionDirective(SourceLocation HashLoc, const Token & /*IncludeTok*/,
+  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
                           llvm::StringRef FileName, bool IsAngled,
                           CharSourceRange FilenameRange, const FileEntry *File,
                           llvm::StringRef /*SearchPath*/,
@@ -44,6 +44,7 @@ class RecordHeaders : public PPCallbacks {
       Inc.Resolved = std::string(File ? File->tryGetRealPathName() : "");
       Inc.HashOffset = SM.getFileOffset(HashLoc);
       Inc.FileKind = FileKind;
+      Inc.Directive = IncludeTok.getIdentifierInfo()->getPPKeywordID();
     }
     if (File) {
       auto *IncludingFileEntry = SM.getFileEntryForID(SM.getFileID(HashLoc));

clang-tools-extra/clangd/Headers.h

@@ -13,6 +13,7 @@
 #include "Protocol.h"
 #include "SourceCode.h"
 #include "index/Symbol.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Format/Format.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/PPCallbacks.h"
@@ -22,6 +23,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/VirtualFileSystem.h"
+#include <string>
 
 namespace clang {
 namespace clangd {
@@ -50,9 +52,10 @@ llvm::SmallVector<llvm::StringRef, 1> getRankedIncludes(const Symbol &Sym);
 
 // An #include directive that we found in the main file.
 struct Inclusion {
-  Range R;             // Inclusion range.
-  std::string Written; // Inclusion name as written e.g. <vector>.
-  Path Resolved;       // Resolved path of included file. Empty if not resolved.
+  Range R;                      // Inclusion range.
+  tok::PPKeywordKind Directive; // Directive used for inclusion, e.g. import
+  std::string Written;          // Inclusion name as written e.g. <vector>.
+  Path Resolved; // Resolved path of included file. Empty if not resolved.
   unsigned HashOffset = 0; // Byte offset from start of file to #.
   SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User;
 };

clang-tools-extra/clangd/Preamble.cpp

@@ -8,11 +8,39 @@
 
 #include "Preamble.h"
 #include "Compiler.h"
+#include "Headers.h"
 #include "Logger.h"
 #include "Trace.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+#include <iterator>
+#include <memory>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -74,6 +102,81 @@ class CppFilePreambleCallbacks : public PreambleCallbacks {
   const SourceManager *SourceMgr = nullptr;
 };
 
+// Runs preprocessor over preamble section.
+class PreambleOnlyAction : public PreprocessorFrontendAction {
+protected:
+  void ExecuteAction() override {
+    Preprocessor &PP = getCompilerInstance().getPreprocessor();
+    auto &SM = PP.getSourceManager();
+    PP.EnterMainSourceFile();
+    auto Bounds = ComputePreambleBounds(getCompilerInstance().getLangOpts(),
+                                        SM.getBuffer(SM.getMainFileID()), 0);
+    Token Tok;
+    do {
+      PP.Lex(Tok);
+      assert(SM.isInMainFile(Tok.getLocation()));
+    } while (Tok.isNot(tok::eof) &&
+             SM.getDecomposedLoc(Tok.getLocation()).second < Bounds.Size);
+  }
+};
+
+/// Gets the includes in the preamble section of the file by running
+/// preprocessor over \p Contents. Returned includes do not contain resolved
+/// paths. \p VFS and \p Cmd is used to build the compiler invocation, which
+/// might stat/read files.
+llvm::Expected<std::vector<Inclusion>>
+scanPreambleIncludes(llvm::StringRef Contents,
+                     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
+                     const tooling::CompileCommand &Cmd) {
+  // Build and run Preprocessor over the preamble.
+  ParseInputs PI;
+  PI.Contents = Contents.str();
+  PI.FS = std::move(VFS);
+  PI.CompileCommand = Cmd;
+  IgnoringDiagConsumer IgnoreDiags;
+  auto CI = buildCompilerInvocation(PI, IgnoreDiags);
+  if (!CI)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "failed to create compiler invocation");
+  CI->getDiagnosticOpts().IgnoreWarnings = true;
+  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents);
+  auto Clang = prepareCompilerInstance(
+      std::move(CI), nullptr, std::move(ContentsBuffer),
+      // Provide an empty FS to prevent preprocessor from performing IO. This
+      // also implies missing resolved paths for includes.
+      new llvm::vfs::InMemoryFileSystem, IgnoreDiags);
+  if (Clang->getFrontendOpts().Inputs.empty())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "compiler instance had no inputs");
+  // We are only interested in main file includes.
+  Clang->getPreprocessorOpts().SingleFileParseMode = true;
+  PreambleOnlyAction Action;
+  if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "failed BeginSourceFile");
+  Preprocessor &PP = Clang->getPreprocessor();
+  IncludeStructure Includes;
+  PP.addPPCallbacks(
+      collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
+  if (llvm::Error Err = Action.Execute())
+    return std::move(Err);
+  Action.EndSourceFile();
+  return Includes.MainFileIncludes;
+}
+
+const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
+  switch (IncludeDirective) {
+  case tok::pp_include:
+    return "include";
+  case tok::pp_import:
+    return "import";
+  case tok::pp_include_next:
+    return "include_next";
+  default:
+    break;
+  }
+  llvm_unreachable("not an include directive");
+}
 } // namespace
 
 PreambleData::PreambleData(const ParseInputs &Inputs,
@@ -166,5 +269,78 @@ bool isPreambleCompatible(const PreambleData &Preamble,
          Preamble.Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds,
                                     Inputs.FS.get());
 }
+
+PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
+                                    const ParseInputs &Modified,
+                                    const PreambleData &Baseline) {
+  // First scan the include directives in Baseline and Modified. These will be
+  // used to figure out newly added directives in Modified. Scanning can fail,
+  // the code just bails out and creates an empty patch in such cases, as:
+  // - If scanning for Baseline fails, no knowledge of existing includes hence
+  //   patch will contain all the includes in Modified. Leading to rebuild of
+  //   whole preamble, which is terribly slow.
+  // - If scanning for Modified fails, cannot figure out newly added ones so
+  //   there's nothing to do but generate an empty patch.
+  auto BaselineIncludes = scanPreambleIncludes(
+      // Contents needs to be null-terminated.
+      Baseline.Preamble.getContents().str(),
+      Baseline.StatCache->getConsumingFS(Modified.FS), Modified.CompileCommand);
+  if (!BaselineIncludes) {
+    elog("Failed to scan includes for baseline of {0}: {1}", FileName,
+         BaselineIncludes.takeError());
+    return {};
+  }
+  auto ModifiedIncludes = scanPreambleIncludes(
+      Modified.Contents, Baseline.StatCache->getConsumingFS(Modified.FS),
+      Modified.CompileCommand);
+  if (!ModifiedIncludes) {
+    elog("Failed to scan includes for modified contents of {0}: {1}", FileName,
+         ModifiedIncludes.takeError());
+    return {};
+  }
+
+  PreamblePatch PP;
+  // This shouldn't coincide with any real file name.
+  llvm::SmallString<128> PatchName;
+  llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
+                          "__preamble_patch__.h");
+  PP.PatchFileName = PatchName.str().str();
+
+  // We are only interested in newly added includes, record the ones in Baseline
+  // for exclusion.
+  llvm::DenseSet<std::pair<tok::PPKeywordKind, llvm::StringRef>>
+      ExistingIncludes;
+  for (const auto &Inc : *BaselineIncludes)
+    ExistingIncludes.insert({Inc.Directive, Inc.Written});
+  // Calculate extra includes that needs to be inserted.
+  llvm::raw_string_ostream Patch(PP.PatchContents);
+  for (const auto &Inc : *ModifiedIncludes) {
+    if (ExistingIncludes.count({Inc.Directive, Inc.Written}))
+      continue;
+    Patch << llvm::formatv("#{0} {1}\n", spellingForIncDirective(Inc.Directive),
+                           Inc.Written);
+  }
+  Patch.flush();
+
+  // FIXME: Handle more directives, e.g. define/undef.
+  return PP;
+}
+
+void PreamblePatch::apply(CompilerInvocation &CI) const {
+  // No need to map an empty file.
+  if (PatchContents.empty())
+    return;
+  auto &PPOpts = CI.getPreprocessorOpts();
+  auto PatchBuffer =
+      // we copy here to ensure contents are still valid if CI outlives the
+      // PreamblePatch.
+      llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
+  // CI will take care of the lifetime of the buffer.
+  PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
+  // The patch will be parsed after loading the preamble ast and before parsing
+  // the main file.
+  PPOpts.Includes.push_back(PatchFileName);
+}
+
 } // namespace clangd
 } // namespace clang