Merge branch 'sycl' into private/vlazarev/deps_docs

Author: vladimirlaz

README.md

@@ -4,23 +4,25 @@
 
 Intel staging area for llvm.org contribution.
 Home for Intel LLVM-based projects:
- - oneAPI Data Parallel C++ compiler - see **sycl** branch. More information on
+
+* oneAPI Data Parallel C++ compiler - see **sycl** branch. More information on
    oneAPI and DPC++ is available at
 [https://www.oneapi.com/](https://www.oneapi.com/)
-   - [![Linux Post Commit Checks](https://github.com/intel/llvm/workflows/Linux%20Post%20Commit%20Checks/badge.svg)](https://github.com/intel/llvm/actions?query=workflow%3A%22Linux+Post+Commit+Checks%22)
-   [![Generate Doxygen documentation](https://github.com/intel/llvm/workflows/Generate%20Doxygen%20documentation/badge.svg)](https://github.com/intel/llvm/actions?query=workflow%3A%22Generate+Doxygen+documentation%22)
-
-
+  * [![Linux Post Commit Checks](https://github.com/intel/llvm/workflows/Linux%20Post%20Commit%20Checks/badge.svg)](https://github.com/intel/llvm/actions?query=workflow%3A%22Linux+Post+Commit+Checks%22)
+    [![Generate Doxygen documentation](https://github.com/intel/llvm/workflows/Generate%20Doxygen%20documentation/badge.svg)](https://github.com/intel/llvm/actions?query=workflow%3A%22Generate+Doxygen+documentation%22)
 
 ## License
+
 See [LICENSE.txt](sycl/LICENSE.TXT) for details.
 
 ## Contributing
+
 See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
 
 ## Sub-projects Documentation
- - oneAPI Data Parallel C++ compiler - See
-   [GetStartedGuide.md](sycl/doc/GetStartedGuide.md)
+
+* oneAPI Data Parallel C++ compiler - See
+  [GetStartedGuide.md](sycl/doc/GetStartedGuide.md)
 
 ## DPC++ extensions
 
@@ -29,4 +31,3 @@ SYCL\* standards. DPC++ extends these standards with a number of extensions,
 which can be found in [sycl/doc/extensions](sycl/doc/extensions) directory.
 
 \*Other names and brands may be claimed as the property of others.
-

clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp

@@ -207,6 +207,8 @@ class NameLookup {
 /// flag indicating the multiple resolutions.
 NameLookup findDeclInBases(const CXXRecordDecl &Parent, StringRef DeclName,
                            bool AggressiveTemplateLookup) {
+  if (!Parent.hasDefinition())
+    return NameLookup(nullptr);
   if (const NamedDecl *InClassRef = findDecl(Parent, DeclName))
     return NameLookup(InClassRef);
   const NamedDecl *Found = nullptr;

clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.cpp

@@ -112,6 +112,12 @@ void TransformerClangTidyCheck::check(
   }
 }
 
+void TransformerClangTidyCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "IncludeStyle", IncludeStyle,
+                IncludeSorter::getMapping());
+}
+
 } // namespace utils
 } // namespace tidy
 } // namespace clang

clang-tools-extra/clang-tidy/utils/TransformerClangTidyCheck.h

@@ -67,6 +67,10 @@ class TransformerClangTidyCheck : public ClangTidyCheck {
   void registerMatchers(ast_matchers::MatchFinder *Finder) final;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) final;
 
+  /// Derived classes that override this function should call this method from
+  /// the overridden method.
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
 private:
   Optional<transformer::RewriteRule> Rule;
   const IncludeSorter::IncludeStyle IncludeStyle;

clang-tools-extra/clangd/CodeComplete.cpp

@@ -1777,7 +1777,7 @@ SignatureHelp signatureHelp(PathRef FileName,
                             const SymbolIndex *Index) {
   auto Offset = positionToOffset(Contents, Pos);
   if (!Offset) {
-    elog("Code completion position was invalid {0}", Offset.takeError());
+    elog("Signature help position was invalid {0}", Offset.takeError());
     return SignatureHelp();
   }
   SignatureHelp Result;

clang-tools-extra/clangd/Compiler.cpp

@@ -82,6 +82,10 @@ buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
   CI->getPreprocessorOpts().PCHThroughHeader.clear();
   CI->getPreprocessorOpts().PCHWithHdrStop = false;
   CI->getPreprocessorOpts().PCHWithHdrStopCreate = false;
+
+  // Recovery expression currently only works for C++.
+  if (CI->getLangOpts()->CPlusPlus)
+    CI->getLangOpts()->RecoveryAST = Inputs.Opts.BuildRecoveryAST;
   return CI;
 }
 

clang-tools-extra/clangd/Diagnostics.cpp

@@ -124,7 +124,9 @@ Range diagnosticRange(const clang::Diagnostic &D, const LangOptions &L) {
 bool adjustDiagFromHeader(Diag &D, const clang::Diagnostic &Info,
                           const LangOptions &LangOpts) {
   // We only report diagnostics with at least error severity from headers.
-  if (D.Severity < DiagnosticsEngine::Level::Error)
+  // Use default severity to avoid noise with -Werror.
+  if (!Info.getDiags()->getDiagnosticIDs()->isDefaultMappingAsError(
+          Info.getID()))
     return false;
 
   const SourceManager &SM = Info.getSourceManager();
@@ -514,7 +516,8 @@ void StoreDiags::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
   if (Info.getLocation().isInvalid()) {
     // Handle diagnostics coming from command-line arguments. The source manager
     // is *not* available at this point, so we cannot use it.
-    if (DiagLevel < DiagnosticsEngine::Level::Error) {
+    if (!Info.getDiags()->getDiagnosticIDs()->isDefaultMappingAsError(
+            Info.getID())) {
       IgnoreDiagnostics::log(DiagLevel, Info);
       return; // non-errors add too much noise, do not show them.
     }

clang-tools-extra/clangd/ParsedAST.cpp

@@ -262,9 +262,6 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   const PrecompiledPreamble *PreamblePCH =
       Preamble ? &Preamble->Preamble : nullptr;
 
-  // Recovery expression currently only works for C++.
-  if (CI->getLangOpts()->CPlusPlus)
-    CI->getLangOpts()->RecoveryAST = Inputs.Opts.BuildRecoveryAST;
   // This is on-by-default in windows to allow parsing SDK headers, but it
   // breaks many features. Disable it for the main-file (not preamble).
   CI->getLangOpts()->DelayedTemplateParsing = false;
@@ -314,18 +311,12 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
         std::string CheckName = CTContext->getCheckName(Info.getID());
         bool IsClangTidyDiag = !CheckName.empty();
         if (IsClangTidyDiag) {
-          // Check for warning-as-error.
-          // We deliberately let this take precedence over suppression comments
-          // to match clang-tidy's behaviour.
-          if (DiagLevel == DiagnosticsEngine::Warning &&
-              CTContext->treatAsError(CheckName)) {
-            return DiagnosticsEngine::Error;
-          }
-
           // Check for suppression comment. Skip the check for diagnostics not
           // in the main file, because we don't want that function to query the
           // source buffer for preamble files. For the same reason, we ask
           // shouldSuppressDiagnostic to avoid I/O.
+          // We let suppression comments take precedence over warning-as-error
+          // to match clang-tidy's behaviour.
           bool IsInsideMainFile =
               Info.hasSourceManager() &&
               isInsideMainFile(Info.getLocation(), Info.getSourceManager());
@@ -334,6 +325,12 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
                                              /*AllowIO=*/false)) {
             return DiagnosticsEngine::Ignored;
           }
+
+          // Check for warning-as-error.
+          if (DiagLevel == DiagnosticsEngine::Warning &&
+              CTContext->treatAsError(CheckName)) {
+            return DiagnosticsEngine::Error;
+          }
         }
       }
       return DiagLevel;

clang-tools-extra/clangd/Preamble.cpp

@@ -219,10 +219,6 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   // to read back. We rely on dynamic index for the comments instead.
   CI.getPreprocessorOpts().WriteCommentListToPCH = false;
 
-  // Recovery expression currently only works for C++.
-  if (CI.getLangOpts()->CPlusPlus)
-    CI.getLangOpts()->RecoveryAST = Inputs.Opts.BuildRecoveryAST;
-
   CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);
   if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
     log("Couldn't set working directory when building the preamble.");

clang-tools-extra/clangd/Selection.cpp

@@ -9,6 +9,7 @@
 #include "Selection.h"
 #include "SourceCode.h"
 #include "support/Logger.h"
+#include "support/Trace.h"
 #include "clang/AST/ASTTypeTraits.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
@@ -35,6 +36,21 @@ namespace {
 using Node = SelectionTree::Node;
 using ast_type_traits::DynTypedNode;
 
+// Measure the fraction of selections that were enabled by recovery AST.
+void recordMetrics(const SelectionTree &S) {
+  static constexpr trace::Metric SelectionUsedRecovery(
+      "selection_recovery", trace::Metric::Distribution);
+  const auto *Common = S.commonAncestor();
+  for (const auto *N = Common; N; N = N->Parent) {
+    if (N->ASTNode.get<RecoveryExpr>()) {
+      SelectionUsedRecovery.record(1); // used recovery ast.
+      return;
+    }
+  }
+  if (Common)
+    SelectionUsedRecovery.record(0); // unused.
+}
+
 // An IntervalSet maintains a set of disjoint subranges of an array.
 //
 // Initially, it contains the entire array.
@@ -774,6 +790,7 @@ SelectionTree::SelectionTree(ASTContext &AST, const syntax::TokenBuffer &Tokens,
            .printToString(SM));
   Nodes = SelectionVisitor::collect(AST, Tokens, PrintPolicy, Begin, End, FID);
   Root = Nodes.empty() ? nullptr : &Nodes.front();
+  recordMetrics(*this);
   dlog("Built selection tree\n{0}", *this);
 }
 

clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp

@@ -83,6 +83,12 @@ static void DexQueries(benchmark::State &State) {
 }
 BENCHMARK(DexQueries);
 
+static void DexBuild(benchmark::State &State) {
+  for (auto _ : State)
+    buildDex();
+}
+BENCHMARK(DexBuild);
+
 } // namespace
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/index/dex/Dex.cpp

@@ -39,28 +39,60 @@ namespace {
 const Token RestrictedForCodeCompletion =
     Token(Token::Kind::Sentinel, "Restricted For Code Completion");
 
-// Returns the tokens which are given symbol's characteristics. Currently, the
-// generated tokens only contain fuzzy matching trigrams and symbol's scope,
-// but in the future this will also return path proximity tokens and other
-// types of tokens such as symbol type (if applicable).
-// Returns the tokens which are given symbols's characteristics. For example,
-// trigrams and scopes.
-// FIXME(kbobyrev): Support more token types:
-// * Namespace proximity
-std::vector<Token> generateSearchTokens(const Symbol &Sym) {
-  std::vector<Token> Result = generateIdentifierTrigrams(Sym.Name);
-  Result.emplace_back(Token::Kind::Scope, Sym.Scope);
-  // Skip token generation for symbols with unknown declaration location.
-  if (!llvm::StringRef(Sym.CanonicalDeclaration.FileURI).empty())
-    for (const auto &ProximityURI :
-         generateProximityURIs(Sym.CanonicalDeclaration.FileURI))
-      Result.emplace_back(Token::Kind::ProximityURI, ProximityURI);
-  if (Sym.Flags & Symbol::IndexedForCodeCompletion)
-    Result.emplace_back(RestrictedForCodeCompletion);
-  if (!Sym.Type.empty())
-    Result.emplace_back(Token::Kind::Type, Sym.Type);
-  return Result;
-}
+// Helper to efficiently assemble the inverse index (token -> matching docs).
+// The output is a nice uniform structure keyed on Token, but constructing
+// the Token object every time we want to insert into the map is wasteful.
+// Instead we have various maps keyed on things that are cheap to compute,
+// and produce the Token keys once at the end.
+class IndexBuilder {
+  llvm::DenseMap<Trigram, std::vector<DocID>> TrigramDocs;
+  std::vector<DocID> RestrictedCCDocs;
+  llvm::StringMap<std::vector<DocID>> TypeDocs;
+  llvm::StringMap<std::vector<DocID>> ScopeDocs;
+  llvm::StringMap<std::vector<DocID>> ProximityDocs;
+  std::vector<Trigram> TrigramScratch;
+
+public:
+  // Add the tokens which are given symbol's characteristics.
+  // This includes fuzzy matching trigrams, symbol's scope, etc.
+  // FIXME(kbobyrev): Support more token types:
+  // * Namespace proximity
+  void add(const Symbol &Sym, DocID D) {
+    generateIdentifierTrigrams(Sym.Name, TrigramScratch);
+    for (Trigram T : TrigramScratch)
+      TrigramDocs[T].push_back(D);
+    ScopeDocs[Sym.Scope].push_back(D);
+    if (!llvm::StringRef(Sym.CanonicalDeclaration.FileURI).empty())
+      for (const auto &ProximityURI :
+           generateProximityURIs(Sym.CanonicalDeclaration.FileURI))
+        ProximityDocs[ProximityURI].push_back(D);
+    if (Sym.Flags & Symbol::IndexedForCodeCompletion)
+      RestrictedCCDocs.push_back(D);
+    if (!Sym.Type.empty())
+      TypeDocs[Sym.Type].push_back(D);
+  }
+
+  // Assemble the final compressed posting lists for the added symbols.
+  llvm::DenseMap<Token, PostingList> build() {
+    llvm::DenseMap<Token, PostingList> Result(/*InitialReserve=*/
+                                              TrigramDocs.size() +
+                                              RestrictedCCDocs.size() +
+                                              TypeDocs.size() +
+                                              ScopeDocs.size() +
+                                              ProximityDocs.size());
+    for (const auto &E : TrigramDocs)
+      Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()), E.second);
+    for (const auto &E : TypeDocs)
+      Result.try_emplace(Token(Token::Kind::Type, E.first()), E.second);
+    for (const auto &E : ScopeDocs)
+      Result.try_emplace(Token(Token::Kind::Scope, E.first()), E.second);
+    for (const auto &E : ProximityDocs)
+      Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()), E.second);
+    if (!RestrictedCCDocs.empty())
+      Result.try_emplace(RestrictedForCodeCompletion, RestrictedCCDocs);
+    return Result;
+  }
+};
 
 } // namespace
 
@@ -86,18 +118,11 @@ void Dex::buildIndex() {
     Symbols[I] = ScoredSymbols[I].second;
   }
 
-  // Populate TempInvertedIndex with lists for index symbols.
-  llvm::DenseMap<Token, std::vector<DocID>> TempInvertedIndex;
-  for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) {
-    const auto *Sym = Symbols[SymbolRank];
-    for (const auto &Token : generateSearchTokens(*Sym))
-      TempInvertedIndex[Token].push_back(SymbolRank);
-  }
-
-  // Convert lists of items to posting lists.
-  for (const auto &TokenToPostingList : TempInvertedIndex)
-    InvertedIndex.insert(
-        {TokenToPostingList.first, PostingList(TokenToPostingList.second)});
+  // Build posting lists for symbols.
+  IndexBuilder Builder;
+  for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
+    Builder.add(*Symbols[SymbolRank], SymbolRank);
+  InvertedIndex = Builder.build();
 }
 
 std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {