Merge from 'master' to 'sycl-web' (#146)

  CONFLICT (content): Merge conflict in clang/include/clang/Basic/LangOptions.h

Author: bader

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -528,6 +528,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
   CCOpts.IncludeFixIts = Params.capabilities.CompletionFixes;
   if (!CCOpts.BundleOverloads.hasValue())
     CCOpts.BundleOverloads = Params.capabilities.HasSignatureHelp;
+  CCOpts.DocumentationFormat =
+      Params.capabilities.CompletionDocumentationFormat;
   DiagOpts.EmbedFixesInDiagnostics = Params.capabilities.DiagnosticFixes;
   DiagOpts.SendDiagnosticCategory = Params.capabilities.DiagnosticCategory;
   DiagOpts.EmitRelatedLocations =

clang-tools-extra/clangd/CodeComplete.cpp

@@ -26,6 +26,7 @@
 #include "FileDistance.h"
 #include "FuzzyMatch.h"
 #include "Headers.h"
+#include "Hover.h"
 #include "Preamble.h"
 #include "Protocol.h"
 #include "Quality.h"
@@ -371,12 +372,19 @@ struct CodeCompletionBuilder {
       S.SnippetSuffix = std::string(C.IndexResult->CompletionSnippetSuffix);
       S.ReturnType = std::string(C.IndexResult->ReturnType);
     }
-    if (ExtractDocumentation && Completion.Documentation.empty()) {
-      if (C.IndexResult)
-        Completion.Documentation = std::string(C.IndexResult->Documentation);
-      else if (C.SemaResult)
-        Completion.Documentation = getDocComment(*ASTCtx, *C.SemaResult,
-                                                 /*CommentsFromHeader=*/false);
+    if (ExtractDocumentation && !Completion.Documentation) {
+      auto SetDoc = [&](llvm::StringRef Doc) {
+        if (!Doc.empty()) {
+          Completion.Documentation.emplace();
+          parseDocumentation(Doc, *Completion.Documentation);
+        }
+      };
+      if (C.IndexResult) {
+        SetDoc(C.IndexResult->Documentation);
+      } else if (C.SemaResult) {
+        SetDoc(getDocComment(*ASTCtx, *C.SemaResult,
+                             /*CommentsFromHeader=*/false));
+      }
     }
   }
 
@@ -1816,6 +1824,21 @@ bool isIndexedForCodeCompletion(const NamedDecl &ND, ASTContext &ASTCtx) {
   return false;
 }
 
+// FIXME: find a home for this (that can depend on both markup and Protocol).
+static MarkupContent renderDoc(const markup::Document &Doc, MarkupKind Kind) {
+  MarkupContent Result;
+  Result.kind = Kind;
+  switch (Kind) {
+  case MarkupKind::PlainText:
+    Result.value.append(Doc.asPlainText());
+    break;
+  case MarkupKind::Markdown:
+    Result.value.append(Doc.asMarkdown());
+    break;
+  }
+  return Result;
+}
+
 CompletionItem CodeCompletion::render(const CodeCompleteOptions &Opts) const {
   CompletionItem LSP;
   const auto *InsertInclude = Includes.empty() ? nullptr : &Includes[0];
@@ -1830,9 +1853,16 @@ CompletionItem CodeCompletion::render(const CodeCompleteOptions &Opts) const {
                    ? std::string(llvm::formatv("[{0} overloads]", BundleSize))
                    : ReturnType;
   LSP.deprecated = Deprecated;
-  if (InsertInclude)
-    LSP.detail += "\n" + InsertInclude->Header;
-  LSP.documentation = Documentation;
+  // Combine header information and documentation in LSP `documentation` field.
+  // This is not quite right semantically, but tends to display well in editors.
+  if (InsertInclude || Documentation) {
+    markup::Document Doc;
+    if (InsertInclude)
+      Doc.addParagraph().appendText("From ").appendCode(InsertInclude->Header);
+    if (Documentation)
+      Doc.append(*Documentation);
+    LSP.documentation = renderDoc(Doc, Opts.DocumentationFormat);
+  }
   LSP.sortText = sortText(Score.Total, Name);
   LSP.filterText = Name;
   LSP.textEdit = {CompletionTokenRange, RequiredQualifier + Name};

clang-tools-extra/clangd/CodeComplete.h

@@ -15,6 +15,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETE_H
 
+#include "FormattedString.h"
 #include "Headers.h"
 #include "Protocol.h"
 #include "Quality.h"
@@ -73,6 +74,9 @@ struct CodeCompleteOptions {
   /// If more results are available, we set CompletionList.isIncomplete.
   size_t Limit = 0;
 
+  /// Whether to present doc comments as plain-text or markdown.
+  MarkupKind DocumentationFormat = MarkupKind::PlainText;
+
   enum IncludeInsertion {
     IWYU,
     NeverInsert,
@@ -161,7 +165,8 @@ struct CodeCompletion {
   std::string SnippetSuffix;
   // Type to be displayed for this completion.
   std::string ReturnType;
-  std::string Documentation;
+  // The parsed documentation comment.
+  llvm::Optional<markup::Document> Documentation;
   CompletionItemKind Kind = CompletionItemKind::Missing;
   // This completion item may represent several symbols that can be inserted in
   // the same way, such as function overloads. In this case BundleSize > 1, and

clang-tools-extra/clangd/FormattedString.cpp

@@ -271,6 +271,9 @@ class Ruler : public Block {
     OS << "\n---\n";
   }
   void renderPlainText(llvm::raw_ostream &OS) const override { OS << '\n'; }
+  std::unique_ptr<Block> clone() const override {
+    return std::make_unique<Ruler>(*this);
+  }
   bool isRuler() const override { return true; }
 };
 
@@ -287,6 +290,10 @@ class CodeBlock : public Block {
     OS << '\n' << Contents << "\n\n";
   }
 
+  std::unique_ptr<Block> clone() const override {
+    return std::make_unique<CodeBlock>(*this);
+  }
+
   CodeBlock(std::string Contents, std::string Language)
       : Contents(std::move(Contents)), Language(std::move(Language)) {}
 
@@ -358,10 +365,28 @@ void Paragraph::renderMarkdown(llvm::raw_ostream &OS) const {
   OS << "  \n";
 }
 
+std::unique_ptr<Block> Paragraph::clone() const {
+  return std::make_unique<Paragraph>(*this);
+}
+
+/// Choose a marker to delimit `Text` from a prioritized list of options.
+/// This is more readable than escaping for plain-text.
+llvm::StringRef chooseMarker(llvm::ArrayRef<llvm::StringRef> Options,
+                             llvm::StringRef Text) {
+  // Prefer a delimiter whose characters don't appear in the text.
+  for (llvm::StringRef S : Options)
+    if (Text.find_first_of(S) == llvm::StringRef::npos)
+      return S;
+  return Options.front();
+}
+
 void Paragraph::renderPlainText(llvm::raw_ostream &OS) const {
   llvm::StringRef Sep = "";
   for (auto &C : Chunks) {
-    OS << Sep << C.Contents;
+    llvm::StringRef Marker = "";
+    if (C.Preserve && C.Kind == Chunk::InlineCode)
+      Marker = chooseMarker({"`", "'", "\""}, C.Contents);
+    OS << Sep << Marker << C.Contents << Marker;
     Sep = " ";
   }
   OS << '\n';
@@ -396,22 +421,39 @@ Paragraph &Paragraph::appendText(llvm::StringRef Text) {
   return *this;
 }
 
-Paragraph &Paragraph::appendCode(llvm::StringRef Code) {
+Paragraph &Paragraph::appendCode(llvm::StringRef Code, bool Preserve) {
   std::string Norm = canonicalizeSpaces(std::move(Code));
   if (Norm.empty())
     return *this;
   Chunks.emplace_back();
   Chunk &C = Chunks.back();
   C.Contents = std::move(Norm);
   C.Kind = Chunk::InlineCode;
+  C.Preserve = Preserve;
   return *this;
 }
 
+std::unique_ptr<Block> BulletList::clone() const {
+  return std::make_unique<BulletList>(*this);
+}
+
 class Document &BulletList::addItem() {
   Items.emplace_back();
   return Items.back();
 }
 
+Document &Document::operator=(const Document &Other) {
+  Children.clear();
+  for (const auto &C : Other.Children)
+    Children.push_back(C->clone());
+  return *this;
+}
+
+void Document::append(Document Other) {
+  std::move(Other.Children.begin(), Other.Children.end(),
+            std::back_inserter(Children));
+}
+
 Paragraph &Document::addParagraph() {
   Children.push_back(std::make_unique<Paragraph>());
   return *static_cast<Paragraph *>(Children.back().get());

clang-tools-extra/clangd/FormattedString.h

@@ -30,6 +30,7 @@ class Block {
 public:
   virtual void renderMarkdown(llvm::raw_ostream &OS) const = 0;
   virtual void renderPlainText(llvm::raw_ostream &OS) const = 0;
+  virtual std::unique_ptr<Block> clone() const = 0;
   std::string asMarkdown() const;
   std::string asPlainText() const;
 
@@ -44,22 +45,24 @@ class Paragraph : public Block {
 public:
   void renderMarkdown(llvm::raw_ostream &OS) const override;
   void renderPlainText(llvm::raw_ostream &OS) const override;
+  std::unique_ptr<Block> clone() const override;
 
   /// Append plain text to the end of the string.
   Paragraph &appendText(llvm::StringRef Text);
 
   /// Append inline code, this translates to the ` block in markdown.
-  Paragraph &appendCode(llvm::StringRef Code);
+  /// \p Preserve indicates the code span must be apparent even in plaintext.
+  Paragraph &appendCode(llvm::StringRef Code, bool Preserve = false);
 
 private:
   struct Chunk {
     enum {
       PlainText,
       InlineCode,
     } Kind = PlainText;
+    // Preserve chunk markers in plaintext.
+    bool Preserve = false;
     std::string Contents;
-    /// Language for code block chunks. Ignored for other chunks.
-    std::string Language;
   };
   std::vector<Chunk> Chunks;
 };
@@ -70,6 +73,7 @@ class BulletList : public Block {
 public:
   void renderMarkdown(llvm::raw_ostream &OS) const override;
   void renderPlainText(llvm::raw_ostream &OS) const override;
+  std::unique_ptr<Block> clone() const override;
 
   class Document &addItem();
 
@@ -81,6 +85,14 @@ class BulletList : public Block {
 /// markdown and plaintext.
 class Document {
 public:
+  Document() = default;
+  Document(const Document &Other) { *this = Other; }
+  Document &operator=(const Document &);
+  Document(Document &&) = default;
+  Document &operator=(Document &&) = default;
+
+  void append(Document Other);
+
   /// Adds a semantical block that will be separate from others.
   Paragraph &addParagraph();
   /// Inserts a horizontal separator to the document.

clang-tools-extra/clangd/Hover.cpp

@@ -648,6 +648,7 @@ bool isHardLineBreakIndicator(llvm::StringRef Rest) {
 }
 
 bool isHardLineBreakAfter(llvm::StringRef Line, llvm::StringRef Rest) {
+  // Should we also consider whether Line is short?
   return punctuationIndicatesLineBreak(Line) || isHardLineBreakIndicator(Rest);
 }
 
@@ -876,7 +877,7 @@ void parseDocumentationLine(llvm::StringRef Line, markup::Paragraph &Out) {
       case '`':
         if (auto Range = getBacktickQuoteRange(Line, I)) {
           Out.appendText(Line.substr(0, I));
-          Out.appendCode(Range->trim("`"));
+          Out.appendCode(Range->trim("`"), /*Preserve=*/true);
           return parseDocumentationLine(Line.substr(I+Range->size()), Out);
         }
         break;

clang-tools-extra/clangd/Hover.h

@@ -80,6 +80,7 @@ struct HoverInfo {
 };
 
 // Try to infer structure of a documentation comment (e.g. line breaks).
+// FIXME: move to another file so CodeComplete doesn't depend on Hover.
 void parseDocumentation(llvm::StringRef Input, markup::Document &Output);
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const HoverInfo::Param &);

clang-tools-extra/clangd/Protocol.cpp

@@ -311,6 +311,12 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) {
       if (auto *Item = Completion->getObject("completionItem")) {
         if (auto SnippetSupport = Item->getBoolean("snippetSupport"))
           R.CompletionSnippets = *SnippetSupport;
+        if (auto DocumentationFormat = Item->getArray("documentationFormat")) {
+          for (const auto &Format : *DocumentationFormat) {
+            if (fromJSON(Format, R.CompletionDocumentationFormat))
+              break;
+          }
+        }
       }
       if (auto *ItemKind = Completion->getObject("completionItemKind")) {
         if (auto *ValueSet = ItemKind->get("valueSet")) {
@@ -334,11 +340,8 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) {
     if (auto *Hover = TextDocument->getObject("hover")) {
       if (auto *ContentFormat = Hover->getArray("contentFormat")) {
         for (const auto &Format : *ContentFormat) {
-          MarkupKind K = MarkupKind::PlainText;
-          if (fromJSON(Format, K)) {
-            R.HoverContentFormat = K;
+          if (fromJSON(Format, R.HoverContentFormat))
             break;
-          }
         }
       }
     }
@@ -891,7 +894,7 @@ llvm::json::Value toJSON(const CompletionItem &CI) {
     Result["kind"] = static_cast<int>(CI.kind);
   if (!CI.detail.empty())
     Result["detail"] = CI.detail;
-  if (!CI.documentation.empty())
+  if (CI.documentation)
     Result["documentation"] = CI.documentation;
   if (!CI.sortText.empty())
     Result["sortText"] = CI.sortText;

clang-tools-extra/clangd/Protocol.h

@@ -430,6 +430,10 @@ struct ClientCapabilities {
   /// textDocument.completion.completionItemKind.valueSet
   llvm::Optional<CompletionItemKindBitset> CompletionItemKinds;
 
+  /// The documentation format that should be used for textDocument/completion.
+  /// textDocument.completion.completionItem.documentationFormat
+  MarkupKind CompletionDocumentationFormat = MarkupKind::PlainText;
+
   /// Client supports CodeAction return value for textDocument/codeAction.
   /// textDocument.codeAction.codeActionLiteralSupport.
   bool CodeActionStructure = false;
@@ -1105,7 +1109,7 @@ struct CompletionItem {
   std::string detail;
 
   /// A human-readable string that represents a doc-comment.
-  std::string documentation;
+  llvm::Optional<MarkupContent> documentation;
 
   /// A string that should be used when comparing this item with other items.
   /// When `falsy` the label is used.