Revert "Revert "Enable cmark smart punctuation in doc comments""

This reverts commit f723b86 and
updates the IDE tests that incidentally included some punctuation.
No new tests are necessary - the character level tests are exercised
in cmark itself.

Author: bitjammer

lib/Markup/Markup.cpp

@@ -320,7 +320,7 @@ parseElement(MarkupContext &MC, LineList &LL, ParseState State) {
 Document *llvm::markup::parseDocument(MarkupContext &MC, LineList &LL) {
   auto Comment = LL.str();
   auto CMarkDoc = cmark_parse_document(Comment.c_str(), Comment.size(),
-                                       CMARK_OPT_DEFAULT);
+                                       CMARK_OPT_SMART);
 
   if (CMarkDoc == nullptr)
     return nullptr;

test/IDE/comment_extensions.swift

@@ -3,7 +3,7 @@
 // Content is in separate file in ./Inputs due to the "requires" keyword getting
 // recognized by lit.
 
-// CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>attention()</Name><USR>s:F14swift_ide_test9attentionFT_T_</USR><Declaration>func attention()</Declaration><Discussion><Attention><Para>This function is so hip and exciting, it can&apos;t be trusted.</Para></Attention></Discussion></Function>]
+// CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}} line="{{.*}}" column="{{.*}}"><Name>attention()</Name><USR>s:F14swift_ide_test9attentionFT_T_</USR><Declaration>func attention()</Declaration><Discussion><Attention><Para>This function is so hip and exciting, it can’t be trusted.</Para></Attention></Discussion></Function>]
 
 // CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>author()</Name><USR>s:F14swift_ide_test6authorFT_T_</USR><Declaration>func author()</Declaration><Discussion><Author><Para>Stephen</Para></Author></Discussion></Function>]
 
@@ -38,7 +38,7 @@
 
 // CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>requires()</Name><USR>s:F14swift_ide_test8requiresFT_T_</USR><Declaration>func requires()</Declaration><Discussion><Requires><Para></Para><List-Bullet><Item><Para>explicit package name. Just kidding!</Para></Item></List-Bullet></Requires></Discussion></Function>]
 
-// CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>see()</Name><USR>s:F14swift_ide_test3seeFT_T_</USR><Declaration>func see()</Declaration><Discussion><See><Para>the pie (it&apos;s very good).</Para></See></Discussion></Function>]
+// CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>see()</Name><USR>s:F14swift_ide_test3seeFT_T_</USR><Declaration>func see()</Declaration><Discussion><See><Para>the pie (it’s very good).</Para></See></Discussion></Function>] CommentXMLValid
 
 // CHECK: {{.*}}DocCommentAsXML=[<Function file="{{.*}}" line="{{.*}}" column="{{.*}}"><Name>since()</Name><USR>s:F14swift_ide_test5sinceFT_T_</USR><Declaration>func since()</Declaration><Discussion><Since><Para>1809</Para></Since></Discussion></Function>]
 

test/SourceKit/CursorInfo/cursor_info.swift

@@ -257,7 +257,7 @@ typealias MyVoid = ()
 // CHECK6-NEXT: FooSwiftModule
 // CHECK6-NEXT: <Declaration>func fooSwiftFunc() -&gt; <Type usr="s:Si">Int</Type></Declaration>
 // CHECK6-NEXT: <decl.function.free><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>fooSwiftFunc</decl.name>() -&gt; <decl.function.returntype><ref.struct usr="s:Si">Int</ref.struct></decl.function.returntype></decl.function.free>
-// CHECK6-NEXT: {{^}}<Function><Name>fooSwiftFunc()</Name><USR>s:F14FooSwiftModule12fooSwiftFuncFT_Si</USR><Declaration>func fooSwiftFunc() -&gt; Int</Declaration><Abstract><Para>This is &apos;fooSwiftFunc&apos; from &apos;FooSwiftModule&apos;.</Para></Abstract></Function>{{$}}
+// CHECK6-NEXT: {{^}}<Function><Name>fooSwiftFunc()</Name><USR>s:F14FooSwiftModule12fooSwiftFuncFT_Si</USR><Declaration>func fooSwiftFunc() -&gt; Int</Declaration><Abstract><Para>This is ‘fooSwiftFunc’ from ‘FooSwiftModule’.</Para></Abstract></Function>{{$}}
 
 // RUN: %sourcekitd-test -req=cursor -pos=14:10 %s -- -F %S/../Inputs/libIDE-mock-sdk -I %t.tmp %mcp_opt %s | FileCheck -check-prefix=CHECK7 %s
 // CHECK7:      source.lang.swift.ref.struct (13:8-13:10)

test/SourceKit/DocSupport/doc_clang_module.swift.response

@@ -6224,7 +6224,7 @@ var FooSubUnnamedEnumeratorA1: Int { get }
         key.name: "toIntMax()",
         key.usr: "s:FSi8toIntMaxFT_Vs5Int64::SYNTHESIZED::c:@E@FooComparisonResult",
         key.original_usr: "s:FSi8toIntMaxFT_Vs5Int64",
-        key.doc.full_as_xml: "<Function><Name>toIntMax()</Name><USR>s:FSi8toIntMaxFT_Vs5Int64</USR><Declaration>func toIntMax() -&gt; IntMax</Declaration><Abstract><Para>Represent this number using Swift&apos;s widest native signed integer type.</Para></Abstract></Function>",
+        key.doc.full_as_xml: "<Function><Name>toIntMax()</Name><USR>s:FSi8toIntMaxFT_Vs5Int64</USR><Declaration>func toIntMax() -&gt; IntMax</Declaration><Abstract><Para>Represent this number using Swift’s widest native signed integer type.</Para></Abstract></Function>",
         key.offset: 1169,
         key.length: 25,
         key.fully_annotated_decl: "<decl.function.method.instance><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>toIntMax</decl.name>() -&gt; <decl.function.returntype><ref.typealias usr=\"s:s6IntMax\">IntMax</ref.typealias></decl.function.returntype></decl.function.method.instance>",
@@ -6463,7 +6463,7 @@ var FooSubUnnamedEnumeratorA1: Int { get }
         key.name: "init(bitPattern:)",
         key.usr: "s:FSicFT10bitPatternSu_Si::SYNTHESIZED::c:@E@FooComparisonResult",
         key.original_usr: "s:FSicFT10bitPatternSu_Si",
-        key.doc.full_as_xml: "<Function><Name>init(bitPattern:)</Name><USR>s:FSicFT10bitPatternSu_Si</USR><Declaration>init(bitPattern: UInt)</Declaration><Abstract><Para>Construct a <codeVoice>Int</codeVoice> having the same memory representation as the <codeVoice>UInt</codeVoice> <codeVoice>bitPattern</codeVoice>.  No range or overflow checking occurs, and the resulting <codeVoice>Int</codeVoice> may not have the same numeric value as <codeVoice>bitPattern</codeVoice>--it is only guaranteed to use the same pattern of bits.</Para></Abstract></Function>",
+        key.doc.full_as_xml: "<Function><Name>init(bitPattern:)</Name><USR>s:FSicFT10bitPatternSu_Si</USR><Declaration>init(bitPattern: UInt)</Declaration><Abstract><Para>Construct a <codeVoice>Int</codeVoice> having the same memory representation as the <codeVoice>UInt</codeVoice> <codeVoice>bitPattern</codeVoice>.  No range or overflow checking occurs, and the resulting <codeVoice>Int</codeVoice> may not have the same numeric value as <codeVoice>bitPattern</codeVoice>–it is only guaranteed to use the same pattern of bits.</Para></Abstract></Function>",
         key.offset: 1520,
         key.length: 33,
         key.fully_annotated_decl: "<decl.function.constructor><syntaxtype.keyword>init</syntaxtype.keyword>(<decl.var.parameter><decl.var.parameter.argument_label>bitPattern</decl.var.parameter.argument_label>: <decl.var.parameter.type><ref.struct usr=\"s:Su\">UInt</ref.struct></decl.var.parameter.type></decl.var.parameter>)</decl.function.constructor>",
@@ -6539,7 +6539,7 @@ var FooSubUnnamedEnumeratorA1: Int { get }
         key.name: "init(_:radix:)",
         key.usr: "s:FSicFTSS5radixSi_GSqSi_::SYNTHESIZED::c:@E@FooComparisonResult",
         key.original_usr: "s:FSicFTSS5radixSi_GSqSi_",
-        key.doc.full_as_xml: "<Function><Name>init(_:radix:)</Name><USR>s:FSicFTSS5radixSi_GSqSi_</USR><Declaration>init?(_ text: String, radix: Int = default)</Declaration><Abstract><Para>Construct from an ASCII representation in the given <codeVoice>radix</codeVoice>.</Para></Abstract><Discussion><Para>If <codeVoice>text</codeVoice> does not match the regular expression &quot;[+-]?[0-9a-zA-Z]+&quot;, or the value it denotes in the given <codeVoice>radix</codeVoice> is not representable, the result is <codeVoice>nil</codeVoice>.</Para></Discussion></Function>",
+        key.doc.full_as_xml: "<Function><Name>init(_:radix:)</Name><USR>s:FSicFTSS5radixSi_GSqSi_</USR><Declaration>init?(_ text: String, radix: Int = default)</Declaration><Abstract><Para>Construct from an ASCII representation in the given <codeVoice>radix</codeVoice>.</Para></Abstract><Discussion><Para>If <codeVoice>text</codeVoice> does not match the regular expression “[+-]?[0-9a-zA-Z]+”, or the value it denotes in the given <codeVoice>radix</codeVoice> is not representable, the result is <codeVoice>nil</codeVoice>.</Para></Discussion></Function>",
         key.offset: 1640,
         key.length: 49,
         key.fully_annotated_decl: "<decl.function.constructor><syntaxtype.keyword>init</syntaxtype.keyword>?(<decl.var.parameter><decl.var.parameter.argument_label>_</decl.var.parameter.argument_label> <decl.var.parameter.name>text</decl.var.parameter.name>: <decl.var.parameter.type><ref.struct usr=\"s:SS\">String</ref.struct></decl.var.parameter.type></decl.var.parameter>, <decl.var.parameter><decl.var.parameter.argument_label>radix</decl.var.parameter.argument_label>: <decl.var.parameter.type><ref.struct usr=\"s:Si\">Int</ref.struct></decl.var.parameter.type> = <syntaxtype.keyword>default</syntaxtype.keyword></decl.var.parameter>)</decl.function.constructor>",
@@ -7132,7 +7132,7 @@ var FooSubUnnamedEnumeratorA1: Int { get }
         key.description: "RawValue : BitwiseOperations"
       }
     ],
-    key.doc.full_as_xml: "<Other><Name></Name><Declaration>extension FooRuncingOptions where RawValue : BitwiseOperations</Declaration><Abstract><Para><codeVoice>OptionSet</codeVoice> requirements for which default implementations are supplied when <codeVoice>RawValue</codeVoice> conforms to <codeVoice>BitwiseOperations</codeVoice>, which is the usual case.  Each distinct bit of an option set&apos;s <codeVoice>.rawValue</codeVoice> corresponds to a disjoint element of the option set.</Para></Abstract><Discussion><Note><Para>A type conforming to <codeVoice>OptionSet</codeVoice> can implement any of these initializers or methods, and those implementations will be used in lieu of these defaults.</Para></Note><List-Bullet><Item><Para><codeVoice>union</codeVoice> is implemented as a bitwise &quot;or&quot; (<codeVoice>|</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item><Item><Para><codeVoice>intersection</codeVoice> is implemented as a bitwise &quot;and&quot; (<codeVoice>|</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item><Item><Para><codeVoice>exclusiveOr</codeVoice> is implemented as a bitwise &quot;exclusive or&quot; (<codeVoice>^</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item></List-Bullet></Discussion></Other>",
+    key.doc.full_as_xml: "<Other><Name></Name><Declaration>extension FooRuncingOptions where RawValue : BitwiseOperations</Declaration><Abstract><Para><codeVoice>OptionSet</codeVoice> requirements for which default implementations are supplied when <codeVoice>RawValue</codeVoice> conforms to <codeVoice>BitwiseOperations</codeVoice>, which is the usual case.  Each distinct bit of an option set’s <codeVoice>.rawValue</codeVoice> corresponds to a disjoint element of the option set.</Para></Abstract><Discussion><Note><Para>A type conforming to <codeVoice>OptionSet</codeVoice> can implement any of these initializers or methods, and those implementations will be used in lieu of these defaults.</Para></Note><List-Bullet><Item><Para><codeVoice>union</codeVoice> is implemented as a bitwise “or” (<codeVoice>|</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item><Item><Para><codeVoice>intersection</codeVoice> is implemented as a bitwise “and” (<codeVoice>|</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item><Item><Para><codeVoice>exclusiveOr</codeVoice> is implemented as a bitwise “exclusive or” (<codeVoice>^</codeVoice>) of <codeVoice>rawValue</codeVoice>s</Para></Item></List-Bullet></Discussion></Other>",
     key.offset: 3039,
     key.length: 206,
     key.extends: {

test/SourceKit/DocSupport/doc_swift_module.swift.response

@@ -2347,7 +2347,7 @@ func genfoo<T1 : Prot, T2 : cake.C1 where T1.Element == Int, T2.Element == T1.El
         key.name: "toIntMax()",
         key.usr: "s:FSi8toIntMaxFT_Vs5Int64::SYNTHESIZED::s:O4cake6MyEnum",
         key.original_usr: "s:FSi8toIntMaxFT_Vs5Int64",
-        key.doc.full_as_xml: "<Function><Name>toIntMax()</Name><USR>s:FSi8toIntMaxFT_Vs5Int64</USR><Declaration>func toIntMax() -&gt; IntMax</Declaration><Abstract><Para>Represent this number using Swift&apos;s widest native signed integer type.</Para></Abstract></Function>",
+        key.doc.full_as_xml: "<Function><Name>toIntMax()</Name><USR>s:FSi8toIntMaxFT_Vs5Int64</USR><Declaration>func toIntMax() -&gt; IntMax</Declaration><Abstract><Para>Represent this number using Swift’s widest native signed integer type.</Para></Abstract></Function>",
         key.offset: 678,
         key.length: 25,
         key.fully_annotated_decl: "<decl.function.method.instance><syntaxtype.keyword>func</syntaxtype.keyword> <decl.name>toIntMax</decl.name>() -&gt; <decl.function.returntype><ref.typealias usr=\"s:s6IntMax\">IntMax</ref.typealias></decl.function.returntype></decl.function.method.instance>",
@@ -2586,7 +2586,7 @@ func genfoo<T1 : Prot, T2 : cake.C1 where T1.Element == Int, T2.Element == T1.El
         key.name: "init(bitPattern:)",
         key.usr: "s:FSicFT10bitPatternSu_Si::SYNTHESIZED::s:O4cake6MyEnum",
         key.original_usr: "s:FSicFT10bitPatternSu_Si",
-        key.doc.full_as_xml: "<Function><Name>init(bitPattern:)</Name><USR>s:FSicFT10bitPatternSu_Si</USR><Declaration>init(bitPattern: UInt)</Declaration><Abstract><Para>Construct a <codeVoice>Int</codeVoice> having the same memory representation as the <codeVoice>UInt</codeVoice> <codeVoice>bitPattern</codeVoice>.  No range or overflow checking occurs, and the resulting <codeVoice>Int</codeVoice> may not have the same numeric value as <codeVoice>bitPattern</codeVoice>--it is only guaranteed to use the same pattern of bits.</Para></Abstract></Function>",
+        key.doc.full_as_xml: "<Function><Name>init(bitPattern:)</Name><USR>s:FSicFT10bitPatternSu_Si</USR><Declaration>init(bitPattern: UInt)</Declaration><Abstract><Para>Construct a <codeVoice>Int</codeVoice> having the same memory representation as the <codeVoice>UInt</codeVoice> <codeVoice>bitPattern</codeVoice>.  No range or overflow checking occurs, and the resulting <codeVoice>Int</codeVoice> may not have the same numeric value as <codeVoice>bitPattern</codeVoice>–it is only guaranteed to use the same pattern of bits.</Para></Abstract></Function>",
         key.offset: 1029,
         key.length: 33,
         key.fully_annotated_decl: "<decl.function.constructor><syntaxtype.keyword>init</syntaxtype.keyword>(<decl.var.parameter><decl.var.parameter.argument_label>bitPattern</decl.var.parameter.argument_label>: <decl.var.parameter.type><ref.struct usr=\"s:Su\">UInt</ref.struct></decl.var.parameter.type></decl.var.parameter>)</decl.function.constructor>",
@@ -2662,7 +2662,7 @@ func genfoo<T1 : Prot, T2 : cake.C1 where T1.Element == Int, T2.Element == T1.El
         key.name: "init(_:radix:)",
         key.usr: "s:FSicFTSS5radixSi_GSqSi_::SYNTHESIZED::s:O4cake6MyEnum",
         key.original_usr: "s:FSicFTSS5radixSi_GSqSi_",
-        key.doc.full_as_xml: "<Function><Name>init(_:radix:)</Name><USR>s:FSicFTSS5radixSi_GSqSi_</USR><Declaration>init?(_ text: String, radix: Int = default)</Declaration><Abstract><Para>Construct from an ASCII representation in the given <codeVoice>radix</codeVoice>.</Para></Abstract><Discussion><Para>If <codeVoice>text</codeVoice> does not match the regular expression &quot;[+-]?[0-9a-zA-Z]+&quot;, or the value it denotes in the given <codeVoice>radix</codeVoice> is not representable, the result is <codeVoice>nil</codeVoice>.</Para></Discussion></Function>",
+        key.doc.full_as_xml: "<Function><Name>init(_:radix:)</Name><USR>s:FSicFTSS5radixSi_GSqSi_</USR><Declaration>init?(_ text: String, radix: Int = default)</Declaration><Abstract><Para>Construct from an ASCII representation in the given <codeVoice>radix</codeVoice>.</Para></Abstract><Discussion><Para>If <codeVoice>text</codeVoice> does not match the regular expression “[+-]?[0-9a-zA-Z]+”, or the value it denotes in the given <codeVoice>radix</codeVoice> is not representable, the result is <codeVoice>nil</codeVoice>.</Para></Discussion></Function>",
         key.offset: 1149,
         key.length: 49,
         key.fully_annotated_decl: "<decl.function.constructor><syntaxtype.keyword>init</syntaxtype.keyword>?(<decl.var.parameter><decl.var.parameter.argument_label>_</decl.var.parameter.argument_label> <decl.var.parameter.name>text</decl.var.parameter.name>: <decl.var.parameter.type><ref.struct usr=\"s:SS\">String</ref.struct></decl.var.parameter.type></decl.var.parameter>, <decl.var.parameter><decl.var.parameter.argument_label>radix</decl.var.parameter.argument_label>: <decl.var.parameter.type><ref.struct usr=\"s:Si\">Int</ref.struct></decl.var.parameter.type> = <syntaxtype.keyword>default</syntaxtype.keyword></decl.var.parameter>)</decl.function.constructor>",