Docstrings highlighting with pygments (#5462)

* add 'docstrings-with-pygments' feature without packages checks and tests

* move syntax_highlight doc filter in compatibility module and define it conditionally

* typo fixed

* add test for optional code highlight ('pygments' and 'markdown' packages must be installed)

Author: nastasi-oq

rest_framework/compat.py

@@ -244,6 +244,7 @@ def apply_markdown(text):
         md = markdown.Markdown(
             extensions=extensions, extension_configs=extension_configs
         )
+        md_filter_add_syntax_highlight(md)
         return md.convert(text)
 except ImportError:
     apply_markdown = None
@@ -273,6 +274,38 @@ def pygments_highlight(text, lang, style):
     def pygments_css(style):
         return None
 
+if markdown is not None and pygments is not None:
+    # starting from this blogpost and modified to support current markdown extensions API
+    # https://zerokspot.com/weblog/2008/06/18/syntax-highlighting-in-markdown-with-pygments/
+
+    from markdown.preprocessors import Preprocessor
+    import re
+
+    class CodeBlockPreprocessor(Preprocessor):
+        pattern = re.compile(
+            r'^\s*@@ (.+?) @@\s*(.+?)^\s*@@', re.M|re.S)
+
+        formatter = HtmlFormatter()
+
+        def run(self, lines):
+            def repl(m):
+                try:
+                    lexer = get_lexer_by_name(m.group(1))
+                except (ValueError, NameError):
+                    lexer = TextLexer()
+                code = m.group(2).replace('\t','    ')
+                code = pygments.highlight(code, lexer, self.formatter)
+                code = code.replace('\n\n', '\n&nbsp;\n').replace('\n', '<br />').replace('\\@','@')
+                return '\n\n%s\n\n' % code
+            ret = self.pattern.sub(repl, "\n".join(lines))
+            return ret.split("\n")
+
+    def md_filter_add_syntax_highlight(md):
+        md.preprocessors.add('highlight', CodeBlockPreprocessor(), "_begin")
+        return True
+else:
+    def md_filter_add_syntax_highlight(md):
+        return False
 
 try:
     import pytz

tests/test_description.py

@@ -24,11 +24,36 @@
 
 indented
 
-# hash style header #"""
+# hash style header #
+
+@@ json @@
+[{
+    "alpha": 1,
+    "beta: "this is a string"
+}]
+@@"""
 
 # If markdown is installed we also test it's working
 # (and that our wrapped forces '=' to h2 and '-' to h3)
 
+MARKED_DOWN_HILITE = """
+<div class="highlight"><pre><span></span><span \
+class="p">[{</span><br />    <span class="nt">&quot;alpha&quot;</span><span\
+ class="p">:</span> <span class="mi">1</span><span class="p">,</span><br />\
+    <span class="nt">&quot;beta: &quot;</span><span class="err">this</span>\
+ <span class="err">is</span> <span class="err">a</span> <span class="err">\
+string&quot;</span><br /><span class="p">}]</span><br /></pre></div>
+
+<p><br /></p>"""
+
+MARKED_DOWN_NOT_HILITE = """
+<p>@@ json @@
+[{
+    "alpha": 1,
+    "beta: "this is a string"
+}]
+@@</p>"""
+
 # We support markdown < 2.1 and markdown >= 2.1
 MARKED_DOWN_lt_21 = """<h2>an example docstring</h2>
 <ul>
@@ -39,7 +64,7 @@
 <pre><code>code block
 </code></pre>
 <p>indented</p>
-<h2 id="hash_style_header">hash style header</h2>"""
+<h2 id="hash_style_header">hash style header</h2>%s"""
 
 MARKED_DOWN_gte_21 = """<h2 id="an-example-docstring">an example docstring</h2>
 <ul>
@@ -50,7 +75,7 @@
 <pre><code>code block
 </code></pre>
 <p>indented</p>
-<h2 id="hash-style-header">hash style header</h2>"""
+<h2 id="hash-style-header">hash style header</h2>%s"""
 
 
 class TestViewNamesAndDescriptions(TestCase):
@@ -78,7 +103,14 @@ class MockView(APIView):
 
             indented
 
-            # hash style header #"""
+            # hash style header #
+
+            @@ json @@
+            [{
+                "alpha": 1,
+                "beta: "this is a string"
+            }]
+            @@"""
 
         assert MockView().get_view_description() == DESCRIPTION
 
@@ -118,8 +150,16 @@ def test_markdown(self):
         Ensure markdown to HTML works as expected.
         """
         if apply_markdown:
-            gte_21_match = apply_markdown(DESCRIPTION) == MARKED_DOWN_gte_21
-            lt_21_match = apply_markdown(DESCRIPTION) == MARKED_DOWN_lt_21
+            gte_21_match = (
+                apply_markdown(DESCRIPTION) == (
+                    MARKED_DOWN_gte_21 % MARKED_DOWN_HILITE) or
+                apply_markdown(DESCRIPTION) == (
+                    MARKED_DOWN_gte_21 % MARKED_DOWN_NOT_HILITE))
+            lt_21_match = (
+                apply_markdown(DESCRIPTION) == (
+                    MARKED_DOWN_lt_21 % MARKED_DOWN_HILITE) or
+                apply_markdown(DESCRIPTION) == (
+                    MARKED_DOWN_lt_21 % MARKED_DOWN_NOT_HILITE))
             assert gte_21_match or lt_21_match