Fix and add test for myst block token

Author: chrisjsewell

markdown_it/__init__.py

@@ -1,4 +1,4 @@
 from .main import MarkdownIt  # noqa: F401
 
 
-__version__ = "0.4.2"
+__version__ = "0.4.3"

markdown_it/extensions/front_matter/index.py

@@ -120,7 +120,7 @@ def frontMatter(state: StateBlock, startLine: int, endLine: int, silent: bool):
         state.parentType = old_parent
         state.lineMax = old_line_max
         state.line = nextLine + (1 if auto_closed else 0)
-        token.map = [startLine, state.line - 1]
+        token.map = [startLine, state.line]
 
         return True
 

markdown_it/extensions/myst_blocks/index.py

@@ -82,11 +82,11 @@ def block_break(state: StateBlock, startLine: int, endLine: int, silent: bool):
     cnt = 1
     while pos < maximum:
         ch = charCodeAt(state.src, pos)
-        pos += 1
         if ch != marker and not isSpace(ch):
             break
         if ch == marker:
             cnt += 1
+        pos += 1
 
     if cnt < 3:
         return False
@@ -98,9 +98,9 @@ def block_break(state: StateBlock, startLine: int, endLine: int, silent: bool):
 
     token = state.push("myst_block_break", "hr", 0)
     token.attrSet("class", "myst-block")
-    token.content = state.src[pos - 1 : maximum].strip()
+    token.content = state.src[pos:maximum].strip()
     token.map = [startLine, state.line]
-    token.markup = chr(marker) * (cnt + 1)
+    token.markup = chr(marker) * cnt
 
     return True
 

tests/test_front_matter/test_fixtures.py

@@ -29,7 +29,7 @@ def test_token():
             tag="",
             nesting=0,
             attrs=None,
-            map=[0, 2],
+            map=[0, 3],
             level=0,
             children=None,
             content="a: 1",

tests/test_myst_block/test_fixtures.py

@@ -3,6 +3,7 @@
 import pytest
 
 from markdown_it import MarkdownIt
+from markdown_it.token import Token
 from markdown_it.utils import read_fixture_file
 from markdown_it.extensions.myst_blocks import myst_block_plugin
 
@@ -16,3 +17,66 @@ def test_all(line, title, input, expected):
     text = md.render(input)
     print(text)
     assert text.rstrip() == expected.rstrip()
+
+
+def test_block_token():
+    md = MarkdownIt("commonmark").use(myst_block_plugin)
+    tokens = md.parse("+++")
+    assert tokens == [
+        Token(
+            type="myst_block_break",
+            tag="hr",
+            nesting=0,
+            attrs=[["class", "myst-block"]],
+            map=[0, 1],
+            level=0,
+            children=None,
+            content="",
+            markup="+++",
+            info="",
+            meta={},
+            block=True,
+            hidden=False,
+        )
+    ]
+
+    tokens = md.parse("\n+ + + abc")
+    assert tokens == [
+        Token(
+            type="myst_block_break",
+            tag="hr",
+            nesting=0,
+            attrs=[["class", "myst-block"]],
+            map=[1, 2],
+            level=0,
+            children=None,
+            content="abc",
+            markup="+++",
+            info="",
+            meta={},
+            block=True,
+            hidden=False,
+        )
+    ]
+
+
+def test_comment_token():
+    md = MarkdownIt("commonmark").use(myst_block_plugin)
+    tokens = md.parse("\n\n% abc")
+    assert tokens == [
+        Token(
+            type="myst_line_comment",
+            tag="",
+            nesting=0,
+            attrs=[["class", "myst-line-comment"]],
+            map=[2, 3],
+            level=0,
+            children=None,
+            content="abc",
+            markup="%",
+            info="",
+            meta={},
+            block=True,
+            hidden=False,
+        )
+    ]