🔧 Minor improvement to directive parsing code (#741)

Author: chrisjsewell

myst_parser/mdit_to_docutils/base.py

@@ -732,26 +732,27 @@ def render_code_block(self, token: SyntaxTreeNode) -> None:
         self.current_node.append(node)
 
     def render_fence(self, token: SyntaxTreeNode) -> None:
-        text = token.content
-        # Ensure that we'll have an empty string if info exists but is only spaces
-        info = token.info.strip() if token.info else token.info
-        language = info.split()[0] if info else ""
+        """Render a fenced code block."""
+        # split the info into possible ```name arguments
+        parts = (token.info.strip() if token.info else "").split(maxsplit=1)
+        name = parts[0] if parts else ""
+        arguments = parts[1] if len(parts) > 1 else ""
 
         if (not self.md_config.commonmark_only) and (not self.md_config.gfm_only):
-            if language == "{eval-rst}":
+            if name == "{eval-rst}":
                 return self.render_restructuredtext(token)
-            if language.startswith("{") and language.endswith("}"):
-                return self.render_directive(token)
+            if name.startswith("{") and name.endswith("}"):
+                return self.render_directive(token, name[1:-1], arguments)
 
-        if not language and self.sphinx_env is not None:
+        if not name and self.sphinx_env is not None:
             # use the current highlight setting, via the ``highlight`` directive,
             # or ``highlight_language`` configuration.
-            language = self.sphinx_env.temp_data.get(
+            name = self.sphinx_env.temp_data.get(
                 "highlight_language", self.sphinx_env.config.highlight_language
             )
 
         lineno_start = 1
-        number_lines = language in self.md_config.number_code_blocks
+        number_lines = name in self.md_config.number_code_blocks
         emphasize_lines = (
             str(token.attrs.get("emphasize-lines"))
             if "emphasize-lines" in token.attrs
@@ -763,8 +764,8 @@ def render_fence(self, token: SyntaxTreeNode) -> None:
                 number_lines = True
 
         node = self.create_highlighted_code_block(
-            text,
-            language,
+            token.content,
+            name,
             number_lines=number_lines,
             lineno_start=lineno_start,
             source=self.document["source"],
@@ -1525,10 +1526,11 @@ def render_myst_role(self, token: SyntaxTreeNode) -> None:
         self.current_node += _nodes + messages2
 
     def render_colon_fence(self, token: SyntaxTreeNode) -> None:
-        """Render a code fence with ``:`` colon delimiters."""
-
-        info = token.info.strip() if token.info else token.info
-        name = info.split()[0] if info else ""
+        """Render a div block, with ``:`` colon delimiters."""
+        # split the info into possible :::name arguments
+        parts = (token.info.strip() if token.info else "").split(maxsplit=1)
+        name = parts[0] if parts else ""
+        arguments = parts[1] if len(parts) > 1 else ""
 
         if name.startswith("{") and name.endswith("}"):
             if token.content.startswith(":::"):
@@ -1538,7 +1540,7 @@ def render_colon_fence(self, token: SyntaxTreeNode) -> None:
                 linear_token = token.token.copy()
                 linear_token.content = "\n" + linear_token.content
                 token.token = linear_token
-            return self.render_directive(token)
+            return self.render_directive(token, name[1:-1], arguments)
 
         container = nodes.container(is_div=True)
         self.add_line_and_source_path(container, token)
@@ -1661,18 +1663,26 @@ def render_restructuredtext(self, token: SyntaxTreeNode) -> None:
                 self.document.note_explicit_target(node, node)
         self.current_node.extend(newdoc.children)
 
-    def render_directive(self, token: SyntaxTreeNode) -> None:
-        """Render special fenced code blocks as directives."""
-        first_line = token.info.split(maxsplit=1)
-        name = first_line[0][1:-1]
-        arguments = "" if len(first_line) == 1 else first_line[1]
-        content = token.content
+    def render_directive(
+        self, token: SyntaxTreeNode, name: str, arguments: str
+    ) -> None:
+        """Render special fenced code blocks as directives.
+
+        :param token: the token to render
+        :param name: the name of the directive
+        :param arguments: The remaining text on the same line as the directive name.
+        """
         position = token_line(token)
-        nodes_list = self.run_directive(name, arguments, content, position)
+        nodes_list = self.run_directive(name, arguments, token.content, position)
         self.current_node += nodes_list
 
     def run_directive(
-        self, name: str, first_line: str, content: str, position: int
+        self,
+        name: str,
+        first_line: str,
+        content: str,
+        position: int,
+        additional_options: dict[str, str] | None = None,
     ) -> list[nodes.Element]:
         """Run a directive and return the generated nodes.
 
@@ -1681,6 +1691,8 @@ def run_directive(
             May be an argument or body text, dependent on the directive
         :param content: All text after the first line. Can include options.
         :param position: The line number of the first line
+        :param additional_options: Additional options to add to the directive,
+            above those parsed from the content.
 
         """
         self.document.current_line = position
@@ -1706,7 +1718,12 @@ def run_directive(
             directive_class.option_spec["heading-offset"] = directives.nonnegative_int
 
         try:
-            parsed = parse_directive_text(directive_class, first_line, content)
+            parsed = parse_directive_text(
+                directive_class,
+                first_line,
+                content,
+                additional_options=additional_options,
+            )
         except MarkupError as error:
             error = self.reporter.error(
                 f"Directive '{name}': {error}",

myst_parser/parsers/directives.py

@@ -65,21 +65,28 @@ def parse_directive_text(
     directive_class: type[Directive],
     first_line: str,
     content: str,
+    *,
     validate_options: bool = True,
+    additional_options: dict[str, str] | None = None,
 ) -> DirectiveParsingResult:
     """Parse (and validate) the full directive text.
 
     :param first_line: The text on the same line as the directive name.
         May be an argument or body text, dependent on the directive
     :param content: All text after the first line. Can include options.
     :param validate_options: Whether to validate the values of options
+    :param additional_options: Additional options to add to the directive,
+        above those parsed from the content (content options take priority).
 
     :raises MarkupError: if there is a fatal parsing/validation error
     """
     parse_errors: list[str] = []
     if directive_class.option_spec:
         body, options, option_errors = parse_directive_options(
-            content, directive_class, validate=validate_options
+            content,
+            directive_class,
+            validate=validate_options,
+            additional_options=additional_options,
         )
         parse_errors.extend(option_errors)
         body_lines = body.splitlines()
@@ -114,7 +121,10 @@ def parse_directive_text(
 
 
 def parse_directive_options(
-    content: str, directive_class: type[Directive], validate: bool = True
+    content: str,
+    directive_class: type[Directive],
+    validate: bool = True,
+    additional_options: dict[str, str] | None = None,
 ) -> tuple[str, dict, list[str]]:
     """Parse (and validate) the directive option section.
 
@@ -162,6 +172,10 @@ def parse_directive_options(
         # but since its for testing only we accept all options
         return content, options, validation_errors
 
+    if additional_options:
+        # The YAML block takes priority over additional options
+        options = {**additional_options, **options}
+
     # check options against spec
     options_spec: dict[str, Callable] = directive_class.option_spec
     unknown_options: list[str] = []

tests/test_renderers/test_parse_directives.py

@@ -49,3 +49,34 @@ def test_parsing(file_params):
 def test_parsing_errors(descript, klass, arguments, content):
     with pytest.raises(MarkupError):
         parse_directive_text(klass, arguments, content)
+
+
+def test_additional_options():
+    """Allow additional options to be passed to a directive."""
+    # this should be fine
+    result = parse_directive_text(
+        Note, "", "content", additional_options={"class": "bar"}
+    )
+    assert not result.warnings
+    assert result.options == {"class": ["bar"]}
+    assert result.body == ["content"]
+    # body on first line should also be fine
+    result = parse_directive_text(
+        Note, "content", "other", additional_options={"class": "bar"}
+    )
+    assert not result.warnings
+    assert result.options == {"class": ["bar"]}
+    assert result.body == ["content", "other"]
+    # additional option should not take precedence
+    result = parse_directive_text(
+        Note, "content", ":class: foo", additional_options={"class": "bar"}
+    )
+    assert not result.warnings
+    assert result.options == {"class": ["foo"]}
+    assert result.body == ["content"]
+    # this should warn about the unknown option
+    result = parse_directive_text(
+        Note, "", "content", additional_options={"foo": "bar"}
+    )
+    assert len(result.warnings) == 1
+    assert "Unknown option" in result.warnings[0]