✨ NEW: Add Docutils MyST config and CLI

myst_parser/docutils_.py

@@ -1,36 +1,157 @@
 """A module for compatibility with the docutils>=0.17 `include` directive, in RST documents::
 
-   .. include::
+   .. include:: path/to/file.md
       :parser: myst_parser.docutils_
 """
-from typing import Tuple
+from contextlib import suppress
+from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
 
-from docutils import nodes
+from attr import Attribute
+from docutils import frontend, nodes
+from docutils.core import default_description, publish_cmdline
 from docutils.parsers.rst import Parser as RstParser
 from markdown_it.token import Token
 
 from myst_parser.main import MdParserConfig, default_parser
 
 
-class Parser(RstParser):
-    """Docutils parser for Markedly Structured Text (MyST)."""
+def _validate_int(
+    setting, value, option_parser, config_parser=None, config_section=None
+) -> int:
+    """Validate an integer setting."""
+    return int(value)
 
-    supported: Tuple[str, ...] = ("md", "markdown", "myst")
-    """Aliases this parser supports."""
 
-    settings_spec = RstParser.settings_spec
-    """Runtime settings specification.
+def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]:
+    """Create a validator for a tuple of length `length`."""
+
+    def _validate(
+        setting, value, option_parser, config_parser=None, config_section=None
+    ):
+        string_list = frontend.validate_comma_separated_list(
+            setting, value, option_parser, config_parser, config_section
+        )
+        if len(string_list) != length:
+            raise ValueError(
+                f"Expecting {length} items in {setting}, got {len(string_list)}."
+            )
+        return tuple(string_list)
+
+    return _validate
+
+
+DOCUTILS_UNSET = object()
+"""Sentinel for arguments not set through docutils.conf."""
+
 
-    Defines runtime settings and associated command-line options, as used by
-    `docutils.frontend.OptionParser`.  This is a concatenation of tuples of:
+DOCUTILS_EXCLUDED_ARGS = (
+    # docutils.conf can't represent callables
+    "heading_slug_func",
+    # docutils.conf can't represent dicts
+    "html_meta",
+    "substitutions",
+    # we can't add substitutions so not needed
+    "sub_delimiters",
+    # heading anchors are currently sphinx only
+    "heading_anchors",
+    # sphinx.ext.mathjax only options
+    "update_mathjax",
+    "mathjax_classes",
+    # We don't want to set the renderer from docutils.conf
+    "renderer",
+)
+"""Names of settings that cannot be set in docutils.conf."""
 
-    - Option group title (string or `None` which implies no group, just a list
-      of single options).
 
-    - Description (string or `None`).
+def _docutils_optparse_options_of_attribute(
+    at: Attribute, default: Any
+) -> Tuple[dict, str]:
+    """Convert an ``MdParserConfig`` attribute into a Docutils optparse options dict."""
+    if at.type is int:
+        return {"validator": _validate_int}, f"(type: int, default: {default})"
+    if at.type is bool:
+        return {
+            "validator": frontend.validate_boolean
+        }, f"(type: bool, default: {default})"
+    if at.type is str:
+        return {}, f"(type: str, default: '{default}')"
+    if at.type == Iterable[str] or at.name == "url_schemes":
+        return {
+            "validator": frontend.validate_comma_separated_list
+        }, f"(type: comma-delimited, default: '{','.join(default)}')"
+    if at.type == Tuple[str, str]:
+        return {
+            "validator": _create_validate_tuple(2)
+        }, f"(type: str,str, default: '{','.join(default)}')"
+    if at.type == Union[int, type(None)] and at.default is None:
+        return {
+            "validator": _validate_int,
+            "default": None,
+        }, f"(type: null|int, default: {default})"
+    if at.type == Union[Iterable[str], type(None)] and at.default is None:
+        return {
+            "validator": frontend.validate_comma_separated_list,
+            "default": None,
+        }, f"(type: comma-delimited, default: '{default or ','.join(default)}')"
+    raise AssertionError(
+        f"Configuration option {at.name} not set up for use in docutils.conf."
+        f"Either add {at.name} to docutils_.DOCUTILS_EXCLUDED_ARGS,"
+        "or add a new entry in _docutils_optparse_of_attribute."
+    )
 
-    - A sequence of option tuples
-    """
+
+def _docutils_setting_tuple_of_attribute(
+    attribute: Attribute, default: Any
+) -> Tuple[str, Any, Any]:
+    """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple."""
+    name = f"myst_{attribute.name}"
+    flag = "--" + name.replace("_", "-")
+    options = {"dest": name, "default": DOCUTILS_UNSET}
+    at_options, type_str = _docutils_optparse_options_of_attribute(attribute, default)
+    options.update(at_options)
+    help_str = attribute.metadata.get("help", "") if attribute.metadata else ""
+    return (f"{help_str} {type_str}", [flag], options)
+
+
+def _myst_docutils_setting_tuples():
+    """Return a list of Docutils setting for the MyST section."""
+    defaults = MdParserConfig()
+    return tuple(
+        _docutils_setting_tuple_of_attribute(at, getattr(defaults, at.name))
+        for at in MdParserConfig.get_fields()
+        if at.name not in DOCUTILS_EXCLUDED_ARGS
+    )
+
+
+def create_myst_config(settings: frontend.Values):
+    """Create a ``MdParserConfig`` from the given settings."""
+    values = {}
+    for attribute in MdParserConfig.get_fields():
+        if attribute.name in DOCUTILS_EXCLUDED_ARGS:
+            continue
+        setting = f"myst_{attribute.name}"
+        val = getattr(settings, setting, DOCUTILS_UNSET)
+        with suppress(AttributeError):
+            delattr(settings, setting)
+        if val is not DOCUTILS_UNSET:
+            values[attribute.name] = val
+    values["renderer"] = "docutils"
+    return MdParserConfig(**values)
+
+
+class Parser(RstParser):
+    """Docutils parser for Markedly Structured Text (MyST)."""
+
+    supported: Tuple[str, ...] = ("md", "markdown", "myst")
+    """Aliases this parser supports."""
+
+    settings_spec = (
+        *RstParser.settings_spec,
+        "MyST options",
+        None,
+        _myst_docutils_setting_tuples(),
+    )
+    """Runtime settings specification."""
 
     config_section = "myst parser"
     config_section_dependencies = ("parsers",)
@@ -41,9 +162,12 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
 
         :param inputstring: The source string to parse
         :param document: The root docutils node to add AST elements to
-
         """
-        config = MdParserConfig(renderer="docutils")
+        try:
+            config = create_myst_config(document.settings)
+        except (TypeError, ValueError) as error:
+            document.reporter.error(f"myst configuration invalid: {error.args[0]}")
+            config = MdParserConfig(renderer="docutils")
         parser = default_parser(config)
         parser.options["document"] = document
         env: dict = {}
@@ -53,3 +177,40 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
             # specified in the sphinx configuration
             tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens
         parser.renderer.render(tokens, parser.options, env)
+
+
+def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]):
+    """Run the command line interface for a particular writer."""
+    publish_cmdline(
+        parser=Parser(),
+        writer_name=writer_name,
+        description=(
+            f"Generates {writer_description} from standalone MyST sources.\n{default_description}"
+        ),
+        argv=argv,
+    )
+
+
+def cli_html(argv: Optional[List[str]] = None) -> None:
+    """Cmdline entrypoint for converting MyST to HTML."""
+    _run_cli("html", "(X)HTML documents", argv)
+
+
+def cli_html5(argv: Optional[List[str]] = None):
+    """Cmdline entrypoint for converting MyST to HTML5."""
+    _run_cli("html5", "HTML5 documents", argv)
+
+
+def cli_latex(argv: Optional[List[str]] = None):
+    """Cmdline entrypoint for converting MyST to LaTeX."""
+    _run_cli("latex", "LaTeX documents", argv)
+
+
+def cli_xml(argv: Optional[List[str]] = None):
+    """Cmdline entrypoint for converting MyST to XML."""
+    _run_cli("xml", "Docutils-native XML", argv)
+
+
+def cli_pseudoxml(argv: Optional[List[str]] = None):
+    """Cmdline entrypoint for converting MyST to pseudo-XML."""
+    _run_cli("pseudoxml", "pseudo-XML", argv)

myst_parser/docutils_renderer.py

@@ -52,9 +52,9 @@
 from .utils import is_external_url
 
 
-def make_document(source_path="notset") -> nodes.document:
-    """Create a new docutils document."""
-    settings = OptionParser(components=(RSTParser,)).get_default_values()
+def make_document(source_path="notset", parser_cls=RSTParser) -> nodes.document:
+    """Create a new docutils document, with the parser classes' default settings."""
+    settings = OptionParser(components=(parser_cls,)).get_default_values()
     return new_document(source_path, settings=settings)
 
 
@@ -474,6 +474,15 @@ def render_fence(self, token: SyntaxTreeNode) -> None:
         self.add_line_and_source_path(node, token)
         self.current_node.append(node)
 
+    @property
+    def blocks_mathjax_processing(self) -> bool:
+        """Only add mathjax ignore classes if using sphinx and myst_update_mathjax is True."""
+        return (
+            self.sphinx_env is not None
+            and "myst_update_mathjax" in self.sphinx_env.config
+            and self.sphinx_env.config.myst_update_mathjax
+        )
+
     def render_heading(self, token: SyntaxTreeNode) -> None:
 
         if self.md_env.get("match_titles", None) is False:
@@ -494,13 +503,7 @@ def render_heading(self, token: SyntaxTreeNode) -> None:
         self.add_line_and_source_path(title_node, token)
 
         new_section = nodes.section()
-        if level == 1 and (
-            self.sphinx_env is None
-            or (
-                "myst_update_mathjax" in self.sphinx_env.config
-                and self.sphinx_env.config.myst_update_mathjax
-            )
-        ):
+        if level == 1 and self.blocks_mathjax_processing:
             new_section["classes"].extend(["tex2jax_ignore", "mathjax_ignore"])
         self.add_line_and_source_path(new_section, token)
         new_section.append(title_node)

myst_parser/main.py

@@ -37,19 +37,46 @@ class MdParserConfig:
     renderer: str = attr.ib(
         default="sphinx", validator=in_(["sphinx", "html", "docutils"])
     )
-    commonmark_only: bool = attr.ib(default=False, validator=instance_of(bool))
-    enable_extensions: Iterable[str] = attr.ib(factory=lambda: ["dollarmath"])
+    commonmark_only: bool = attr.ib(
+        default=False,
+        validator=instance_of(bool),
+        metadata={"help": "Use strict CommonMark parser"},
+    )
+    enable_extensions: Iterable[str] = attr.ib(
+        factory=lambda: ["dollarmath"], metadata={"help": "Enable extensions"}
+    )
 
-    dmath_allow_labels: bool = attr.ib(default=True, validator=instance_of(bool))
-    dmath_allow_space: bool = attr.ib(default=True, validator=instance_of(bool))
-    dmath_allow_digits: bool = attr.ib(default=True, validator=instance_of(bool))
-    dmath_double_inline: bool = attr.ib(default=False, validator=instance_of(bool))
+    dmath_allow_labels: bool = attr.ib(
+        default=True,
+        validator=instance_of(bool),
+        metadata={"help": "Parse `$$...$$ (label)`"},
+    )
+    dmath_allow_space: bool = attr.ib(
+        default=True,
+        validator=instance_of(bool),
+        metadata={"help": "dollarmath: allow initial/final spaces in `$ ... $`"},
+    )
+    dmath_allow_digits: bool = attr.ib(
+        default=True,
+        validator=instance_of(bool),
+        metadata={"help": "dollarmath: allow initial/final digits `1$ ...$2`"},
+    )
+    dmath_double_inline: bool = attr.ib(
+        default=False,
+        validator=instance_of(bool),
+        metadata={"help": "dollarmath: parse inline `$$ ... $$`"},
+    )
 
-    update_mathjax: bool = attr.ib(default=True, validator=instance_of(bool))
+    update_mathjax: bool = attr.ib(
+        default=True,
+        validator=instance_of(bool),
+        metadata={"help": "Update sphinx.ext.mathjax configuration"},
+    )
 
     mathjax_classes: str = attr.ib(
         default="tex2jax_process|mathjax_process|math|output_area",
         validator=instance_of(str),
+        metadata={"help": "MathJax classes to add to math HTML"},
     )
 
     @enable_extensions.validator
@@ -77,41 +104,59 @@ def check_extensions(self, attribute, value):
     disable_syntax: Iterable[str] = attr.ib(
         factory=list,
         validator=deep_iterable(instance_of(str), instance_of((list, tuple))),
+        metadata={"help": "Disable syntax elements"},
     )
 
     # see https://en.wikipedia.org/wiki/List_of_URI_schemes
     url_schemes: Optional[Iterable[str]] = attr.ib(
         default=cast(Optional[Iterable[str]], ("http", "https", "mailto", "ftp")),
         validator=optional(deep_iterable(instance_of(str), instance_of((list, tuple)))),
+        metadata={"help": "URL schemes to allow in links"},
     )
 
     heading_anchors: Optional[int] = attr.ib(
-        default=None, validator=optional(in_([1, 2, 3, 4, 5, 6, 7]))
+        default=None,
+        validator=optional(in_([1, 2, 3, 4, 5, 6, 7])),
+        metadata={"help": "Heading level depth to assign HTML anchors"},
     )
 
     heading_slug_func: Optional[Callable[[str], str]] = attr.ib(
-        default=None, validator=optional(is_callable())
+        default=None,
+        validator=optional(is_callable()),
+        metadata={"help": "Function for creating heading anchors"},
     )
 
     html_meta: Dict[str, str] = attr.ib(
         factory=dict,
         validator=deep_mapping(instance_of(str), instance_of(str), instance_of(dict)),
         repr=lambda v: str(list(v)),
+        metadata={"help": "HTML meta tags"},
     )
 
-    footnote_transition: bool = attr.ib(default=True, validator=instance_of(bool))
+    footnote_transition: bool = attr.ib(
+        default=True,
+        validator=instance_of(bool),
+        metadata={"help": "Place a transition before any footnotes"},
+    )
 
     substitutions: Dict[str, Union[str, int, float]] = attr.ib(
         factory=dict,
         validator=deep_mapping(
             instance_of(str), instance_of((str, int, float)), instance_of(dict)
         ),
         repr=lambda v: str(list(v)),
+        metadata={"help": "Substitutions"},
     )
 
-    sub_delimiters: Tuple[str, str] = attr.ib(default=("{", "}"))
+    sub_delimiters: Tuple[str, str] = attr.ib(
+        default=("{", "}"), metadata={"help": "Substitution delimiters"}
+    )
 
-    words_per_minute: int = attr.ib(default=200, validator=instance_of(int))
+    words_per_minute: int = attr.ib(
+        default=200,
+        validator=instance_of(int),
+        metadata={"help": "For reading speed calculations"},
+    )
 
     @sub_delimiters.validator
     def check_sub_delimiters(self, attribute, value):