From c868b3a82fc6c3d42d3e8eded400f8c37b113310 Mon Sep 17 00:00:00 2001
From: James McKinney <26463+jpmckinney@users.noreply.github.com>
Date: Mon, 8 Feb 2021 17:03:19 -0500
Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Store=20rawtext=20in?=
=?UTF-8?q?=20AST=20nodes=20(for=20gettext)=20(#301)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
In this commit we ensure that the raw text is propagated
from the Markdown tokens to the Sphinx AST.
In particular, this is used by the `gettext` builder,
to generate translation POT templates.
---
myst_parser/docutils_renderer.py | 12 ++--
tests/test_sphinx/sourcedirs/gettext/conf.py | 4 ++
tests/test_sphinx/sourcedirs/gettext/index.md | 42 +++++++++++++
tests/test_sphinx/test_sphinx_builds.py | 24 ++++++++
.../test_sphinx_builds/test_gettext.pot | 61 +++++++++++++++++++
5 files changed, 139 insertions(+), 4 deletions(-)
create mode 100644 tests/test_sphinx/sourcedirs/gettext/conf.py
create mode 100644 tests/test_sphinx/sourcedirs/gettext/index.md
create mode 100644 tests/test_sphinx/test_sphinx_builds/test_gettext.pot
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index 2749eaf1..666f336d 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -325,7 +325,7 @@ def renderInlineAsText(self, tokens: List[Token]) -> str:
# ### render methods for commonmark tokens
def render_paragraph_open(self, token: NestedTokens):
- para = nodes.paragraph("")
+ para = nodes.paragraph(token.children[0].content if token.children else "")
self.add_line_and_source_path(para, token)
with self.current_node_context(para, append=True):
self.render_children(token)
@@ -449,7 +449,7 @@ def render_heading_open(self, token: NestedTokens):
if self.is_section_level(level, self.current_node):
self.current_node = cast(nodes.Element, self.current_node.parent)
- title_node = nodes.title()
+ title_node = nodes.title(token.children[0].content if token.children else "")
self.add_line_and_source_path(title_node, token)
new_section = nodes.section()
@@ -719,7 +719,9 @@ def render_table_row(self, token: Token):
with self.current_node_context(row, append=True):
for child in token.children or []:
entry = nodes.entry()
- para = nodes.paragraph("")
+ para = nodes.paragraph(
+ child.children[0].content if child.children else ""
+ )
style = child.attrGet("style") # i.e. the alignment when using e.g. :--
if style:
entry["classes"].append(style)
@@ -865,7 +867,9 @@ def render_dl_open(self, token: NestedTokens):
item = nodes.definition_list_item()
self.add_line_and_source_path(item, child)
with self.current_node_context(item, append=True):
- term = nodes.term()
+ term = nodes.term(
+ child.children[0].content if child.children else ""
+ )
self.add_line_and_source_path(term, child)
with self.current_node_context(term, append=True):
self.render_children(child)
diff --git a/tests/test_sphinx/sourcedirs/gettext/conf.py b/tests/test_sphinx/sourcedirs/gettext/conf.py
new file mode 100644
index 00000000..0dff9fe3
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/gettext/conf.py
@@ -0,0 +1,4 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build"]
+copyright = "2020, Executable Book Project"
+myst_enable_extensions = ["deflist"]
diff --git a/tests/test_sphinx/sourcedirs/gettext/index.md b/tests/test_sphinx/sourcedirs/gettext/index.md
new file mode 100644
index 00000000..59bc4674
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/gettext/index.md
@@ -0,0 +1,42 @@
+# **bold** text 1
+
+**bold** text 2
+
+> **bold** text 3
+
+```{eval-rst}
+.. note::
+
+ **bold** text 4
+```
+
+* **bold** text 5
+
+1. **bold** text 6
+
+**bold** text 7
+: **bold** text 8
+
+| **bold** text 9 |
+| ---------------- |
+| **bold** text 10 |
+
+
+
+**bold** text 11
+
+
+
+ **skip** text
+
+```
+**skip** text
+```
+
+```json
+{
+ "skip": "text"
+}
+```
+
+**skip** text
diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py
index b9ced0c9..2bd75b92 100644
--- a/tests/test_sphinx/test_sphinx_builds.py
+++ b/tests/test_sphinx/test_sphinx_builds.py
@@ -3,6 +3,7 @@
see conftest.py for fixture usage
"""
import os
+import re
import pytest
@@ -272,3 +273,26 @@ def test_substitutions(
)
finally:
get_sphinx_app_output(app, filename="index.html", regress_html=True)
+
+
+@pytest.mark.sphinx(
+ buildername="gettext", srcdir=os.path.join(SOURCE_DIR, "gettext"), freshenv=True
+)
+def test_gettext(
+ app,
+ status,
+ warning,
+ get_sphinx_app_output,
+ remove_sphinx_builds,
+ file_regression,
+):
+ """Test gettext message extraction."""
+ app.build()
+ assert "build succeeded" in status.getvalue() # Build succeeded
+ warnings = warning.getvalue().strip()
+ assert warnings == ""
+
+ output = get_sphinx_app_output(app, filename="index.pot", buildername="gettext")
+ output = re.sub(r"POT-Creation-Date: [0-9: +-]+", "POT-Creation-Date: ", output)
+
+ file_regression.check(output, extension=".pot")
diff --git a/tests/test_sphinx/test_sphinx_builds/test_gettext.pot b/tests/test_sphinx/test_sphinx_builds/test_gettext.pot
new file mode 100644
index 00000000..1496914d
--- /dev/null
+++ b/tests/test_sphinx/test_sphinx_builds/test_gettext.pot
@@ -0,0 +1,61 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Executable Book Project
+# This file is distributed under the same license as the Python package.
+# FIRST AUTHOR , YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: Python \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: \n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME \n"
+"Language-Team: LANGUAGE \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: ../../index.md:1
+msgid "**bold** text 1"
+msgstr ""
+
+#: ../../index.md:3
+msgid "**bold** text 2"
+msgstr ""
+
+#: ../../index.md:5
+msgid "**bold** text 3"
+msgstr ""
+
+#: ../../index.md:10
+msgid "**bold** text 4"
+msgstr ""
+
+#: ../../index.md:13
+msgid "**bold** text 5"
+msgstr ""
+
+#: ../../index.md:15
+msgid "**bold** text 6"
+msgstr ""
+
+#: ../../index.md:17
+msgid "**bold** text 7"
+msgstr ""
+
+#: ../../index.md:18
+msgid "**bold** text 8"
+msgstr ""
+
+#: ../../index.md:0
+msgid "**bold** text 9"
+msgstr ""
+
+#: ../../index.md:0
+msgid "**bold** text 10"
+msgstr ""
+
+#: ../../index.md:26
+msgid "**bold** text 11"
+msgstr ""