stubber: Add --no-clean-rst option and notebook to copy stubs.

Signed-off-by: Jos Verlinde <Jos.Verlinde@microsoft.com>

src/stubber/commands/get_docstubs_cmd.py

@@ -47,12 +47,14 @@
     "--version", "--tag", default="", type=str, help="Version number to use. [default: Git tag]"
 )
 @click.option("--black/--no-black", "-b/-nb", default=True, help="Run black", show_default=True)
+@click.option("--clean-rst/--no-clean-rst", "-b/-nb", default=True, help="remove .rST constructs from the docstrings", show_default=True)
 @click.pass_context
 def cli_docstubs(
     ctx: click.Context,
     path: Optional[str] = None,
     target: Optional[str] = None,
     black: bool = True,
+    clean_rst: bool = True,
     basename: Optional[str] = None,
     version: str = "",
 ):
@@ -88,5 +90,5 @@ def cli_docstubs(
     dst_path = Path(target) / f"{basename}-{utils.clean_version(version, flat=True)}-docstubs"
 
     log.info(f"Get docstubs for MicroPython {utils.clean_version(version, drop_v=False)}")
-    generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black)
+    generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black, clean_rst=clean_rst)
     log.info("::group:: Done")

src/stubber/rst/lookup.py

@@ -12,6 +12,7 @@
     "NONE_VERBS",
     "CHILD_PARENT_CLASS",
     "PARAM_FIXES",
+    "PARAM_RE_FIXES",
     "MODULE_GLUE",
     "RST_DOC_FIXES",
     "DOCSTUB_SKIP",
@@ -494,6 +495,18 @@ class Fix:
     ),
 ]
 
+# and some param fixes that require a regex
+PARAM_RE_FIXES = [
+    Fix(
+        r"\[angle, time=0\]", "[angle], time=0", is_re=True
+    ),  # fix: method:: Servo.angle([angle, time=0])
+    Fix(
+        r"\[speed, time=0\]", "[speed], time=0", is_re=True
+    ),  # fix: .. method:: Servo.speed([speed, time=0])
+    Fix(
+        r"\[service_id, key=None, \*, \.\.\.\]", "[service_id], [key], *, ...", is_re=True
+    ),  # fix: network - AbstractNIC.connect
+]
 # List of classes and their parent classes that should be added to the class definition
 CHILD_PARENT_CLASS = {
     # machine

src/stubber/rst/reader.py

@@ -73,6 +73,7 @@
     CHILD_PARENT_CLASS,
     MODULE_GLUE,
     PARAM_FIXES,
+    PARAM_RE_FIXES,
     RST_DOC_FIXES,
     TYPING_IMPORT,
     ClassSourceDict,
@@ -185,6 +186,7 @@ def __init__(self):
         self.current_module = ""
         self.current_class = ""
         self.current_function = ""  # function & method
+        self.clean_rst = True
         super().__init__()
 
     def read_file(self, filename: Path):
@@ -223,7 +225,7 @@ def at_anchor(self) -> bool:
         # return _l.startswith("..") and not any(_l.startswith(a) for a in self.docstring_anchors)
 
     # @property
-    def at_heading(self, large:bool=False) -> bool:
+    def at_heading(self, large: bool = False) -> bool:
         "stop at heading"
         u_line = self.rst_text[min(self.line_no + 1, self.max_line - 1)].rstrip()
         # Heading  ---, ==, ~~~
@@ -294,24 +296,24 @@ def read_docstring(self, large: bool = False) -> List[str]:
             block[0] = block[0][0].upper() + block[0][1:]
         return block
 
-    @staticmethod
-    def clean_docstr(block: List[str]):
+    # @staticmethod
+    def clean_docstr(self, block: List[str]):
         """Clean up a docstring"""
-        # if a Quoted Literal Block , then remove the first character of each line
-        # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks
-        if block and len(block[0]) > 0 and block[0][0] != " ":
-            q_char = block[0][0]
-            if all(l.startswith(q_char) for l in block):
-                # all lines start with the same character, so skip that character
-                block = [l[1:] for l in block]
+        if self.clean_rst:
+            # if a Quoted Literal Block , then remove the first character of each line
+            # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks
+            if block and len(block[0]) > 0 and block[0][0] != " ":
+                q_char = block[0][0]
+                if all(l.startswith(q_char) for l in block):
+                    # all lines start with the same character, so skip that character
+                    block = [l[1:] for l in block]
         # rstrip all lines
         block = [l.rstrip() for l in block]
         # remove empty lines at beginning/end of block
         while len(block) and len(block[0]) == 0:
             block = block[1:]
         while len(block) and len(block[-1]) == 0:
             block = block[:-1]
-
         # Clean up Synopsis
         if len(block) and ":synopsis:" in block[0]:
             block[0] = re.sub(
@@ -321,9 +323,11 @@ def clean_docstr(block: List[str]):
             )
         return block
 
-    @staticmethod
-    def add_link_to_docstr(block: List[str]):
+    def add_link_to_docstr(self, block: List[str]):
         """Add clickable hyperlinks to CPython docpages"""
+        if not self.clean_rst:
+            return block
+
         for i in range(len(block)):
             # hyperlink to Cpython doc pages
             # https://regex101.com/r/5RN8rj/1
@@ -374,18 +378,6 @@ class RSTParser(RSTReader):
     """
 
     target = ".py"  # py/pyi
-    # TODO: Move to lookup.py
-    PARAM_RE_FIXES = [
-        Fix(
-            r"\[angle, time=0\]", "[angle], time=0", is_re=True
-        ),  # fix: method:: Servo.angle([angle, time=0])
-        Fix(
-            r"\[speed, time=0\]", "[speed], time=0", is_re=True
-        ),  # fix: .. method:: Servo.speed([speed, time=0])
-        Fix(
-            r"\[service_id, key=None, \*, \.\.\.\]", "[service_id], [key], *, ...", is_re=True
-        ),  # fix: network - AbstractNIC.connect
-    ]
 
     def __init__(self, v_tag: str) -> None:
         super().__init__()
@@ -414,7 +406,7 @@ def fix_parameters(self, params: str, name: str = "") -> str:
 
         ## Deal with SQUARE brackets first ( Documentation meaning := [optional])
 
-        for fix in self.PARAM_RE_FIXES:
+        for fix in PARAM_RE_FIXES:
             params = self.apply_fix(fix, params, name)
 
         # ###########################################################################################################

src/stubber/stubs_from_docs.py

@@ -22,6 +22,7 @@ def generate_from_rst(
     pattern: str = "*.rst",
     suffix: str = ".pyi",
     black: bool = True,
+    clean_rst: bool = True,
 ) -> int:
     dst_path.mkdir(parents=True, exist_ok=True)
     if not release:
@@ -36,7 +37,7 @@ def generate_from_rst(
     # files = [f for f in files if "machine" in f.name]
 
     clean_destination(dst_path)
-    make_docstubs(dst_path, v_tag, release, suffix, files)
+    make_docstubs(dst_path, v_tag, release, suffix, files, clean_rst=clean_rst)
 
     log.info("::group:: start post processing of retrieved stubs")
     # do not run stubgen
@@ -72,11 +73,12 @@ def get_rst_sources(rst_path: Path, pattern: str) -> List[Path]:
     return files
 
 
-def make_docstubs(dst_path: Path, v_tag: str, release: str, suffix: str, files: List[Path]):
+def make_docstubs(dst_path: Path, v_tag: str, release: str, suffix: str, files: List[Path],clean_rst:bool):
     """Create the docstubs"""
 
     for file in files:
         reader = RSTWriter(v_tag)
+        reader.clean_rst = clean_rst
         reader.source_release = release
         log.debug(f"Reading: {file}")
         reader.read_file(file)

src/stubber/update_module_list.py

@@ -23,7 +23,7 @@ def read_modules(path: Optional[Path] = None) -> Set[str]:
     read text files with modules per firmware.
     each contains the output of help("modules")
     - lines starting with # are comments.
-    - split the other lines at whitespace seperator,
+    - split the other lines at whitespace separator,
     - and add each module to a set
     """
     path = Path(path or "./data")
@@ -73,7 +73,7 @@ def update_module_list():
         "upip_utarfile",
         "upysh",
         "uasyncio",
-        "builtins",
+        # "builtins", # may be used for type hints
         "re",
     }
     log.info("Update the module list in createstubs.py")