From cd9e5588afc46698f762ce0403a1296d22c9d491 Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Tue, 27 Aug 2024 15:32:05 +0000 Subject: [PATCH] Extend number of macros --- docs/options.md | 2 +- mkdocs_table_reader_plugin/markdown.py | 2 +- mkdocs_table_reader_plugin/plugin.py | 28 ++++++--- mkdocs_table_reader_plugin/readers.py | 64 ++++++++++++++++++--- setup.py | 2 +- tests/fixtures/jinja/docs/index.md | 10 ++++ tests/fixtures/jinja/docs/numeric_table.csv | 4 ++ tests/fixtures/search_problem/docs/index.md | 21 +++++++ tests/fixtures/search_problem/mkdocs.yml | 10 ++++ tests/test_markdown.py | 4 +- 10 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/jinja/docs/numeric_table.csv create mode 100644 tests/fixtures/search_problem/docs/index.md create mode 100644 tests/fixtures/search_problem/mkdocs.yml diff --git a/docs/options.md b/docs/options.md index 0853e17..ed5a5fe 100644 --- a/docs/options.md +++ b/docs/options.md @@ -38,7 +38,7 @@ Default: `False`. When enabled, if a filepath is not found, the plugin will rais ## `select_readers` -Default: Selects all available readers. Specify a list of readers to improve documentation build times for very large sites. +Default: Selects all available readers. Specify a list of readers to improve documentation build times for very large sites. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) ## `enabled` diff --git a/mkdocs_table_reader_plugin/markdown.py b/mkdocs_table_reader_plugin/markdown.py index 919d09e..bc66c6f 100644 --- a/mkdocs_table_reader_plugin/markdown.py +++ b/mkdocs_table_reader_plugin/markdown.py @@ -19,7 +19,7 @@ def replace_unescaped_pipes(text: str) -> str: return re.sub(r"(? str: +def convert_to_md_table(df: pd.DataFrame, **markdown_kwargs: Dict) -> str: """ Convert dataframe to markdown table using tabulate. """ diff --git a/mkdocs_table_reader_plugin/plugin.py b/mkdocs_table_reader_plugin/plugin.py index 444d474..62ab21b 100644 --- a/mkdocs_table_reader_plugin/plugin.py +++ b/mkdocs_table_reader_plugin/plugin.py @@ -5,8 +5,8 @@ from mkdocs.exceptions import ConfigurationError from mkdocs_table_reader_plugin.safe_eval import parse_argkwarg -from mkdocs_table_reader_plugin.readers import READERS -from mkdocs_table_reader_plugin.markdown import fix_indentation, add_indentation +from mkdocs_table_reader_plugin.readers import READERS, MACROS +from mkdocs_table_reader_plugin.markdown import fix_indentation, add_indentation, convert_to_md_table logger = get_plugin_logger("table-reader") @@ -70,13 +70,23 @@ def on_config(self, config, **kwargs): ) if "macros" in config.plugins: - config.plugins["macros"].macros.update(self.readers) - config.plugins["macros"].variables["macros"].update(self.readers) - config.plugins["macros"].env.globals.update(self.readers) - - config.plugins["macros"].filters.update({"add_indentation": add_indentation}) - config.plugins["macros"].variables["filters"].update({"add_indentation": add_indentation}) - config.plugins["macros"].env.filters.update({"add_indentation": add_indentation}) + self.macros = { + macro: MACROS[macro].set_config_context( + mkdocs_config=config, plugin_config=self.config + ) + for macro in MACROS + } + self.filters = { + "add_indentation": add_indentation, + "convert_to_md_table": convert_to_md_table, + } + config.plugins["macros"].macros.update(self.macros) + config.plugins["macros"].variables["macros"].update(self.macros) + config.plugins["macros"].env.globals.update(self.macros) + + config.plugins["macros"].filters.update(self.filters) + config.plugins["macros"].variables["filters"].update(self.filters) + config.plugins["macros"].env.filters.update(self.filters) self.external_jinja_engine = True else: diff --git a/mkdocs_table_reader_plugin/readers.py b/mkdocs_table_reader_plugin/readers.py index 7794625..2144d62 100644 --- a/mkdocs_table_reader_plugin/readers.py +++ b/mkdocs_table_reader_plugin/readers.py @@ -60,22 +60,38 @@ def __call__(self, *args, **kwargs): return self.func(valid_file_paths[0], *args, **kwargs) +@ParseArgs +def pd_read_csv(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_csv) + return pd.read_csv(*args, **read_kwargs) + @ParseArgs def read_csv(*args, **kwargs) -> str: read_kwargs = kwargs_in_func(kwargs, pd.read_csv) df = pd.read_csv(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_csv) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) +@ParseArgs +def pd_read_table(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_table) + return pd.read_table(*args, **read_kwargs) + @ParseArgs def read_table(*args, **kwargs) -> str: read_kwargs = kwargs_in_func(kwargs, pd.read_table) df = pd.read_table(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_table) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) + + +@ParseArgs +def pd_read_fwf(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_fwf) + return pd.read_fwf(*args, **read_kwargs) @ParseArgs @@ -84,7 +100,12 @@ def read_fwf(*args, **kwargs) -> str: df = pd.read_fwf(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_fwf) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) + +@ParseArgs +def pd_read_json(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_json) + return pd.read_json(*args, **read_kwargs) @ParseArgs @@ -93,7 +114,12 @@ def read_json(*args, **kwargs) -> str: df = pd.read_json(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_json) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) + +@ParseArgs +def pd_read_excel(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_excel) + return pd.read_excel(*args, **read_kwargs) @ParseArgs @@ -102,9 +128,16 @@ def read_excel(*args, **kwargs) -> str: df = pd.read_excel(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_excel) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) +@ParseArgs +def pd_read_yaml(*args, **kwargs) -> str: + json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) + with open(args[0], "r") as f: + df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) + return df + @ParseArgs def read_yaml(*args, **kwargs) -> str: json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) @@ -112,7 +145,13 @@ def read_yaml(*args, **kwargs) -> str: df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.json_normalize) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) + + +@ParseArgs +def pd_read_feather(*args, **kwargs) -> str: + read_kwargs = kwargs_in_func(kwargs, pd.read_feather) + return pd.read_feather(*args, **read_kwargs) @ParseArgs @@ -121,7 +160,7 @@ def read_feather(*args, **kwargs) -> str: df = pd.read_feather(*args, **read_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_feather) - return convert_to_md_table(df, markdown_kwargs) + return convert_to_md_table(df, **markdown_kwargs) @ParseArgs @@ -145,3 +184,14 @@ def read_raw(*args, **kwargs) -> str: "read_feather": read_feather, "read_raw": read_raw, } + +MACRO_ONLY = { + "pd_read_csv": pd_read_csv, + "pd_read_table": pd_read_table, + "pd_read_fwf": pd_read_fwf, + "pd_read_excel": pd_read_excel, + "pd_read_yaml": pd_read_yaml, + "pd_read_json": pd_read_json, + "pd_read_feather": pd_read_feather, +} +MACROS = {**READERS, **MACRO_ONLY} \ No newline at end of file diff --git a/setup.py b/setup.py index f3fedda..ecd1904 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="mkdocs-table-reader-plugin", - version="3.0.1", + version="3.1.0", description="MkDocs plugin to directly insert tables from files into markdown.", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/fixtures/jinja/docs/index.md b/tests/fixtures/jinja/docs/index.md index 82d4f8d..97dc580 100644 --- a/tests/fixtures/jinja/docs/index.md +++ b/tests/fixtures/jinja/docs/index.md @@ -27,3 +27,13 @@ This is a table that we load from the docs folder, because we set `data_path` to {% endfor %} +## Filtering results + +{% raw %} +``` +{{ pd_read_csv("numeric_table.csv").query("a >= 3") | convert_to_md_table }} +``` +{% endraw %} + +{{ pd_read_csv("numeric_table.csv").query("a >= 3") | convert_to_md_table }} + diff --git a/tests/fixtures/jinja/docs/numeric_table.csv b/tests/fixtures/jinja/docs/numeric_table.csv new file mode 100644 index 0000000..598f36c --- /dev/null +++ b/tests/fixtures/jinja/docs/numeric_table.csv @@ -0,0 +1,4 @@ +"a","b" +1,2 +3,4 +5,6 \ No newline at end of file diff --git a/tests/fixtures/search_problem/docs/index.md b/tests/fixtures/search_problem/docs/index.md new file mode 100644 index 0000000..3ce06e2 --- /dev/null +++ b/tests/fixtures/search_problem/docs/index.md @@ -0,0 +1,21 @@ +# Homepage + +```json +{ + "configuration": [ + { + "category": "Advanced", + "component": "NetworkModel", + "defaultvalue": "false", + "description": "This will determine if the admin is allowed to deploy.", + "displaytext": "Admin is allowed to deploy", + "group": "Miscellaneous", + "isdynamic": true, + "name": "deploy.anywhere", + "subgroup": "Others", + "type": "Boolean", + "value": "true" + } + ] +} +``` \ No newline at end of file diff --git a/tests/fixtures/search_problem/mkdocs.yml b/tests/fixtures/search_problem/mkdocs.yml new file mode 100644 index 0000000..fa55d35 --- /dev/null +++ b/tests/fixtures/search_problem/mkdocs.yml @@ -0,0 +1,10 @@ +site_name: test search + +theme: + name: material + features: + - navigation.tabs + +plugins: + - search + - table-reader \ No newline at end of file diff --git a/tests/test_markdown.py b/tests/test_markdown.py index f3a11c1..9425820 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -14,6 +14,6 @@ def test_convert_to_md_table(): assert df_good.shape[0] > 0 # Because we escape pipes, the 'bad' df - md_bad = convert_to_md_table(df_bad, markdown_kwargs={}) - md_good = convert_to_md_table(df_good, markdown_kwargs={}) + md_bad = convert_to_md_table(df_bad, **{}) + md_good = convert_to_md_table(df_good, **{}) assert md_bad == md_good \ No newline at end of file