compatible: Add sync_docs.yaml

.github/workflows/_sync_docs_v2.md

@@ -0,0 +1,26 @@
+Workflow file: [_sync_docs_v2.yaml](_sync_docs_v2.yaml)
+
+> [!WARNING]
+> Subject to **breaking changes on patch release**. `_sync_docs_v2.yaml` is experimental & not part of the public interface.
+
+## Usage
+Add `.yaml` file to `.github/workflows/`
+
+```yaml
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: Sync Discourse docs (v2)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:
+
+jobs:
+  sync-docs-v2:
+    name: Sync docs from Discourse (v2)
+    uses: canonical/data-platform-workflows/.github/workflows/_sync_docs_2.yaml@main
+    permissions:
+      contents: write  # Needed to push branch & tag
+      pull-requests: write  # Needed to create PR
+```

.github/workflows/_sync_docs_v2.yaml

@@ -0,0 +1,50 @@
+on:
+  workflow_call:
+    inputs:
+      reviewers:
+        description: Comma separated list of GitHub usernames to request to review pull request (e.g. "canonical/data-platform-engineers,octocat")
+        required: false
+        type: string
+
+jobs:
+  sync-docs-v2:
+    name: Sync Discourse docs (v2)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+  steps:
+    - name: Get workflow version
+      id: workflow-version
+      uses: canonical/get-workflow-version-action@v1
+      with:
+        repository-name: canonical/data-platform-workflows
+        file-name: _sync_docs_v2.yaml
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install CLI
+      run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Download Discourse docs
+      id: sync-docs-v2
+      run:
+    - name: Push `sync-docs` branch
+      run: |
+        git checkout -b sync-docs
+        git add docs/
+        git config user.name "GitHub Actions"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        git commit -m "Sync docs from Discourse"
+        git push origin sync-docs -f
+    - name: Create pull request
+      run: |
+        # Capture output in variable so that step fails if `gh pr list` exits with non-zero code
+        prs=$(gh pr list --head sync-docs --state open --json number)
+        if [[ $prs != "[]" ]]
+        then
+          echo Open pull request already exists
+          exit 0
+        fi
+        gh pr create --head sync-docs --title "Sync docs from Discourse" --body "Sync charm docs from https://discourse.charmhub.io" --reviewer '${{ inputs.reviewers }}'
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -1,18 +1,19 @@
 ## Usage
 ### Workflows
-| Name                                                                       | Description                                                                |
-|----------------------------------------------------------------------------|----------------------------------------------------------------------------|
-| [lint.yaml](.github/workflows/lint.md)                                     | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint` |
-| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm                                                     |
-| [build_snap.yaml](.github/workflows/build_snap.md)                         | Build snap                                                                 |
-| [build_rock.yaml](.github/workflows/build_rock.md)                         | Build rock                                                                 |
-| [build_charm.yaml](.github/workflows/build_charm.md)                       | Build charm                                                                |
-| [release_snap.yaml](.github/workflows/release_snap.md)                     | Release snap to Snap Store                                                 |
-| [release_rock.yaml](.github/workflows/release_rock.md)                     | Release rock to GitHub Container Registry                                  |
-| [release_charm.yaml](.github/workflows/release_charm.md)                   | Release charm to Charmhub                                                  |
-| [update_bundle.yaml](.github/workflows/update_bundle.md)                   | Update charm revisions in bundle                                           |
-| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md)         | Sync GitHub issues to Jira issues                                          |
-| [_sync_docs.yaml](.github/workflows/_sync_docs.md)                         | **Experimental** Sync Discourse documentation to GitHub                    |
+| Name                                                                       | Description                                                                 |
+|----------------------------------------------------------------------------|-----------------------------------------------------------------------------|
+| [lint.yaml](.github/workflows/lint.md)                                     | Lint GitHub Actions workflows (`.github/workflows/`) and `tox run -e lint`  |
+| [integration_test_charm.yaml](.github/workflows/integration_test_charm.md) | Integration test charm                                                      |
+| [build_snap.yaml](.github/workflows/build_snap.md)                         | Build snap                                                                  |
+| [build_rock.yaml](.github/workflows/build_rock.md)                         | Build rock                                                                  |
+| [build_charm.yaml](.github/workflows/build_charm.md)                       | Build charm                                                                 |
+| [release_snap.yaml](.github/workflows/release_snap.md)                     | Release snap to Snap Store                                                  |
+| [release_rock.yaml](.github/workflows/release_rock.md)                     | Release rock to GitHub Container Registry                                   |
+| [release_charm.yaml](.github/workflows/release_charm.md)                   | Release charm to Charmhub                                                   |
+| [update_bundle.yaml](.github/workflows/update_bundle.md)                   | Update charm revisions in bundle                                            |
+| [sync_issue_to_jira.yaml](.github/workflows/sync_issue_to_jira.md)         | Sync GitHub issues to Jira issues                                           |
+| [_sync_docs.yaml](.github/workflows/_sync_docs.md)                         | **Experimental** Sync Discourse documentation to GitHub (gatekeeper action) |
+| [_sync_docs_v2.yaml](.github/workflows/_sync_docs_v2.md)                   | **Experimental** Sync Discourse documentation to GitHub (custom script)     |
 
 ### Version
 Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date.

python/cli/data_platform_workflows_cli/sync_docs_v2.py

@@ -0,0 +1,130 @@
+import csv
+import dataclasses
+import pathlib
+import re
+import shutil
+import requests
+import yaml
+
+NAVTABLE_START_MARKER = "[details=Navigation]"
+NAVTABLE_END_MARKER = "[/details]"
+
+def get_topic(topic_id_: str):
+    """Get markdown content of a discourse.charmhub.io topic"""
+
+    response = requests.get(
+        f"https://discourse.charmhub.io/raw/{topic_id_}/1"
+    )  # "/1" for post 1
+
+    response.raise_for_status()
+    return response.text
+
+
+class NoTopicToDownload(Exception):
+    """No Discourse topic is available to download
+
+    Happens if:
+    - no "Navlink" is provided (e.g. for a navigation group)
+    - "Navlink" is an external URL
+    """
+
+@dataclasses.dataclass
+class Topic:
+    """Discourse topic to download"""
+
+    id: str
+    path: pathlib.Path
+
+    @classmethod
+    def from_csv_row(cls, row_: dict):
+        # Example `row_`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'}
+
+        # Extract Discourse topic ID from "Navlink"
+        # Example `link`: "/t/9707"
+        link = re.fullmatch(r"\[.*?]\((.*?)\)", row_["Navlink"]).group(1)
+        if link == "":
+            raise NoTopicToDownload
+        elif link.startswith("http") and "discourse.charmhub.io" not in link:
+            # Ignore external links (e.g. "https://canonical.com/data/docs/postgresql/iaas")
+            raise NoTopicToDownload
+
+        match = re.fullmatch(r"/t/([0-9]+)", link)
+        if not match:
+            raise ValueError(
+                f'Invalid navlink "{link}". Expected something like "/t/9707"'
+            )
+        # Example `topic_id`: "9707"
+        topic_id = match.group(1)
+
+        # Determine local path to download Markdown file
+        # Example `topic_slug`: "t-overview"
+        topic_slug = row_["Path"]
+        diataxis_directory = {
+            "t-": "tutorial",
+            "h-": "how-to",
+            "r-": "reference",
+            "e-": "explanation",
+        }[topic_slug[:2]]
+
+        # Example `path`: "docs/tutorial/t-overview.md"
+        path = pathlib.Path("docs/") / diataxis_directory / f"{topic_slug}.md"
+
+        return cls(topic_id, path)
+
+def main():
+    """Download Discourse documentation topics to docs/ directory"""
+
+    # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710"
+    overview_topic_link: str = yaml.safe_load(pathlib.Path("metadata.yaml").read_text())["docs"]
+    assert overview_topic_link.startswith("https://discourse.charmhub.io/")
+
+    # Example `topic_id`: "9710"
+    topic_id = overview_topic_link.split("/")[-1]
+    overview_topic_markdown = get_topic(topic_id)
+
+    # Example of an expected markdown table:
+    # | Level | Path | Navlink |
+    # |--------|--------|-------------|
+    # | 1 | tutorial | [Tutorial]() |
+    # | 2 | t-overview | [Overview](/t/9707) |
+    # | 2 | t-set-up | [1. Set up the environment](/t/9709) |
+    # | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) |
+    # | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) |
+
+    # Search for table delimiters NAVTABLE_START_MARKER and NAVTABLE_END_MARKER
+    start_index = overview_topic_markdown.find(NAVTABLE_START_MARKER)
+    if start_index == -1:
+        raise ValueError("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") 
+
+    end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER)
+    if end_index == -1:
+        raise ValueError("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic")
+
+    start_index += len(NAVTABLE_START_MARKER)
+    end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER, start_index)
+
+    table_raw = overview_topic_markdown[start_index:end_index].strip() # remove leading and trailing whitespace
+    if table_raw == "":
+        raise ValueError("Could not find a valid table")
+
+    # Convert Markdown table to list[dict[str, str]]
+    # (https://stackoverflow.com/a/78254495)
+    rows: list[dict] = list(csv.DictReader(table_raw.split("\n"), delimiter="|"))
+    # Remove first row (e.g. "|--------|--------|-------------|")
+    rows = rows[2:]
+    rows: list[dict[str, str]] = [
+        {key.strip(): value.strip() for key, value in row.items() if key != ''}
+        for row in rows
+    ]
+    shutil.rmtree(pathlib.Path("docs/"))
+
+    for row in rows:
+        # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'} 
+        try:
+            topic = Topic.from_csv_row(row)
+        except NoTopicToDownload:
+            continue
+
+        # Download topic markdown to `topic.path`
+        topic.path.parent.mkdir(parents=True, exist_ok=True)
+        topic.path.write_text(get_topic(topic_id))

python/cli/pyproject.toml

@@ -21,6 +21,7 @@ parse-snap-version = "data_platform_workflows_cli.parse_snap_version:main"
 allure-add-default-for-missing-results = "data_platform_workflows_cli.allure_add_default_for_missing_results:main"
 add-ssh-keys = "data_platform_workflows_cli.add_ssh_keys:main"
 tee-log-for-all-models = "data_platform_workflows_cli.tee_log_for_all_models:main"
+sync-docs-v2 = "data_platform_workflows_cli.sync_docs_v2:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"