compatible: Add sync_docs.yaml

.github/workflows/_sync_docs_v2.md

@@ -0,0 +1,74 @@
+Workflow file: [_sync_docs_v2.yaml](_sync_docs_v2.yaml)
+
+> [!WARNING]
+> Subject to **breaking changes on patch release**. `_sync_docs_v2.yaml` is experimental & not part of the public interface.
+
+## Usage
+Add `.yaml` file to `.github/workflows/`
+
+```yaml
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: Sync Discourse docs (v2)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: # Refer to Run schedule below
+
+jobs:
+  sync-docs-v2:
+    name: Sync docs from Discourse (v2)
+    uses: canonical/data-platform-workflows/.github/workflows/_sync_docs_2.yaml@main
+    permissions:
+      contents: write  # Needed to push branch & tag
+      pull-requests: write  # Needed to create PR
+```
+
+## Run schedule
+
+Cron job schedules for Data Platform repositories
+
+### SQL
+| repository                | run time | cron          |
+|:-------------------------:|:--------:|:-------------:|
+| mysql-k8s-operator        | 12:00 AM | `00 00 * * *` |
+| mysql-operator            | 12:10 AM | `10 00 * * *` |
+| mysql-test-app            | 12:20 AM | `20 00 * * *` |
+| mysql-router-k8s-operator | 12:30 AM | `30 00 * * *` |
+| mysql-router-operator     | 12:40 AM | `40 00 * * *` |
+| postgresql-k8s-operator   | 12:50 AM | `50 00 * * *` |
+| postgresql-operator       | 01:00 AM | `00 01 * * *` |
+| postgresql-test-app       | 01:10 AM | `10 01 * * *` |
+| pgbouncer-k8s-operator    | 01:20 AM | `20 01 * * *` |
+| pgbouncer-operator        | 01:30 AM | `30 01 * * *` |
+
+### NoSQL
+| repository                     | run time | cron          |
+|:------------------------------:|:--------:|:-------------:|
+| mongodb-k8s-operator           | 01:40 AM | `40 01 * * *` |
+| mongodb-operator               | 01:50 AM | `50 01 * * *` |
+| mongos-operator                | 02:00 AM | `00 02 * * *` |
+| opensearch-k8s-operator        | 02:10 AM | `10 02 * * *` |
+| opensearch-operator            | 02:20 AM | `20 02 * * *` |
+| opensearch-dashboards-operator | 02:30 AM | `30 02 * * *` |
+| redis-k8s-operator             | 02:40 AM | `40 02 * * *` |
+| redis-operator                 | 02:50 AM | `50 02 * * *` |
+
+### Big Data
+| repository                        | run time | cron          |
+|:---------------------------------:|:--------:|:-------------:|
+| kafka-k8s-operator                | 03:00 AM | `00 03 * * *` |
+| kafka-operator                    | 03:10 AM | `10 03 * * *` |
+| kafka-test-app                    | 03:20 AM | `20 03 * * *` |
+| zookeeper-k8s-operator            | 03:30 AM | `30 03 * * *` |
+| zookeeper-operator                | 03:40 AM | `40 03 * * *` |
+| spark-history-server-k8s-operator | 03:50 AM | `50 03 * * *` |
+| spark-client-snap                 | 04:00 AM | `00 04 * * *` |
+
+### Other
+| repository         | run time | cron          |
+|:------------------:|:--------:|:-------------:|
+| data-integrator    | 04:10 AM | `10 04 * * *` |
+| s3-integrator      | 04:20 AM | `20 04 * * *` |
+| data-platform-libs | 04:30 AM | `30 04 * * *` |

.github/workflows/_sync_docs_v2.yaml

@@ -0,0 +1,51 @@
+on:
+  workflow_call:
+    inputs:
+      reviewers:
+        description: Comma separated list of GitHub usernames to request to review pull request (e.g. "canonical/data-platform-engineers,octocat")
+        required: false
+        type: string
+      secrets:
+        token:
+          description: |
+            GitHub App token or personal access token (not GITHUB_TOKEN)
+
+            Permissions needed for App token:
+            - Access: Read & write for Repository permissions: Pull requests
+            - Access: Read & write for Repository permissions: Contents
+            - If GitHub team is requested for pull request review,
+              Access: Read-only for Organization permissions: Members
+
+            Permissions needed for personal access token: write access to repository, read:org
+            Personal access tokens with fine grained access are not supported (by GraphQL API, which is used by GitHub CLI).
+
+            The GITHUB_TOKEN can create a pull request or push a branch, but `on: pull_request` workflows will not be triggered.
+
+            Source: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+          required: true
+
+jobs:
+  sync-docs-v2:
+    name: Sync Discourse docs (v2)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+  steps:
+        - name: Get workflow version
+          id: workflow-version
+          uses: canonical/get-workflow-version-action@v1
+          with:
+            repository-name: canonical/data-platform-workflows
+            file-name: update_bundle.yaml
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+        - name: Install CLI
+          run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli
+        - name: Checkout
+          uses: actions/checkout@v4
+          with:
+            token: ${{ secrets.token }}
+        - name: Download Discourse docs
+          id: sync-docs-v2
+          run:
+        - name: Push `sync-docs-v2` branch
+
+        - name: Create pull request

python/cli/data_platform_workflows_cli/download_discourse_topics.py

@@ -0,0 +1,137 @@
+import csv
+import dataclasses
+import pathlib
+import re
+
+import requests
+import yaml
+
+NAVTABLE_START_MARKER = "[details=Navigation]"
+NAVTABLE_END_MARKER = "[/details]"
+
+def get_topic(topic_id_: str):
+    """Get markdown content of a discourse.charmhub.io topic"""
+
+    response = requests.get(
+        f"https://discourse.charmhub.io/raw/{topic_id_}/1"
+    )  # "/1" for post 1
+
+    response.raise_for_status()
+    return response.text
+
+
+class NoTopicToDownload(Exception):
+    """No Discourse topic is available to download
+
+    Happens if:
+    - no "Navlink" is provided (e.g. for a navigation group)
+    - "Navlink" is an external URL
+    """
+
+class NoTableToParse(Exception):
+    """No markdown navigation table is available 
+
+    Happens if:
+    - Navtable markers do not exist in the topic (i.e. [details=Navigation][/details])
+    - The navtable markers in the topic have a typo
+    - The table is empty
+    """
+
+@dataclasses.dataclass
+class Topic:
+    """Discourse topic to download"""
+
+    id: str
+    path: pathlib.Path
+
+    @classmethod
+    def from_csv_row(cls, row_: dict):
+        # Example `row_`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'}
+
+        # Extract Discourse topic ID from "Navlink"
+        # Example `link`: "/t/9707"
+        link = re.fullmatch(r"\[.*?]\((.*?)\)", row_["Navlink"]).group(1)
+        if link == "":
+            raise NoTopicToDownload
+        elif link.startswith("http") and "charmhub.io" not in link:
+            # Ignore external links (e.g. "https://canonical.com/data/docs/postgresql/iaas")
+            raise NoTopicToDownload
+
+        match = re.fullmatch(r"/t/([0-9]+)", link)
+        if not match:
+            raise ValueError(
+                f'Invalid navlink "{link}". Expected something like "/t/9707"'
+            )
+        # Example `topic_id`: "9707"
+        topic_id = match.group(1)
+
+        # Determine local path to download Markdown file
+        # Example `topic_slug`: "t-overview"
+        topic_slug = row_["Path"]
+        diataxis_directory = {
+            "t-": "tutorial",
+            "h-": "how-to",
+            "r-": "reference",
+            "e-": "explanation",
+        }[topic_slug[:2]]
+
+        # Example `path`: "docs/tutorial/t-overview.md"
+        path = pathlib.Path("docs/") / diataxis_directory / f"{topic_slug}.md"
+
+        return cls(topic_id, path)
+
+def main():
+    # Example `overview_topic_link`: "https://discourse.charmhub.io/t/charmed-postgresql-documentation/9710"
+    overview_topic_link: str = yaml.safe_load(pathlib.Path("disc2github/metadata.yaml").read_text())["docs"]
+    assert overview_topic_link.startswith("https://discourse.charmhub.io/")
+
+    # Example `topic_id`: "9710"
+    topic_id = overview_topic_link.split("/")[-1]
+    overview_topic_markdown = get_topic(topic_id)
+
+    # Example:
+    # | Level | Path | Navlink |
+    # |--------|--------|-------------|
+    # | 1 | tutorial | [Tutorial]() |
+    # | 2 | t-overview | [Overview](/t/9707) |
+    # | 2 | t-set-up | [1. Set up the environment](/t/9709) |
+    # | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) |
+    # | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) |
+
+    # Search for table delimiters NAVTABLE_START_MARKER and NAVTABLE_END_MARKER
+    start_index = overview_topic_markdown.find(NAVTABLE_START_MARKER)
+    if start_index == -1:
+        raise NoTableToParse("Could not find Navtable start marker " + NAVTABLE_START_MARKER + " in the overview topic") 
+
+    end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER)
+    if end_index == -1:
+        raise NoTableToParse("Could not find Navtable end marker " + NAVTABLE_END_MARKER + " in the overview topic")
+
+    start_index += len(NAVTABLE_START_MARKER)
+    end_index = overview_topic_markdown.find(NAVTABLE_END_MARKER, start_index)
+
+    table_raw = overview_topic_markdown[start_index:end_index].strip() # remove leading and trailing whitespace
+    if table_raw == "":
+        raise NoTableToParse
+
+    # Convert Markdown table to list[dict[str, str]]
+    # (https://stackoverflow.com/a/78254495)
+    rows: list[dict] = list(csv.DictReader(table_raw.split("\n"), delimiter="|"))
+
+    # Remove first row (e.g. "|--------|--------|-------------|")
+    rows = rows[2:]
+    rows: list[dict[str, str]] = [
+        {key.strip(): value.strip() for key, value in row.items() if key != ''}
+        for row in rows
+    ]
+
+    # Example `row`: {'Level': '2', 'Path': 't-overview', 'Navlink': '[Overview](/t/9707)'}
+    for row in rows:
+        try:
+            topic = Topic.from_csv_row(row)
+        except NoTopicToDownload:
+            continue
+
+        # Download topic markdown to `topic.path`
+        topic.path.parent.mkdir(parents=True, exist_ok=True)
+        topic.path.write_text(get_topic(topic_id))