From 1b8b6801d4f896409a765d302e7e0d50ff089692 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Thu, 11 Jul 2024 23:43:26 +1000 Subject: [PATCH] [CI] Add index page generator. (#23737) --- .github/workflows/ci_build_major_branch.yml | 24 +++- .gitignore | 1 + util/ci/index_generator.py | 109 +++++++++++++++ util/ci/requirements.txt | 2 + util/ci/templates/index.html.j2 | 139 ++++++++++++++++++++ 5 files changed, 269 insertions(+), 6 deletions(-) create mode 100755 util/ci/index_generator.py create mode 100644 util/ci/templates/index.html.j2 diff --git a/.github/workflows/ci_build_major_branch.yml b/.github/workflows/ci_build_major_branch.yml index 77755ba71f41..0e772c7d5361 100644 --- a/.github/workflows/ci_build_major_branch.yml +++ b/.github/workflows/ci_build_major_branch.yml @@ -77,44 +77,56 @@ jobs: runs-on: ubuntu-latest steps: + - name: Disable safe.directory check + run: | + git config --global --add safe.directory '*' + + - name: Checkout QMK Firmware + uses: actions/checkout@v4 + - name: Download firmwares uses: actions/download-artifact@v4 with: pattern: firmware-* - path: firmwares + path: . merge-multiple: true + - name: Generate index page + run: | + python3 -m pip install -r ./util/ci/requirements.txt + ./util/ci/index_generator.py > index.html + - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }} uses: jakejarvis/s3-sync-action@master with: - args: --acl public-read --follow-symlinks --delete + args: --acl public-read --follow-symlinks --delete --exclude '*' --include 'index.html' --include '*.hex' --include '*.bin' --include '*.uf2' env: AWS_S3_BUCKET: ${{ vars.CI_QMK_FM_SPACES_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }} AWS_REGION: ${{ vars.CI_QMK_FM_SPACES_REGION }} AWS_S3_ENDPOINT: ${{ vars.CI_QMK_FM_SPACES_ENDPOINT }} - SOURCE_DIR: firmwares + SOURCE_DIR: . DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }} - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest uses: jakejarvis/s3-sync-action@master with: - args: --acl public-read --follow-symlinks --delete + args: --acl public-read --follow-symlinks --delete --exclude '*' --include 'index.html' --include '*.hex' --include '*.bin' --include '*.uf2' env: AWS_S3_BUCKET: ${{ vars.CI_QMK_FM_SPACES_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }} AWS_REGION: ${{ vars.CI_QMK_FM_SPACES_REGION }} AWS_S3_ENDPOINT: ${{ vars.CI_QMK_FM_SPACES_ENDPOINT }} - SOURCE_DIR: firmwares + SOURCE_DIR: . DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest - name: Check if failure marker file exists id: check_failure_marker uses: andstor/file-existence-action@v3 with: - files: firmwares/.failed + files: ./.failed - name: Fail build if needed if: steps.check_failure_marker.outputs.files_exists == 'true' diff --git a/.gitignore b/.gitignore index 35b128606d7d..5de38c161cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ *.la *.stackdump *.sym +index.html # QMK-specific api_data/v1 diff --git a/util/ci/index_generator.py b/util/ci/index_generator.py new file mode 100755 index 000000000000..b6440bbffb71 --- /dev/null +++ b/util/ci/index_generator.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import os +import re +import shlex +import subprocess +from pathlib import Path + +from ansi2html import Ansi2HTMLConverter +from jinja2 import Environment, FileSystemLoader, select_autoescape + +orig_cwd = os.getcwd() +qmk_firmware_dir = Path(os.path.realpath(__file__)).parents[2] +build_dir = qmk_firmware_dir / ".build" + +KEYBOARD_PATTERN = re.compile("CI Metadata: KEYBOARD=(?P.*)\r?\n") +KEYMAP_PATTERN = re.compile("CI Metadata: KEYMAP=(?P.*)\r?\n") + +env = Environment( + loader=FileSystemLoader(Path(os.path.realpath(__file__)).parent / "templates"), + autoescape=select_autoescape(), +) + + +def _run(command, capture_output=True, combined_output=False, text=True, **kwargs): + if isinstance(command, str): + command = shlex.split(command) + if capture_output: + kwargs["stdout"] = subprocess.PIPE + kwargs["stderr"] = subprocess.PIPE + if combined_output: + kwargs["stderr"] = subprocess.STDOUT + if "stdin" in kwargs and kwargs["stdin"] is None: + del kwargs["stdin"] + if text: + kwargs["universal_newlines"] = True + return subprocess.run(command, **kwargs) + + +def _ansi2html(value): + return Ansi2HTMLConverter().convert(value, full=False) + + +def _ansi2html_styles(): + from ansi2html.style import get_styles + + styles = get_styles(scheme="dracula") + return "\n".join([str(s) for s in styles]) + + +def _git_log(count = 4): + os.chdir(qmk_firmware_dir) + ret = _run(f"git log -n {count} --color=always --no-merges --topo-order --stat").stdout.strip() + os.chdir(orig_cwd) + return ret + +def _git_describe(): + os.chdir(qmk_firmware_dir) + ret = _run("git describe --tags --always --dirty").stdout.strip() + os.chdir(orig_cwd) + return ret + +def _git_revision(): + os.chdir(qmk_firmware_dir) + ret = _run("git rev-parse HEAD").stdout.strip() + os.chdir(orig_cwd) + return ret + + +env.filters["ansi2html"] = _ansi2html + +binaries = [] +binaries.extend(qmk_firmware_dir.glob("*.bin")) +binaries.extend(qmk_firmware_dir.glob("*.hex")) +binaries.extend(qmk_firmware_dir.glob("*.uf2")) +binaries = list(sorted(binaries)) + +failures = [] +for mdfile in list(sorted(build_dir.glob("failed.log.*"))): + txt = Path(mdfile).read_text() + + m_kb = KEYBOARD_PATTERN.search(txt) + if not m_kb: + raise Exception("Couldn't determine the keyboard from the failure output") + m_km = KEYMAP_PATTERN.search(txt) + if not m_km: + raise Exception("Couldn't determine the keymap from the failure output") + + txt = KEYBOARD_PATTERN.sub("", KEYMAP_PATTERN.sub("", txt)).strip() + + failures.append( + { + "stdout": txt, + "keyboard": m_kb.group("keyboard"), + "keymap": m_km.group("keymap"), + } + ) + +template = env.get_template("index.html.j2") +print( + template.render( + ansi2html_styles=_ansi2html_styles(), + git_log=_git_log(), + git_describe=_git_describe(), + git_revision=_git_revision(), + binaries=binaries, + failures=failures, + ) +) diff --git a/util/ci/requirements.txt b/util/ci/requirements.txt index 3196568e1a7c..47acf85644f5 100644 --- a/util/ci/requirements.txt +++ b/util/ci/requirements.txt @@ -1 +1,3 @@ discord-webhook +Jinja2 +ansi2html diff --git a/util/ci/templates/index.html.j2 b/util/ci/templates/index.html.j2 new file mode 100644 index 000000000000..18086987db06 --- /dev/null +++ b/util/ci/templates/index.html.j2 @@ -0,0 +1,139 @@ + + + + + + + + + +
+
+
+ +
+
+
+ CI Build {% if failures | length > 0 %}❌{% else %}✅{% endif %} +
+
+ {{ git_describe }} +
+
+ {{ git_revision }} +
+
+ {{ failures | length }} failure{% if failures | length != 1 %}s{% endif %} +
+
+ + {% if binaries | length > 0 %}Firmwares | {% endif %} + Commit info + {% if failures | length > 0 %} | Failure Logs{% endif %} + +
+
+
+ +
+

Git commit

+
+ 
{{ git_log | ansi2html }}
+
+
+ {% if failures | length > 0 %} + +
+

Build failure logs

+ +
+ {% for failure in failures %} + +
+

Build failure — {{ failure.keyboard }}:{{ failure.keymap }}

+
+ 
{{ failure.stdout | ansi2html }}
+
+
+ {% endfor %} + {% endif %} +
+ {% if binaries | length > 0 %} +
+ +
+

Firmware downloads

+
+ {% for binary in binaries %} +

{{ binary.name }}

+ {%- endfor %} +
+
+
+ {% endif %} + + +