From 9346a77af167d64871c74b64e5e50e0f18de9046 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 22 Jul 2023 14:06:40 +0200 Subject: [PATCH] # This is a combination of 9 commits. # The 1st commit message will be skipped: # ci: update the books via a GitHub workflow # # With this commit, the SHAs corresponding to the various repositories # containing the ProGit Book and its translations are stored in a # sparse-checkout'able directory. # # This information is then used by a scheduled workflow to determine what # needs to be updated (if anything) and then performing that task. # # Signed-off-by: Johannes Schindelin # The commit message #2 will be skipped: # fixup! ci: update the books via a GitHub workflow # The commit message #3 will be skipped: # fixup! ci: update the books via a GitHub workflow # This is the commit message #4: # amend! ci: update the books via a GitHub workflow ci: update the books via a GitHub workflow With this commit, the SHAs corresponding to the various repositories containing the ProGit Book and its translations are stored in a sparse-checkout'able directory. This information is then used by a scheduled workflow to determine what needs to be updated (if anything) and then performing that task. When GitHub workflows push new changes, they cannot trigger other GitHub workflows (to avoid infinite loops). Therefore, this new GitHub workflow not only synchronizes the books, but also builds the site and deploys it. Note: The code to build the site and to deploy it is provided in a custom Action, to make it reusable. It will come in handy over the next commits, where other GitHub workflows are added that likewise need to synchronize changes that desire a site rebuild & deployment. Signed-off-by: Johannes Schindelin # The commit message #5 will be skipped: # fixup! fixup! ci: update the books via a GitHub workflow # The commit message #6 will be skipped: # fixup! fixup! ci: update the books via a GitHub workflow # The commit message #7 will be skipped: # fixup! ci: update the books via a GitHub workflow # The commit message #8 will be skipped: # fixup! ci: update the books via a GitHub workflow # The commit message #9 will be skipped: # fixup! ci: update the books via a GitHub workflow --- .../actions/deploy-to-github-pages/action.yml | 62 ++++++++ .github/workflows/update-book.yml | 136 ++++++++++++++++++ _sync_state/.gitignore | 0 script/ci-helper.js | 65 +++++++++ 4 files changed, 263 insertions(+) create mode 100644 .github/actions/deploy-to-github-pages/action.yml create mode 100644 .github/workflows/update-book.yml create mode 100644 _sync_state/.gitignore create mode 100644 script/ci-helper.js diff --git a/.github/actions/deploy-to-github-pages/action.yml b/.github/actions/deploy-to-github-pages/action.yml new file mode 100644 index 0000000000..5b2db98139 --- /dev/null +++ b/.github/actions/deploy-to-github-pages/action.yml @@ -0,0 +1,62 @@ +name: 'Run Hugo/Pagefind and deploy to GitHub Pages' +description: 'Runs Hugo and Pagefind and then deploys the result to GitHub Pages.' +# This composite Action requires the following things in the calling workflow: +# +# permissions: +# contents: write # to push changes (if any) +# pages: write # to deploy to GitHub Pages +# id-token: write # to verify that the deployment source is legit +# environment: +# name: github-pages +# url: ${{ steps..outputs.url }} +outputs: + url: + description: The URL to which the site was deployed + value: ${{ steps.deploy.outputs.page_url }} +runs: + using: "composite" + steps: + - name: push changes + shell: bash + run: git push origin HEAD + + - name: un-sparse worktree to prepare for deployment + shell: bash + run: git sparse-checkout disable + + - name: setup GitHub Pages + id: pages + uses: actions/configure-pages@v4 + + - name: configure Hugo and Pagefind version + shell: bash + run: | + set -x && + echo "HUGO_VERSION=$(sed -n 's/^ *hugo_version: *//p' >$GITHUB_ENV + echo "PAGEFIND_VERSION=$(sed -n 's/^ *pagefind_version: *//p' >$GITHUB_ENV + + - name: install Hugo ${{ env.HUGO_VERSION }} + shell: bash + run: | + set -x && + curl -Lo /tmp/hugo.deb https://github.com/gohugoio/hugo/releases/download/v$HUGO_VERSION/hugo_extended_${HUGO_VERSION}_linux-amd64.deb && + sudo dpkg -i /tmp/hugo.deb + + - name: run Hugo to build the pages + env: + HUGO_RELATIVEURLS: false + shell: bash + run: hugo config && hugo --minify --baseURL "${{ steps.pages.outputs.base_url }}/" + + - name: run Pagefind ${{ env.PAGEFIND_VERSION }} to build the search index + shell: bash + run: npx -y pagefind${{ env.PAGEFIND_VERSION }} --site public + + - name: upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v2 + with: + path: ./public + + - name: deploy + id: deploy + uses: actions/deploy-pages@v3 diff --git a/.github/workflows/update-book.yml b/.github/workflows/update-book.yml new file mode 100644 index 0000000000..ca67876242 --- /dev/null +++ b/.github/workflows/update-book.yml @@ -0,0 +1,136 @@ +name: Update Progit Book + +on: + workflow_dispatch: + schedule: + # check daily for updates + - cron: '29 4 * * *' + +jobs: + check-for-updates: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + _sync_state + script + - uses: actions/github-script@v7 + id: get-pending + with: + script: | + const { getPendingBookUpdates } = require('./script/ci-helper.js') + + const pending = await getPendingBookUpdates(github) + // an empty matrix is invalid and makes the workflow run fail, unfortunately + return pending.length ? pending : [''] + - name: ruby setup + # Technically, we do not need Ruby in this job. But we do want to cache + # Ruby & The Gems for use in the matrix in the next job. + if: steps.get-pending.outputs.result != '[""]' + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + outputs: + matrix: ${{ steps.get-pending.outputs.result }} + update-book: + needs: check-for-updates + if: needs.check-for-updates.outputs.matrix != '[""]' + runs-on: ubuntu-latest + strategy: + matrix: + language: ${{ fromJson(needs.check-for-updates.outputs.matrix) }} + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + _sync_state + script + data + content/book/${{ matrix.language.lang }} + static/book/${{ matrix.language.lang }} + - name: clone ${{ matrix.language.repository }} + run: | + printf '%s\n' /progit-clone/ /vendor >>.git/info/exclude && + + # Clone the book's sources + git clone --depth 1 --single-branch \ + https://github.com/${{ matrix.language.repository }} progit-clone + - name: ruby setup + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - name: update book/${{ matrix.language.lang }} + run: | + # this seems to be needed to let `bundle exec` see `vendor/bundle/` + { bundle check || bundle install --frozen; } && + + # generate the HTML + bundle exec ruby ./script/update-book2.rb ${{ matrix.language.lang }} progit-clone + - name: commit changes + run: | + # record the commit hash + mkdir -p _sync_state && + git -C progit-clone rev-parse HEAD >_sync_state/book-${{ matrix.language.lang }}.sha && + + # commit it all + git add -A \ + _sync_state \ + data/book-${{ matrix.language.lang }}.yml \ + content/book && + # there might be images + if test -d static/book + then + git add -A static/book + fi && + git -c user.name=${{ github.actor }} \ + -c user.email=${{ github.actor }}@noreply.github.com \ + commit -m 'book: update ${{ matrix.language.lang }}' \ + -m 'Updated via the `update-book.yml` GitHub workflow.' + - name: verify that there are no uncommitted changes + run: | + git update-index --refresh && + if test -n "$(git diff HEAD)$(git ls-files --exclude-standard --other)" + then + echo '::error::there are uncommitted changes!' >&2 + git status >&2 + exit 1 + fi + - name: generate the bundle + run: | + git branch -m book-${{ matrix.language.lang }} + git bundle create ${{ matrix.language.lang }}.bundle refs/remotes/origin/${{ github.ref_name }}..book-${{ matrix.language.lang }} + - uses: actions/upload-artifact@v3 + with: + name: bundle-${{ matrix.language.lang }} + path: ${{ matrix.language.lang }}.bundle + push-updates: + needs: [check-for-updates, update-book] + if: needs.check-for-updates.outputs.matrix != '[""]' + permissions: + contents: write # to push changes (if any) + pages: write # to deploy to GitHub Pages + id-token: write # to verify that the deployment source is legit + environment: + name: github-pages + url: ${{ steps.deploy.outputs.url }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + - name: apply updates + id: apply + run: | + for lang in $(echo '${{ needs.check-for-updates.outputs.matrix }}' | + sed -n 's/\[\?{[^}]*"lang":"\([^"]*\)[^}]*},\?\]\?/\1 /gp') + do + git -c core.editor=: \ + -c user.name=${{ github.actor }} \ + -c user.email=${{ github.actor }}@noreply.github.com \ + pull --no-rebase bundle-$lang/$lang.bundle book-$lang || + exit 1 + done + - name: deploy to GitHub Pages + id: deploy + uses: ./.github/actions/deploy-to-github-pages diff --git a/_sync_state/.gitignore b/_sync_state/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/script/ci-helper.js b/script/ci-helper.js new file mode 100644 index 0000000000..e955494dbd --- /dev/null +++ b/script/ci-helper.js @@ -0,0 +1,65 @@ +const fs = require('fs') + +const getFileContents = async (path) => { + return (await fs.promises.readFile(path)).toString('utf-8').trim() +} + +const getAllBooks = async () => { + const book_rb = await getFileContents("script/book.rb"); + const begin = book_rb.indexOf('@@all_books = {') + const end = book_rb.indexOf('}', begin + 1) + if (begin < 0 || end < 0) throw new Error(`Could not find @@all_books in:\n${book_rb}`) + return book_rb + .substring(begin, end) + .split('\n') + .reduce((allBooks, line) => { + const match = line.match(/"([^"]+)" => "([^"]+)"/) + if (match) allBooks[match[1]] = match[2] + return allBooks + }, {}) +} + +const getPendingBookUpdates = async (octokit) => { + const books = await getAllBooks() + const result = [] + for (const lang of Object.keys(books)) { + try { + const localSha = await getFileContents(`_sync_state/book-${lang}.sha`) + + const [owner, repo] = books[lang].split('/') + const { data: { default_branch: remoteDefaultBranch } } = + await octokit.rest.repos.get({ + owner, + repo + }) + const { data: { object: { sha: remoteSha } } } = + await octokit.rest.git.getRef({ + owner, + repo, + ref: `heads/${remoteDefaultBranch}` + }) + + if (localSha === remoteSha) continue + } catch (e) { + // It's okay for the `.sha` file not to exist yet.` + if (e.code !== 'ENOENT') throw e + } + result.push({ + lang, + repository: books[lang] + }) + } + return result +} + +// for testing locally, needs `npm install @octokit/rest` to work +if (require.main === module) { + (async () => { + const { Octokit } = require('@octokit/rest') + console.log(await getPendingBookUpdates(new Octokit())) + })().catch(console.log) +} + +module.exports = { + getPendingBookUpdates +} \ No newline at end of file