Implement workflows for dependency updates (#979)

These workflows are similar to the ones in the OPTIMADE Gateway (see
https://github.com/Materials-Consortia/optimade-gateway).

The following is done:
- Run Dependabot updates every Monday between 5 and 6 UTC.
- Merge into/reset permanent, dedicated dependabot branch whenever there
  is a new push to `master` (the default branch).
- Create a PR every Wednesday at 6:30 UTC, requesting to merge a copy of
  the dedicated dependabot branch into `master`.
  As a bonus, before opening this PR, an attempt is made at also
  updating the `pre-commit` hooks and running `pre-commit` to commit any
  changes.
- Enable "auto-merge" for all Dependabot-created PRs.

Remaining issues/manual labour:
- Update `setup.py` with dependencies according to the Wednesday PR.
- Fix `pre-commit` if issues arise when running it.
- Fix Dependabot dependency PRs if they do not pass the CI tests.

.github/dependabot.yml

@@ -3,7 +3,9 @@ updates:
 - package-ecosystem: pip
   directory: "/"
   schedule:
-    interval: monthly
+    interval: weekly
+    day: monday
+    time: "05:43"
   # Needs to be larger than the number of total requirements (currently 31)
   open-pull-requests-limit: 50
   target-branch: dependabot_updates
@@ -15,13 +17,15 @@ updates:
   directory: "/"
   schedule:
     interval: daily
+    time: "05:38"
   target-branch: master
   labels:
   - providers_updates
 - package-ecosystem: github-actions
   directory: "/"
   schedule:
-    interval: monthly
+    interval: daily
+    time: "05:33"
   target-branch: dependabot_updates
   labels:
   - CI

.github/utils/single_dependency_pr_body.txt

@@ -0,0 +1,5 @@
+### Update dependencies
+
+Automatically created PR from [`dependabot_updates`](https://github.com/Materials-Consortia/optimade-python-tools/tree/dependabot_updates).
+
+For more information see the ["Dependabot updates" workflow](https://github.com/Materials-Consortia/optimade-python-tools/blob/master/.github/workflows/ci_dependabot.yml).

.github/workflows/ci_automerge_dependabot.yml

@@ -0,0 +1,27 @@
+name: CI - Activate auto-merging for Dependabot PRs
+
+on:
+  pull_request_target:
+    branches:
+      - dependabot_updates
+
+jobs:
+
+  update-dependabot-branch:
+    name: Activate auto-merge into `dependabot_updates`
+    if: github.repository_owner == 'Materials-Consortia' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]'
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.pull_request.head.ref }}
+
+    - name: Activate auto-merge
+      run: |
+        PR_ID="$(gh api graphql -F owner='{owner}' -F name='{repo}' -f query='query($owner: String!, $name: String!) {repository(owner: $owner, name: $name) {pullRequest(number: ${{ github.event.pull_request.number }}) {id}}}' --jq '.data.repository.pullRequest.id')"
+        gh api graphql -f pr_id="$PR_ID" -f query='mutation($pr_id: String!) {enablePullRequestAutoMerge(input:{mergeMethod:SQUASH,pullRequestId:$pr_id }) {pullRequest {number}}}'
+      env:
+        GITHUB_TOKEN: ${{ secrets.RELEASE_PAT_CASPER }}

.github/workflows/ci_cd_updated_master.yml

@@ -0,0 +1,120 @@
+name: CI/CD - New updates to `master`
+
+on:
+  push:
+    branches:
+      - master
+
+env:
+  DEFAULT_REPO_BRANCH: master
+  GIT_USER_NAME: OPTIMADE Developers
+  GIT_USER_EMAIL: "dev@optimade.org"
+
+jobs:
+  deploy-docs:
+    name: Deploy `latest` documentation
+    if: github.repository_owner == 'Materials-Consortia'
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Release check
+      run: |
+        COMMIT_MSG="$(gh api /repos/${{ github.repository}}/commits/${{ env.DEFAULT_REPO_BRANCH }} --jq '.commit.message')"
+        if [[ "${COMMIT_MSG}" =~ ^Release\ v.*\ -\ Changelog$ ]]; then
+          echo "In a release - do not run this job !"
+          echo "RELEASE_RUN=true" >> $GITHUB_ENV
+        else
+          echo "Not a release - update docs"
+          echo "RELEASE_RUN=false" >> $GITHUB_ENV
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Checkout repository
+      if: env.RELEASE_RUN == 'false'
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        submodules: true
+
+    - name: Set up Python 3.8
+      if: env.RELEASE_RUN == 'false'
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      if: env.RELEASE_RUN == 'false'
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U setuptools wheel
+        pip install -U -e .[all]
+
+    - name: Set up git user
+      if: env.RELEASE_RUN == 'false'
+      run: |
+        git config --global user.name "${{ env.GIT_USER_NAME }}"
+        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
+
+    - name: Check API Reference and landing page
+      if: env.RELEASE_RUN == 'false'
+      run: |
+        invoke create-api-reference-docs --pre-clean
+
+        if [ -n "$(git status --porcelain docs/api_reference)" ]; then
+          echo "The following files in the documentation have not been committed:"
+          git status --porcelain docs/api_reference
+          exit 1
+        fi
+
+    - name: Deploy documentation
+      if: env.RELEASE_RUN == 'false'
+      run: mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml latest ${{ env.DEFAULT_REPO_BRANCH }}
+
+  update-dependabot-branch:
+    name: Update permanent dependabot branch `dependabot_updates`
+    if: github.repository_owner == 'Materials-Consortia'
+    runs-on: ubuntu-latest
+
+    env:
+      DEPENDABOT_BRANCH: dependabot_updates
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ env.DEPENDABOT_BRANCH }}
+        fetch-depth: 0
+
+    - name: Set up git config
+      run: |
+        git config --global user.name "${{ env.GIT_USER_NAME }}"
+        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
+
+    - name: Update '${{ env.DEPENDABOT_BRANCH }}'
+      run: |
+        git fetch origin
+
+        LATEST_PR_BODY="$(gh api /repos/${{ github.repository}}/pulls -X GET -f state=closed -f per_page=1 -f sort=updated -f direction=desc --jq '.[].body')"
+        if [ "${LATEST_PR_BODY}" == "$(cat .github/utils/single_dependency_pr_body.txt)" ]; then
+          # The dependency branch has just been merged into ${DEFAULT_REPO_BRANCH}
+          # The dependency branch should be reset to ${DEFAULT_REPO_BRANCH}
+          echo "The dependencies have just been updated! Reset to ${{ env.DEFAULT_REPO_BRANCH }}."
+          git reset --hard origin/${{ env.DEFAULT_REPO_BRANCH }}
+          echo "FORCE_PUSH=yes" >> $GITHUB_ENV
+        else
+          # Normal procedure: Merge ${DEFAULT_REPO_BRANCH} into ${DEPENDABOT_BRANCH}
+          echo "Merge new updates to ${{ env.DEFAULT_REPO_BRANCH }} into ${DEPENDABOT_BRANCH}"
+          git merge -m "Keep '${{ env.DEPENDABOT_BRANCH }}' up-to-date with '${{ env.DEFAULT_REPO_BRANCH }}'" origin/${{ env.DEFAULT_REPO_BRANCH }}
+          echo "FORCE_PUSH=no" >> $GITHUB_ENV
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Push to '${{ env.DEPENDABOT_BRANCH }}'
+      uses: CasperWA/push-protected@v2
+      with:
+        token: ${{ secrets.RELEASE_PAT_CASPER }}
+        branch: ${{ env.DEPENDABOT_BRANCH }}
+        sleep: 15
+        force: ${{ env.FORCE_PUSH }}

.github/workflows/ci_dependabot.yml

@@ -0,0 +1,96 @@
+name: CI - Single Dependabot PR
+
+on:
+  schedule:
+    # At 6:30 UTC every Wednesday
+    # Dependabot runs once a week (every Monday) (pip)
+    # and every day (GH Actions) at 5:00 UTC
+    - cron: "30 6 * * 3"
+
+jobs:
+
+  create-collected-pr:
+    name: Single dependabot PR
+    if: github.repository_owner == 'Materials-Consortia'
+    runs-on: ubuntu-latest
+    env:
+      DEPENDABOT_BRANCH: dependabot_updates
+      GIT_USER_NAME: OPTIMADE Developers
+      GIT_USER_EMAIL: "dev@optimade.org"
+      DEFAULT_REPO_BRANCH: master
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ env.DEFAULT_REPO_BRANCH }}
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install `pre-commit` and dependencies
+      run: |
+        python -m pip install -U pip
+        pip install -U setuptools wheel
+        pip install -e .
+        pip install -r requirements.txt
+        pip install -r requirements-dev.txt
+
+    - name: Set up git user info
+      run: |
+        git config --global user.name "${{ env.GIT_USER_NAME }}"
+        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
+
+    - name: Reset to '${{ env.DEPENDABOT_BRANCH }}'
+      run: |
+        git fetch origin ${{ env.DEPENDABOT_BRANCH }}:${{ env.DEPENDABOT_BRANCH }}
+        git reset --hard ${{ env.DEPENDABOT_BRANCH }}
+
+    - name: Auto-update `pre-commit` hooks
+      run: |
+        pre-commit autoupdate
+
+        if [ -n "$(git status --porcelain .pre-commit-config.yaml)" ]; then
+          # Set environment variable notifying next steps that the hooks changed
+          echo "Pre-commit hooks have been updated !"
+          echo "UPDATED_PRE_COMMIT_HOOKS=true" >> $GITHUB_ENV
+        else
+          echo "No pre-commit hooks have been updated."
+          echo "UPDATED_PRE_COMMIT_HOOKS=false" >> $GITHUB_ENV
+        fi
+
+    - name: Possibly run `pre-commit` with updated hooks
+      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
+      continue-on-error: true  # Still create the PR if this step fails
+      run: pre-commit run --all-files
+
+    - name: Possibly commit changes and updates
+      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
+      run: |
+        git add .
+        git commit -m "Update \`pre-commit\` hooks"
+
+    - name: Fetch PR body
+      id: pr_body
+      uses: chuhlomin/render-template@v1.2
+      with:
+        template: .github/utils/single_dependency_pr_body.txt
+
+    - name: Create PR
+      id: cpr
+      uses: peter-evans/create-pull-request@v3
+      with:
+        token: ${{ secrets.RELEASE_PAT_CASPER }}
+        commit-message: New @dependabot-fueled updates
+        committer: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
+        author: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
+        branch: ci/update-dependencies
+        delete-branch: true
+        title: Update dependencies
+        body: ${{ steps.pr_body.outputs.result }}
+        labels: CI,dependency_updates
+
+    - name: Information
+      run: 'echo "${{ steps.cpr.outputs.pull-request-operation }} PR #${{ steps.cpr.outputs.pull-request-number }}: ${{ steps.cpr.outputs.pull-request-url }}"'