Decompose the job scheduler (#520)

* make job scheduler more modular * up * Move to traits * up * add redis feature * add basic scheduling test * use minio run by github actions * update cargo-dist * update openidconnect * changes * remove slow test * up * im cryign * new snapshot
kitsune-soc · Apr 29, 2024 · f42d425 · f42d425
1 parent a247eb0
commit f42d425
Show file tree

Hide file tree

Showing 77 changed files with 2,602 additions and 1,718 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -12,6 +12,13 @@ jobs:
     name: Test coverage
     runs-on: ubuntu-latest
     services:
+      minio:
+        image: bitnami/minio
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        ports:
+          - 9000:9000
       postgres:
         image: postgres
         ports:
@@ -46,6 +53,7 @@ jobs:
         run: cargo llvm-cov nextest --all-features --workspace --lcov --output-path lcov.info
         env:
           DATABASE_URL: "postgres://postgres:postgres@localhost/test_db"
+          MINIO_URL: "http://127.0.0.1:9000"
           REDIS_URL: "redis://localhost"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,14 +1,14 @@
-# Copyright 2022-2023, axodotdev
+# Copyright 2022-2024, axodotdev
 # SPDX-License-Identifier: MIT or Apache-2.0
 #
 # CI that:
 #
 # * checks for a Git Tag that looks like a release
 # * builds artifacts with cargo-dist (archives, installers, hashes)
 # * uploads those artifacts to temporary workflow zip
-# * on success, uploads the artifacts to a Github Release
+# * on success, uploads the artifacts to a GitHub Release
 #
-# Note that the Github Release will be created with a generated
+# Note that the GitHub Release will be created with a generated
 # title/body based on your changelogs.
 
 name: Release
@@ -31,7 +31,7 @@ permissions:
 # packages versioned/released in lockstep).
 #
 # If you push multiple tags at once, separate instances of this workflow will
-# spin up, creating an independent announcement for each one. However Github
+# spin up, creating an independent announcement for each one. However, GitHub
 # will hard limit this to 3 tags per commit, as it will assume more tags is a
 # mistake.
 #
@@ -62,7 +62,7 @@ jobs:
         # we specify bash to get pipefail; it guards against the `curl` command
         # failing. otherwise `sh` won't catch that `curl` returned non-0
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.2/cargo-dist-installer.sh | sh"
       # sure would be cool if github gave us proper conditionals...
       # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
       # functionality based on whether this is a pull_request, and whether it's from a fork.
@@ -109,6 +109,8 @@ jobs:
         with:
           submodules: recursive
       - uses: swatinem/rust-cache@v2
+        with:
+          key: ${{ join(matrix.targets, '-') }}
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
       # Get the dist-manifest
@@ -135,7 +137,7 @@ jobs:
         run: |
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
@@ -162,7 +164,7 @@ jobs:
           submodules: recursive
       - name: Install cargo-dist
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.2/cargo-dist-installer.sh | sh"
       # Get all the local artifacts for the global tasks to use (for e.g. checksums)
       - name: Fetch local artifacts
         uses: actions/download-artifact@v4
@@ -178,7 +180,7 @@ jobs:
 
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
@@ -207,15 +209,15 @@ jobs:
         with:
           submodules: recursive
       - name: Install cargo-dist
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.2/cargo-dist-installer.sh | sh"
       # Fetch artifacts from scratch-storage
       - name: Fetch artifacts
         uses: actions/download-artifact@v4
         with:
           pattern: artifacts-*
           path: target/distrib/
           merge-multiple: true
-      # This is a harmless no-op for Github Releases, hosting for that happens in "announce"
+      # This is a harmless no-op for GitHub Releases, hosting for that happens in "announce"
       - id: host
         shell: bash
         run: |
@@ -230,7 +232,7 @@ jobs:
           name: artifacts-dist-manifest
           path: dist-manifest.json
 
-  # Create a Github Release while uploading all files to it
+  # Create a GitHub Release while uploading all files to it
   announce:
     needs:
       - plan
@@ -246,7 +248,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: "Download Github Artifacts"
+      - name: "Download GitHub Artifacts"
         uses: actions/download-artifact@v4
         with:
           pattern: artifacts-*
@@ -256,7 +258,7 @@ jobs:
         run: |
           # Remove the granular manifests
           rm -f artifacts/*-dist-manifest.json
-      - name: Create Github Release
+      - name: Create GitHub Release
         uses: ncipollo/release-action@v1
         with:
           tag: ${{ needs.plan.outputs.tag }}

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -39,6 +39,13 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     services:
+      minio:
+        image: bitnami/minio
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        ports:
+          - 9000:9000
       postgres:
         image: postgres
         ports:
@@ -71,4 +78,5 @@ jobs:
       - run: nix develop --impure --command bash -c "unset LD_LIBRARY_PATH && cargo nextest run --all-features"
         env:
           DATABASE_URL: "postgres://postgres:postgres@localhost/test_db"
+          MINIO_URL: "http://127.0.0.1:9000"
           REDIS_URL: "redis://localhost"