ci: centralized job for rocksdb libaries cache (backport #21657) (#21665

) Co-authored-by: auricom <27022259+auricom@users.noreply.github.com> Co-authored-by: Julien Robert <julien@rbrt.fr>
cosmos · Sep 11, 2024 · e7724c6 · e7724c6
1 parent 81ae569
commit e7724c6
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 42 deletions.
diff --git a/.github/scripts/install-rocksdb-deps.sh b/.github/scripts/install-rocksdb-deps.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+sudo apt update && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev build-essential
diff --git a/.github/scripts/install-rocksdb.sh b/.github/scripts/install-rocksdb.sh
@@ -6,9 +6,6 @@ if [ -z "$ROCKSDB_VERSION" ]; then
   exit 1
 fi
 
-# Update and install dependencies
-sudo apt update && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev build-essential
-
 # Clone RocksDB repository
 git clone https://github.com/facebook/rocksdb.git /home/runner/rocksdb
 cd /home/runner/rocksdb || exit 1

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -38,18 +38,12 @@ jobs:
             /usr/local/lib/librocksdb.*
             /usr/local/include/rocksdb
           key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-${{ matrix.go-arch }}
+      - name: Install rocksdb deps
+        if: matrix.go-arch == 'amd64'
+        run: ./.github/scripts/install-rocksdb-deps.sh
       - name: Install rocksdb
         if: matrix.go-arch == 'amd64' && steps.cache-rocksdb.outputs.cache-hit != 'true'
-        id: install_rocksdb
         run: ./.github/scripts/install-rocksdb.sh
-      - name: Saves rocksdb libraries cache
-        if: matrix.go-arch == 'amd64' && steps.install_rocksdb.outcome == 'success'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            /usr/local/lib/librocksdb.*
-            /usr/local/include/rocksdb
-          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-${{ matrix.go-arch }}
         ###################
         #### Build App ####
         ###################

diff --git a/.github/workflows/cache-rocksdb.yml b/.github/workflows/cache-rocksdb.yml
@@ -0,0 +1,62 @@
+name: Cache rocksdb libraries
+on:
+  push:
+    paths:
+      - build.mk
+  schedule:
+    - cron: "*/15 * * * *" # Every 15 minutes
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+
+  check-cache-rocksdb:
+    name: Check existing cache
+    runs-on: ubuntu-latest
+    outputs:
+      cache-hit: ${{ steps.cache-rocksdb.outputs.cache-hit }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+
+
+  save-cache-rocksdb:
+    name: Build rocksdb libraries and save cache
+    runs-on: ubuntu-latest
+    needs: check-cache-rocksdb
+    if: needs.check-cache-rocksdb.outputs.cache-hit != 'true'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        run: ./.github/scripts/install-rocksdb.sh
+
+      - name: Saves rocksdb libraries cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -39,18 +39,11 @@ jobs:
             /usr/local/lib/librocksdb.*
             /usr/local/include/rocksdb
           key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
       - name: Install rocksdb
-        if: env.GIT_DIFF && steps.cache-rocksdb.outputs.cache-hit != 'true'
-        id: install_rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
         run: ./.github/scripts/install-rocksdb.sh
-      - name: Saves rocksdb libraries cache
-        if: steps.install_rocksdb.outcome == 'success'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            /usr/local/lib/librocksdb.*
-            /usr/local/include/rocksdb
-          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
       - name: run linting (long)
         if: env.GIT_DIFF
         id: lint_long

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -799,18 +799,11 @@ jobs:
             /usr/local/lib/librocksdb.*
             /usr/local/include/rocksdb
           key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
       - name: Install rocksdb
-        if: env.GIT_DIFF && steps.cache-rocksdb.outputs.cache-hit != 'true'
-        id: install_rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
         run: ./.github/scripts/install-rocksdb.sh
-      - name: Saves rocksdb libraries cache
-        if: steps.install_rocksdb.outcome == 'success'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            /usr/local/lib/librocksdb.*
-            /usr/local/include/rocksdb
-          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
       - name: tests
         if: env.GIT_DIFF
         run: |
@@ -856,18 +849,11 @@ jobs:
             /usr/local/lib/librocksdb.*
             /usr/local/include/rocksdb
           key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
       - name: Install rocksdb
-        if: env.GIT_DIFF && steps.cache-rocksdb.outputs.cache-hit != 'true'
-        id: install_rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
         run: ./.github/scripts/install-rocksdb.sh
-      - name: Saves rocksdb libraries cache
-        if: steps.install_rocksdb.outcome == 'success'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            /usr/local/lib/librocksdb.*
-            /usr/local/include/rocksdb
-          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
       - name: test & coverage report creation
         if: env.GIT_DIFF
         run: |