maven.yml

# Template: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
#
# Useful links
# - GitHub Actions: https://docs.github.com/en/actions/learn-github-actions/introduction-to-github-actions
# - Service containers: https://docs.github.com/en/actions/guides/creating-postgresql-service-containers
#
# The CI jobs are set up as follows:
# - One job to build and upload artifacts.
# - One job per DBMS test suite.
# - One job to build/test the docker images.
# - One job to publish the docker image.

name: BenchBase (Java with Maven)

on:
  push:
    branches: [ main ]
    # Generate new docker images on release tags.
    tags:
      - 'v*'
  pull_request:
    branches: [ main ]
  # Run these workflows on a schedule so that docker images are regularly updated for security patches.
  schedule:
    - cron: "1 0 * * *"
  # Give us a button to allow running the workflow on demand for testing.
  workflow_dispatch:
    inputs:
      tags:
        description: 'Manual Workflow Run'
        required: false
        type: string

env:
  POM_VERSION: 2023-SNAPSHOT
  JAVA_VERSION: 21
  ERRORS_THRESHOLD: 0.01
  PYTHON_VERSION: "3.10"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  compile-and-test:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          cache: 'maven'
          distribution: 'temurin'

      - name: Check formatting
        run: mvn -B fmt:check --file pom.xml

      - name: Compile with Maven
        run: mvn -B compile test-compile --file pom.xml

      - name: Test with Maven
        run: mvn -B test --file pom.xml

  package-and-upload:
    needs: compile-and-test
    runs-on: ubuntu-latest
    strategy:
      matrix:
        profile: [ 'cockroachdb', 'mariadb', 'mysql', 'oracle', 'phoenix', 'postgres', 'spanner', 'sqlite', 'sqlserver' ]
    steps:
    - name: Checkout repo
      uses: actions/checkout@v4

    - name: Set up JDK
      uses: actions/setup-java@v4
      with:
        java-version: ${{env.JAVA_VERSION}}
        cache: 'maven'
        distribution: 'temurin'

    - name: Package and verify with Maven
      run: |
        mvn -B package verify -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml

    - name: Upload TGZ artifact
      uses: actions/upload-artifact@v4
      with:
        name: benchbase-${{matrix.profile}}
        path: target/benchbase-${{matrix.profile}}.tgz

    # Needed for running a customized service containers using docker/*/up.sh scripts.
    # See postgres example below.
    # https://github.com/actions/runner/issues/2139
    - name: Package docker-compose configs
      if: ${{ matrix.profile == 'postgres' }}
      run: |
        tar czvpf docker-compose-${{matrix.profile}}.tar.gz docker/${{matrix.profile}}-latest

    - name: Upload docker-compose configs
      if: ${{ matrix.profile == 'postgres' }}
      uses: actions/upload-artifact@v4
      with:
        name: docker-compose-${{matrix.profile}}
        path: docker-compose-${{matrix.profile}}.tar.gz

  ## ----------------------------------------------------------------------------------
  ## SQLITE
  ## ----------------------------------------------------------------------------------
  sqlite:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # BROKEN: tpch
        benchmark: [ 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-sqlite

      - name: Extract artifact
        run: |
          tar xvzf benchbase-sqlite.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-sqlite.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Run benchmark
        run: |
          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]] || [[ ${{matrix.benchmark}} == tpcc ]]; then
            # Disable synchronous mode for sqlite tpcc data loading to save some time.
            java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            # Run the templated benchmark.
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            echo "The ${{matrix.benchmark}} benchmark is not supported for sqlite."
            exit 0
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              # Disable synchronous mode for sqlite tpcc data loading to save some time.
              java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.02
          elif [ ${{matrix.benchmark}} == resourcestresser ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == smallbank ]; then
              ERRORS_THRESHOLD=0.03
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MARIADB
  ## ----------------------------------------------------------------------------------
  mariadb:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # FIXME: Add tpch back in (#333).
        benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    services:
      mariadb: # https://hub.docker.com/_/mariadb
        image: mariadb:latest
        env:
          MARIADB_ROOT_PASSWORD: rootyMcRooty
          MARIADB_DATABASE: benchbase
          MARIADB_USER: admin
          MARIADB_PASSWORD: password
        # Use status instead of ping due to exit code issues: https://jira.mariadb.org/browse/MDEV-31550
        options: >-
          --health-cmd "mariadb-admin status -uroot -prootyMcRooty"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 3306:3306
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-mariadb

      - name: Extract artifact
        run: |
          tar xvzf benchbase-mariadb.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-mariadb.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Run benchmark
        env:
          MARIADB_PORT: ${{ job.services.mariadb.ports[3306] }}
        run: |
          mysql -h127.0.0.1 -P$MARIADB_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"

          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh mariadb) &
            java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.02
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MYSQL
  ## ----------------------------------------------------------------------------------
  mysql:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        benchmark: [ 'anonymization', 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    services:
      mysql: # https://hub.docker.com/_/mysql
        image: mysql:latest
        env:
          MYSQL_ROOT_PASSWORD: rootyMcRooty
          MYSQL_DATABASE: benchbase
          MYSQL_USER: admin
          MYSQL_PASSWORD: password
        options: >-
          --health-cmd "mysqladmin ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 3306:3306
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-mysql

      - name: Extract artifact
        run: |
          tar xvzf benchbase-mysql.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-mysql.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Set up Python
        uses: actions/setup-python@v5  
        with:  
          python-version: ${{env.PYTHON_VERSION}}  

      - name: Install Python dependencies  
        working-directory: ./scripts/anonymization
        run: |
          if [[ ${{matrix.benchmark}} == anonymization ]]; then
            python -m pip install --upgrade pip
            pip install -r requirements.txt  
          else
            echo "Dependency installation not necessary for benchmark"
          fi

      - name: Run benchmark
        env:
          MYSQL_PORT: ${{ job.services.mysql.ports[3306] }}
        run: |
          mysql -h127.0.0.1 -P$MYSQL_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"

          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
          # For anonymization, we load tpcc and anonymize a single table. The workload itself is not executed
          # FIXME: 'exit 0' is called because there is no benchmark executed and analyzed. Must be removed once the Anonymization script is
          # fully implemented. See Pull Request 455.
          elif [[ ${{matrix.benchmark}} == anonymization ]]; then
            java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b tpcc -c config/mysql/sample_${{matrix.benchmark}}_config.xml --anonymize=true
            exit 0
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh mysql) &
            java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.02
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## ORACLE
  ## ----------------------------------------------------------------------------------
  oracle:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    services:
      oracle:
        image: gvenzl/oracle-xe:21.3.0-slim-faststart
        ports:
          - "1521:1521"
          - "5500:5500"
        env:
          ORACLE_PASSWORD: password
          ORACLE_CHARACTERSET: AL32UTF8
          APP_USER: benchbase
          APP_USER_PASSWORD: password
        options: >-
          --name oracle
          --health-cmd "echo exit | sqlplus benchbase/password@xepdb1 | grep Connected"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --health-start-period 5s
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-oracle

      - name: Extract artifact
        run: |
          tar xvzf benchbase-oracle.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-oracle.tgz

      - name: Set up user reset script
        run: |
          docker cp config/oracle/scripts/reset.sql oracle:/opt/oracle/reset.sql

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Run benchmark
        run: |
          docker exec oracle sqlplus "sys/password@xepdb1 as sysdba" @reset.sql
          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh oracle) &
            java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.04
          elif [ ${{matrix.benchmark}} == resourcestresser ]; then
              ERRORS_THRESHOLD=0.04
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc ]; then
              ERRORS_THRESHOLD=0.03
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          elif [ ${{matrix.benchmark}} == wikipedia ]; then
              ERRORS_THRESHOLD=0.02
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## POSTGRESQL
  ## ----------------------------------------------------------------------------------
  postgresql:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    steps:
      # Note: we download just the docker-compose scripts/configs rather than the
      # whole source code repo for better testing.
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: docker-compose-postgres

      - name: Extract docker-compose artifacts
        run: |
          tar xvzf docker-compose-postgres.tar.gz

      # Use docker-compose to start the postgres service so we can modify the
      # command line args to include extensions.
      # https://github.com/actions/runner/issues/2139
      - name: Start custom postgres service
        run: |
          ./docker/postgres-latest/up.sh --quiet-pull postgres

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-postgres

      - name: Extract artifact
        run: |
          tar xvzf benchbase-postgres.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-postgres.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Run benchmark
        run: |
          PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists
          PGPASSWORD=password createdb -h localhost -U admin benchbase

          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh postgres) &
            java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.02
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

          # Running the monitor should create at least three files in the 'monitor' directory.
          if ![ $(find "./results/monitor" -maxdepth 1 -mindepth 1 | wc -l) -gt 2]; then
            echo "ERROR: Advanced monitoring unsuccessful, file directory and/or appropriate files not created." >&2
            exit 1
          fi

      - name: Stop custom postgres service
        run: |
          ./docker/postgres-latest/down.sh

  ## ----------------------------------------------------------------------------------
  ## COCKROACHDB
  ## ----------------------------------------------------------------------------------
  cockroachdb:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # TODO: Add tpcc-with-reconnects benchmark support
        # TODO: Add chbenchmark benchmark support
        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    services:
      cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node
        image: timveil/cockroachdb-single-node:latest
        env:
          DATABASE_NAME: benchbase
          # TODO: Expand for additional config adjustments (See Also: #405, #519, #525)
          MEMORY_SIZE: .75
        ports:
          - 26257:26257
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-cockroachdb

      - name: Extract artifact
        run: |
          tar xvzf benchbase-cockroachdb.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-cockroachdb.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Run benchmark
        run: |
          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh cockroachdb) &
            java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == auctionmark ]; then
              ERRORS_THRESHOLD=0.02
          elif [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MSSQL
  ## ----------------------------------------------------------------------------------
  sqlserver:
    needs: package-and-upload
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # TODO: add auctionmark and seats benchmark
        benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
    services:
      sqlserver:
        image: mcr.microsoft.com/mssql/server:latest
        env:
          ACCEPT_EULA: Y
          SA_PASSWORD: SApassword1
        options: >-
          --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P SApassword1 -b -Q 'SELECT 1;'"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 1433:1433
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: benchbase-sqlserver

      - name: Extract artifact
        run: |
          tar xvzf benchbase-sqlserver.tgz --strip-components=1

      - name: Delete artifact
        run: |
          rm -rf benchbase-sqlserver.tgz

      - name: Set up JDK
        uses: actions/setup-java@v4
        with:
          java-version: ${{env.JAVA_VERSION}}
          distribution: 'temurin'

      - name: Cleanup database
        uses: docker://mcr.microsoft.com/mssql-tools:latest
        with:
          entrypoint: /opt/mssql-tools/bin/sqlcmd
          args: -U sa -P SApassword1 -S sqlserver -b -Q "DROP DATABASE IF EXISTS benchbase;"

      - name: Setup database
        uses: docker://mcr.microsoft.com/mssql-tools:latest
        with:
          entrypoint: /opt/mssql-tools/bin/sqlcmd
          args: -U sa -P SApassword1 -S sqlserver -b -Q "CREATE DATABASE benchbase;"

      - name: Setup login
        uses: docker://mcr.microsoft.com/mssql-tools:latest
        with:
          entrypoint: /opt/mssql-tools/bin/sqlcmd
          args: -U sa -P SApassword1 -S sqlserver -Q "CREATE LOGIN benchuser01 WITH PASSWORD='P@ssw0rd';"

      - name: Setup access
        uses: docker://mcr.microsoft.com/mssql-tools:latest
        with:
          entrypoint: /opt/mssql-tools/bin/sqlcmd
          args: -U sa -P SApassword1 -S sqlserver -b -Q "USE benchbase; CREATE USER benchuser01 FROM LOGIN benchuser01; EXEC sp_addrolemember 'db_owner', 'benchuser01';"

      - name: Run benchmark
        # Note: user/pass should match those used in sample configs.
        run: |
          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
          # In this case, we load the tpcc data.
          if [[ ${{matrix.benchmark}} == templated ]]; then
            java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced  --create=false --load=false --execute=true --json-histograms results/histograms.json
          elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then
            # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh
            java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true
            (sleep 10 && ./scripts/interrupt-docker-db-service.sh sqlserver) &
            java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json
          else
            if [ ${{matrix.benchmark}} == chbenchmark ]; then
              java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
            fi
            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json
          fi

          # FIXME: Reduce the error rate so we don't need these overrides.
          results_benchmark=${{matrix.benchmark}}
          if [ ${{matrix.benchmark}} == tatp ]; then
              ERRORS_THRESHOLD=0.05
          elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then
              ERRORS_THRESHOLD=0.02
              results_benchmark=tpcc
          fi
          ./scripts/check_latest_benchmark_results.sh $results_benchmark
          ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

          # Running the monitor should create at least three files in the 'monitor' directory.
          if ![ $(find "./results/monitor" -maxdepth 1 -mindepth 1 | wc -l) -gt 2]; then
            echo "ERROR: Advanced monitoring unsuccessful, file directory and/or appropriate files not created." >&2
            exit 1
          fi

  ## ----------------------------------------------------------------------------------
  ## Docker Build Test Publish
  ## ----------------------------------------------------------------------------------
  docker-build-test-publish:
    runs-on: ubuntu-latest
    env:
      DOCKER_BUILDKIT: 1
      BENCHBASE_PROFILES: 'cockroachdb mariadb mysql oracle phoenix postgres spanner sqlite sqlserver'
      CONTAINER_REGISTRY_NAME: ${{ secrets.ACR_LOGINURL }}
    services:
      postgres: # https://hub.docker.com/_/postgres
        image: postgres:latest
        env:
          POSTGRES_DB: benchbase
          POSTGRES_USER: admin
          POSTGRES_PASSWORD: password
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - name: Setup postgres test DB
        run: |
          PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists
          PGPASSWORD=password createdb -h localhost -U admin benchbase
      - name: Checkout repo
        uses: actions/checkout@v4
      # https://github.com/actions/cache/blob/master/examples.md#java---maven
      - name: Cache local Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: setup-java-${{ runner.os }}-docker-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            setup-java-${{ runner.os }}-docker-maven-
      - name: Pull base image caches for PR builds
        if: ${{ github.ref != 'refs/heads/main' }}
        run: |
          docker pull benchbase.azurecr.io/benchbase-dev:latest || true
          docker pull benchbase.azurecr.io/benchbase:latest || true
      - name: Set NO_CACHE env var for main branch builds
        if: ${{ github.ref == 'refs/heads/main' }}
        run: |
          echo "NO_CACHE=true" >> $GITHUB_ENV
      - name: Build benchbase-dev image
        run: |
          ./docker/benchbase/build-dev-image.sh
      # Note: this script maps the local .m2 cache into the container.
      - name: Build the benchbase docker image with all profiles using the dev image
        env:
          SKIP_TESTS: 'false'
          DO_FORMAT_CHECKS: 'true'
        run: |
          ./docker/benchbase/build-full-image.sh
      - name: Run a basic test from the docker image against postgres test DB
        env:
            benchmark: noop
        run: |
          for image in benchbase benchbase-postgres; do
            # Adjust the sample config to talk to the container service instead of localhost.
            cat "config/postgres/sample_${benchmark}_config.xml" | sed -e 's/localhost:5432/postgres:5432/g' > /tmp/config.xml
            # Lookup the service container's docker network so we can place the benchbase container in it too.
            docker_network="$(docker ps --filter expose=5432 --format '{{.Networks}}')"
            # Map the adjusted config into the container and use it to run the test.
            rm -rf results
            mkdir -p results
            docker run --rm --name benchbase-postgres --network "$docker_network" \
              --env BENCHBASE_PROFILE=postgres -v /tmp/config.xml:/tmp/config.xml -v "$PWD/results:/benchbase/results" \
              "$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
            # Test that the results files were produced.
            ls results/${benchmark}_*.csv
            ./scripts/check_latest_benchmark_results.sh ${benchmark}
            ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
          done
      # Publish the docker image if the build/test was successful.
      # Only do this with approved PRs and if the login secrets are available.
      # Typically we expect to publish to benchbase.azurecr.io,
      # but setting ACR_LOGINURL to something else allows us to do testing on forks.
      - name: Log in to Docker Hub
        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }}
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ACR_LOGINURL }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}
      - name: Push Docker image
        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }}
        run: |
          docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase-dev
          docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase
          for profile in $BENCHBASE_PROFILES; do
            docker push -a ${{ secrets.ACR_LOGINURL }}/benchbase-$profile
          done