cuda_docker_marqo.yml

name: cuda_docker_marqo_API_tests
# runs API Tests on cuda-enabled instance with Vespa and Marqo running in the same container

on:
  workflow_call:
  workflow_dispatch:
    inputs:
      py_marqo_branch:
        required: false
        default: mainline
        description: >
          The "py-marqo" branch this test is running against.
          This is optional. If left as the default value "mainline", we run the test based on the "mainline" branch of
          py-marqo. Otherwise, the specified branch is tested. For example "li/test-xx". You can also use "marqo" and 
          it will use the latest pypi release.
      api_tests_branch:
        required: false
        default: mainline
        description: >
          The "api-tests" branch this test is running against.
          This is optional. If left as the default value "mainline", we run the api tests based at branch
          "mainline". Otherwise, the specified branch is tested. For example "li/test-xx"
      image_to_test:
        required: false
        # This is the name of the docker image that is built by the build script:
        default: marqo_docker_0
        description: >
          This is optional. If left as the default value "marqo_docker_0", the docker image built from this branch is tested.
          Otherwise, the specified docker image is tested. For example "marqoai/marqo:test"
  push:
    branches:
      - mainline
    paths-ignore:
      - '**.md'

permissions:
  contents: read

jobs:
  Start-Runner:
    name: Start self-hosted EC2 runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.LARGEMODELTEST_ACCESSKEY }}
          aws-secret-access-key: ${{ secrets.LARGEMODELTEST_SECRETACCESSKEY }}
          aws-region: ${{ secrets.LARGEMODELTEST_AWS_REGION }}
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

          # CUDA AMD64 instance
          ec2-image-id: ${{ vars.MARQO_CUDA_TESTS_INSTANCE_AMI }}
          ec2-instance-type: g4dn.2xlarge
          subnet-id: ${{ secrets.LARGEMODELTEST_SUBNET_ID }}
          security-group-id: ${{ secrets.LARGEMODELTEST_SECURITY_GROUP }}
  Test-Marqo:
    name: Run CUDA Docker Marqo API Tests
    needs: Start-Runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
                
    environment: marqo-test-suite 
    
    steps:
       
      - name: Checkout marqo repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: "3.8"
          cache: "pip"
          
      - name: Install Dependencies
        run: |
          #pip install -r requirements.txt
          pip install tox==3.26
          pip install flake8
      
      - name: Set MQ_PY_MARQO_BRANCH variable
        run: |
          if [[ "${{ inputs.py_marqo_branch }}" == "marqo" ]]; then
            echo "MQ_PY_MARQO_BRANCH=marqo" >> $GITHUB_ENV
          elif [[ "${{ inputs.py_marqo_branch }}" == "" ]]; then
            echo "MQ_PY_MARQO_BRANCH=git+https://github.com/marqo-ai/py-marqo.git@mainline" >> $GITHUB_ENV
          else
            echo "MQ_PY_MARQO_BRANCH=git+https://github.com/marqo-ai/py-marqo.git@${{ inputs.py_marqo_branch }}" >> $GITHUB_ENV
          fi
  
      - name: Checkout marqo-api-tests repo
        uses: actions/checkout@v3
        with:
          repository: marqo-ai/marqo-api-tests
          ref: ${{ github.event.inputs.api_tests_branch }}
          
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
          
      - name: Set up Environment
        run: |
          # Set up conf file
          echo 'export MARQO_API_TESTS_ROOT="${{ github.workspace }}"' >> conf
          
      - name: Run CUDA Integration Tests - CUDA Docker Marqo
        run: |
          export MQ_API_TEST_BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
          echo "$MQ_API_TEST_BRANCH"
          CUSTOM_TEST_IMG="${{ github.event.inputs.image_to_test }}"
          export MQ_API_TEST_IMG=${CUSTOM_TEST_IMG:-"marqo_docker_0"}
          tox -e py3-cuda_docker_marqo
  
  Stop-Runner:
    name: Stop self-hosted EC2 runner
    needs:
      - Start-Runner # required to get output from the start-runner job
      - Test-Marqo # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.LARGEMODELTEST_ACCESSKEY }}
          aws-secret-access-key: ${{ secrets.LARGEMODELTEST_SECRETACCESSKEY }}
          aws-region: ${{ secrets.LARGEMODELTEST_AWS_REGION }}
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}