Unit test GH workflow

.github/workflows/open-source-unit-tests.yml

@@ -0,0 +1,136 @@
+name: Py-marqo  open source unit tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      push_to:
+        description: 'Docker registry location. Options: "ECR" or "DockerHub"'
+        required: true
+        default: 'DockerHub'
+      image_repo:
+        description: 'Marqo docker image repo name'
+        required: true
+        default: 'marqo'
+      image_tag:
+        description: 'Marqo image tag. Examples: "1.1.0", "test" "latest"'
+        required: true
+  push:
+    branches: 
+      - mainline
+  pull_request:
+    branches: 
+      - mainline
+
+permissions:
+  contents: read
+
+jobs:
+  Start-Runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_EC2_GH_RUNNER_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_EC2_GH_RUNNER_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id:  ${{ secrets.AMD_EC2_IMAGE_ID_200GB }}
+          ec2-instance-type: t3.xlarge
+          subnet-id: ${{ secrets.AMD_SUBNET_ID }}
+          security-group-id: ${{ secrets.AMD_SECURITY_GROUP_ID }}
+
+  Test-Py-Marqo:
+    name: Run Py-Marqo Test Suite
+    needs: Start-Runner
+    runs-on: ${{ needs.start-runner.outputs.label }}
+
+    environment: py-marqo-test-suite
+
+    steps:
+      - name: Checkout py-marqo repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+          cache: "pip"
+
+      - name: "Determine py-marqo's supported Marqo version"
+        id: get_default_marqo_version
+        run: |
+          python -m pip install --upgrade pip
+          pip install marqo
+          echo "::set-output name=version::$(python -c 'from marqo import version; print(version.__marqo_version__)')"
+
+      - name: Log into ECR
+        uses: docker/login-action@v1
+        if: github.event.inputs.push_to == 'ECR'
+        with:
+          registry: ${{ secrets.AWS_ACCOUNT_NUMBER }}.dkr.ecr.us-east-1.amazonaws.com/${{ github.event.inputs.image_repo }}
+          username: ${{ secrets.ECR_READER_AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.ECR_READER_AWS_SECRET_ACCESS_KEY }}
+
+      - name: Set registry and image repo
+        id: prepare
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            if [[ "${{ github.event.inputs.push_to }}" == "ECR" ]]; then
+              echo "::set-output name=registry::${{ secrets.AWS_ACCOUNT_NUMBER }}.dkr.ecr.us-east-1.amazonaws.com"
+            else
+              echo "::set-output name=registry::marqoai"
+            fi
+            echo "::set-output name=image_repo::${{ github.event.inputs.image_repo }}"
+            echo "::set-output name=image_tag::${{ github.event.inputs.image_tag }}"
+          else
+            echo "::set-output name=registry::marqoai"
+            echo "::set-output name=image_repo::marqo"
+            echo "::set-output name=image_tag::${{ steps.get_default_marqo_version.outputs.version }}"
+          fi
+
+      - name: Run Py-Marqo Tests
+        run: |
+          docker pull ${{ steps.prepare.outputs.registry }}/${{ steps.prepare.outputs.image_repo }}:${{ steps.prepare.outputs.image_tag }}
+          docker run --name marqo -d --privileged -p 8882:8882 --add-host host.docker.internal:host-gateway \
+            ${{ steps.prepare.outputs.registry }}/${{ steps.prepare.outputs.image_repo }}:${{ steps.prepare.outputs.image_tag }}
+
+          # wait for marqo to start with timeout of 10 minutes
+          timeout 10m bash -c 'until [[ $(curl -v --silent --insecure http://localhost:8882 2>&1 | grep Marqo) ]]; do sleep 0.1; done;' || (echo "Marqo did not start in time" && exit 1)
+
+          python -m pip install --upgrade pip
+          pip install tox
+          tox
+
+  Stop-Runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - Start-Runner
+      - Test-Py-Marqo
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_EC2_GH_RUNNER_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_EC2_GH_RUNNER_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}