PyPi-push CI changes: unit tests, include releases/* branches

.github/workflows/open-source-unit-tests.yml

@@ -20,12 +20,10 @@ on:
       image_tag:
         description: 'Marqo image tag. Examples: "1.1.0", "test" "latest"'
         required: true
-  push:
-    branches: 
-      - mainline
   pull_request:
     branches: 
       - mainline
+      - 'releases/*'
 
 permissions:
   contents: read

.github/workflows/publish-to-PyPI.yml

@@ -1,18 +1,127 @@
+# GitHub Actions Workflow: Publish to PyPI
+# Description:
+# This workflow automates the process of testing and publishing the `py-marqo` package to PyPI.
+# The process includes:
+# 1. Starting a self-hosted EC2 runner.
+# 2. Running the `py-marqo` test suite on this runner.
+# 3. Stopping the EC2 runner.
+# 4. Publishing the package to PyPI if all previous steps were successful.
+
 name: Publish to PyPI
 
 on:
   push:
     branches:
       - mainline
+      - 'releases/*'
 
 permissions:
   contents: read
 
 jobs:
-  deploy:
+  Start-Runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_EC2_GH_RUNNER_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_EC2_GH_RUNNER_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id:  ${{ secrets.AMD_EC2_IMAGE_ID_200GB }}
+          ec2-instance-type: t3.xlarge
+          subnet-id: ${{ secrets.AMD_SUBNET_ID }}
+          security-group-id: ${{ secrets.AMD_SECURITY_GROUP_ID }}
+
+  Test-Py-Marqo:
+    name: Run Py-Marqo Test Suite
+    needs: Start-Runner
+    runs-on: ${{ needs.start-runner.outputs.label }}
+
+    environment: py-marqo-test-suite
+
+    steps:
+      - name: Checkout py-marqo repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+          cache: "pip"
+
+      - name: "Determine py-marqo's supported Marqo version"
+        id: get_default_marqo_version
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          export PYTHONPATH=$(pwd):$(pwd)/src:$PYTHONPATH
+          SUPPORTED_MQ_VERSION=$(python -c 'from marqo import version; print(version.__marqo_version__)') || exit 1
+
+          # error out if version is empty:
+          if [ -z "$SUPPORTED_MQ_VERSION" ]; then exit 1; fi
+          echo "::set-output name=version::$SUPPORTED_MQ_VERSION"
+
+      - name: Set registry and image repo
+        id: prepare
+        run: |
+          echo "::set-output name=registry::marqoai"
+          echo "::set-output name=image_repo::marqo"
+          echo "::set-output name=image_tag::${{ steps.get_default_marqo_version.outputs.version }}"
+
+      - name: Run Py-Marqo Tests
+        run: |
+          docker pull ${{ steps.prepare.outputs.registry }}/${{ steps.prepare.outputs.image_repo }}:${{ steps.prepare.outputs.image_tag }}
+          docker run --name marqo -d --privileged -p 8882:8882 --add-host host.docker.internal:host-gateway \
+            ${{ steps.prepare.outputs.registry }}/${{ steps.prepare.outputs.image_repo }}:${{ steps.prepare.outputs.image_tag }}
+
+          # wait for marqo to start with timeout of 10 minutes
+          timeout 10m bash -c 'until [[ $(curl -v --silent --insecure http://localhost:8882 2>&1 | grep Marqo) ]]; do sleep 0.1; done;' || (echo "Marqo did not start in time" && exit 1)
+
+          python -m pip install --upgrade pip
+          pip install tox
+          tox
+
+  Stop-Runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - Start-Runner
+      - Test-Py-Marqo
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_EC2_GH_RUNNER_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_EC2_GH_RUNNER_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
 
+  Publish-to-PyPi:
     runs-on: ubuntu-latest
     environment: PyPI
+    needs:
+      - Test-Py-Marqo
+      - Stop-Runner
 
     steps:
     - uses: actions/checkout@v3
@@ -35,4 +144,3 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
-        # verbose: true

tests/v0_tests/test_add_documents.py

@@ -17,7 +17,7 @@ class TestAddDocuments(MarqoTestCase):
     # Create index tests
     def test_create_index(self):
         test_index_name = self.create_test_index(index_name=self.generic_test_index_name)
-
+    
     def test_create_index_double(self):
         if not self.client.config.is_marqo_cloud:
             self.create_test_index(index_name=self.generic_test_index_name)