From 42d8fd1e4fd01a864a5be7c343ffa9b7f5f481a7 Mon Sep 17 00:00:00 2001 From: Raynor Chavez Date: Mon, 1 Jul 2024 14:30:46 +0800 Subject: [PATCH] add new fieldtypes for dict score modifiers (#241) add new fieldtypes for map score modifiers remove __marqo_version__ --- .github/workflows/open-source-unit-tests.yml | 4 ++-- setup.py | 2 +- src/marqo/models/marqo_index.py | 4 ++++ src/marqo/version.py | 11 +++------- tests/v2_tests/test_create_index.py | 22 ++++++++++++++++++++ tests/v2_tests/test_score_modifier_search.py | 14 ++++++------- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/.github/workflows/open-source-unit-tests.yml b/.github/workflows/open-source-unit-tests.yml index 9d1aa095..c867c4f1 100644 --- a/.github/workflows/open-source-unit-tests.yml +++ b/.github/workflows/open-source-unit-tests.yml @@ -2,7 +2,7 @@ # This workflow pulls a Marqo image and runs it. Py-marqo then connects to the # running container for the tests. # Unless otherwise specified, the Marqo version that is used for this test will be -# that specified by py-marqo's `marqo.version.__marqo_version__` +# that specified by py-marqo's `marqo.version.__minimum_supported_marqo_version__` name: Open source unit tests @@ -80,7 +80,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt export PYTHONPATH=$(pwd):$(pwd)/src:$PYTHONPATH - SUPPORTED_MQ_VERSION=$(python -c 'from marqo import version; print(version.__marqo_version__)') || exit 1 + SUPPORTED_MQ_VERSION=$(python -c 'from marqo import version; print(version.__minimum_supported_marqo_version__)') || exit 1 # error out if version is empty: if [ -z "$SUPPORTED_MQ_VERSION" ]; then exit 1; fi diff --git a/setup.py b/setup.py index 21e17450..bc213016 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ "tox" ], name="marqo", - version="3.5.1", + version="3.6.0", author="marqo org", author_email="org@marqo.io", description="Tensor search for humans", diff --git a/src/marqo/models/marqo_index.py b/src/marqo/models/marqo_index.py index 586d9033..f07c3ba7 100644 --- a/src/marqo/models/marqo_index.py +++ b/src/marqo/models/marqo_index.py @@ -25,6 +25,10 @@ class FieldType(str, Enum): ImagePointer = 'image_pointer' MultimodalCombination = 'multimodal_combination' CustomVector = "custom_vector" + MapInt = 'map' + MapLong = 'map' + MapFloat = 'map' + MapDouble = 'map' class VectorNumericType(str, Enum): diff --git a/src/marqo/version.py b/src/marqo/version.py index 14070a28..34541703 100644 --- a/src/marqo/version.py +++ b/src/marqo/version.py @@ -1,13 +1,8 @@ -__marqo_version__ = "2.8.0" -__marqo_release_page__ = f"https://github.com/marqo-ai/marqo/releases/tag/{__marqo_version__}" - -__minimum_supported_marqo_version__ = "2.6.0" - +__minimum_supported_marqo_version__ = "2.9.0" +# NOTE: This isn't used anywhere def supported_marqo_version() -> str: - return f"This Marqo Python client is built for Marqo release ({__marqo_version__}) \n" \ - f"{__marqo_release_page__} \n" \ - f"The minimum supported Marqo version for this client is ({__minimum_supported_marqo_version__}) \n" + return f"The minimum supported Marqo version for this client is ({__minimum_supported_marqo_version__}) \n" def minimum_supported_marqo_version() -> str: diff --git a/tests/v2_tests/test_create_index.py b/tests/v2_tests/test_create_index.py index 0cfb0add..eec8327c 100644 --- a/tests/v2_tests/test_create_index.py +++ b/tests/v2_tests/test_create_index.py @@ -3,6 +3,7 @@ from pytest import mark import numpy as np +from marqo.models.marqo_index import FieldType from marqo.errors import MarqoWebError from tests.marqo_test import MarqoTestCase @@ -244,6 +245,27 @@ def test_create_structured_index_with_custom_model(self): "dimensions": 384, "tokens": 512, "type": "sbert"}, index_settings['modelProperties']) + + def test_create_structured_index_with_map_fields(self): + self.client.create_index( + index_name=self.index_name, + type="structured", + model="hf/all_datasets_v4_MiniLM-L6", + all_fields=[{"name": "test", "type": "text", "features": ["lexical_search"]}, + {"name": "map_score_mod_float_1", "type": FieldType.MapFloat, "features": ["score_modifier"]}, + {"name": "map_score_mod_double_1", "type": FieldType.MapDouble, "features": ["score_modifier"]}, + {"name": "map_score_mod_int_1", "type": FieldType.MapInt, "features": ["score_modifier"]}, + {"name": "map_score_mod_long_1", "type": FieldType.MapLong, "features": ["score_modifier"]},], + tensor_fields=["test"] + ) + documents = [{"test": "test"}] + self.client.index(self.index_name).add_documents(documents) + + lexical_search_res = self.client.index(self.index_name).search(q="test", search_method="LEXICAL") + tensor_search_res = self.client.index(self.index_name).search(q="test", search_method="TENSOR") + + self.assertEqual(1, len(lexical_search_res['hits'])) + self.assertEqual(1, len(tensor_search_res['hits'])) def test_create_structured_image_index_with_preprocessing(self): self.client.create_index(index_name=self.index_name, diff --git a/tests/v2_tests/test_score_modifier_search.py b/tests/v2_tests/test_score_modifier_search.py index 2f9fb13e..4aa7041a 100644 --- a/tests/v2_tests/test_score_modifier_search.py +++ b/tests/v2_tests/test_score_modifier_search.py @@ -21,9 +21,9 @@ def setUp(self) -> None: "my_image_field": "https://marqo-assets.s3.amazonaws.com/tests/images/image2.jpg", # 4 fields "multiply_1": 1, - "multiply_2": 20.0, + "multiply_2": {"a": 20.0}, "add_1": 1.0, - "add_2": 30.0, + "add_2": {"a": 30.0}, "_id": "1" }, {"my_text_field": "A rider is riding a horse jumping over the barrier.", @@ -49,11 +49,11 @@ def test_score_modifier_search_results(self): "multiply_score_by": [{"field_name": "multiply_1", "weight": 1,}, - {"field_name": "multiply_2",}], + {"field_name": "multiply_2.a",}], "add_to_score": [ {"field_name": "add_1", "weight" : -3, }, - {"field_name": "add_2", "weight": 1, + {"field_name": "add_2.a", "weight": 1, }] } @@ -77,11 +77,11 @@ def test_invalid_score_modifiers_format(self): "multiply_score_bys": [{"field_name": "multiply_1", "weight": 1,}, - {"field_name": "multiply_2",}], + {"field_name": "multiply_2.a",}], "add_to_score": [ {"field_name": "add_1", "weight" : 4, }, - {"field_name": "add_2", "weight": 1, + {"field_name": "add_2.a", "weight": 1, }] } @@ -97,7 +97,7 @@ def test_valid_score_modifiers_format(self): "add_to_score": [ {"field_name": "add_1", "weight" : -3, }, - {"field_name": "add_2", "weight": 1, + {"field_name": "add_2.a", "weight": 1, }] } self.search_with_score_modifier(score_modifiers=valid_score_modifiers) \ No newline at end of file