Merge pull request #173 from Materials-Consortia/ml-evs/list_queries

Added default mongo implementations for HAS ALL/ANY/ONLY

optimade/filtertransformers/mongo.py

@@ -58,7 +58,15 @@ def not_implemented_string(self, value):
 
     def value_list(self, arg):
         # value_list: [ OPERATOR ] value ( "," [ OPERATOR ] value )*
-        raise NotImplementedError
+        # NOTE: no support for optional OPERATOR, yet, so this takes the
+        # parsed values and returns an error if that is being attempted
+        for value in arg:
+            if str(value) in self.operator_map.keys():
+                raise NotImplementedError(
+                    f"OPERATOR {value} inside value_list {arg} not implemented."
+                )
+
+        return arg
 
     def value_zip(self, arg):
         # value_zip: [ OPERATOR ] value ":" [ OPERATOR ] value (":" [ OPERATOR ] value)*
@@ -138,14 +146,18 @@ def set_op_rhs(self, arg):
             return {"$in": arg[1:]}
 
         if arg[1] == "ALL":
-            raise NotImplementedError
+            return {"$all": arg[2]}
+
         if arg[1] == "ANY":
-            raise NotImplementedError
+            return {"$in": arg[2]}
+
         if arg[1] == "ONLY":
-            raise NotImplementedError
+            return {"$all": arg[2], "$size": len(arg[2])}
 
         # value with OPERATOR
-        raise NotImplementedError
+        raise NotImplementedError(
+            f"set_op_rhs not implemented for use with OPERATOR. Given: {arg}"
+        )
 
     def length_op_rhs(self, arg):
         # length_op_rhs: LENGTH [ OPERATOR ] value

tests/filterparser/test_filterparser.py

@@ -40,8 +40,8 @@ def test_repr(self):
         self.assertIsNotNone(repr(self.parser))
 
 
-class ParserTestV0_10_0(unittest.TestCase):
-    version = (0, 10, 0)
+class ParserTestV0_10_1(unittest.TestCase):
+    version = (0, 10, 1)
     variant = "default"
 
     @classmethod
@@ -186,6 +186,8 @@ def test_list_properties(self):
 
         # OPTIONAL:
         self.assertIsInstance(self.parse('elements HAS ONLY "H","He","Ga","Ta"'), Tree)
+        self.assertIsInstance(self.parse('elements HAS ALL "H","He","Ga","Ta"'), Tree)
+        self.assertIsInstance(self.parse('elements HAS ANY "H","He","Ga","Ta"'), Tree)
         self.assertIsInstance(
             self.parse(
                 'elements:_exmpl_element_counts HAS "H":6 AND '

tests/filtertransformers/test_mongo.py

@@ -223,36 +223,39 @@ def test_operators(self):
         with self.assertRaises(VisitError):
             self.transform('"some string" > "some other string"')
 
-    def test_list_properties(self):
-        """Test queries using list properties
+    def test_not_implemented(self):
+        """ Test that list properties that are currently not implemented
+        give a sensible response.
 
-        NOTE: Some of these are not implemented yet, these will be tested to raise.
         """
-        # Comparisons of list properties
+        # NOTE: Lark catches underlying filtertransformer exceptions and
+        # raises VisitErrors, most of these actually correspond to NotImplementedError
         with self.assertRaises(VisitError):
-            self.transform("list HAS < 3")
+            try:
+                self.transform("list HAS < 3")
+            except Exception as exc:
+                self.assertTrue("not implemented" in str(exc))
+                raise exc
 
         with self.assertRaises(VisitError):
-            self.transform("list HAS ALL < 3, > 3")
+            try:
+                self.transform("list HAS ALL < 3, > 3")
+            except Exception as exc:
+                self.assertTrue("not implemented" in str(exc))
+                raise exc
 
         with self.assertRaises(VisitError):
-            self.transform("list HAS ANY > 3, < 6")
+            try:
+                self.transform("list HAS ANY > 3, < 6")
+            except Exception as exc:
+                self.assertTrue("not implemented" in str(exc))
+                raise exc
 
         self.assertEqual(self.transform("list LENGTH 3"), {"list": {"$size": 3}})
 
         with self.assertRaises(VisitError):
             self.transform("list:list HAS >=2:<=5")
 
-        with self.assertRaises(VisitError):
-            self.transform(
-                'elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS '
-                'ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta"'
-            )
-
-        # OPTIONAL:
-        with self.assertRaises(VisitError):
-            self.transform('elements HAS ONLY "H","He","Ga","Ta"')
-
         with self.assertRaises(VisitError):
             self.transform(
                 'elements:_exmpl_element_counts HAS "H":6 AND elements:_exmpl_element_counts '
@@ -276,6 +279,40 @@ def test_list_properties(self):
         with self.assertRaises(VisitError):
             self.transform("list LENGTH > 3")
 
+    def test_list_properties(self):
+        """ Test the HAS ALL, ANY and optional ONLY queries.
+
+        """
+        self.assertEqual(
+            self.transform('elements HAS ONLY "H","He","Ga","Ta"'),
+            {"elements": {"$all": ["H", "He", "Ga", "Ta"], "$size": 4}},
+        )
+
+        self.assertEqual(
+            self.transform('elements HAS ANY "H","He","Ga","Ta"'),
+            {"elements": {"$in": ["H", "He", "Ga", "Ta"]}},
+        )
+
+        self.assertEqual(
+            self.transform('elements HAS ALL "H","He","Ga","Ta"'),
+            {"elements": {"$all": ["H", "He", "Ga", "Ta"]}},
+        )
+
+        self.assertEqual(
+            self.transform(
+                'elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS '
+                'ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta"'
+            ),
+            {
+                "$and": [
+                    {"elements": {"$in": ["H"]}},
+                    {"elements": {"$all": ["H", "He", "Ga", "Ta"]}},
+                    {"elements": {"$all": ["H", "He", "Ga", "Ta"], "$size": 4}},
+                    {"elements": {"$in": ["H", "He", "Ga", "Ta"]}},
+                ]
+            },
+        )
+
     def test_properties(self):
         #  Filtering on Properties with unknown value
         # TODO: {'$not': {'$exists': False}} can be simplified to {'$exists': True}

tests/server/test_query_params.py

@@ -331,28 +331,37 @@ def test_page_limit_max(self):
             expected_detail=f"Max allowed page_limit is {CONFIG.page_limit_max}, you requested {CONFIG.page_limit_max + 1}",
         )
 
-    def test_list_has_all(self):
-        request = '/structures?filter=elements HAS ALL "Ba","F","H","Mn","O","Re","Si"'
+    def test_value_list_operator(self):
+        request = "/structures?filter=dimension_types HAS < 1"
         self._check_error_response(
-            request, expected_status=501, expected_title="NotImplementedError"
+            request,
+            expected_status=501,
+            expected_title="NotImplementedError",
+            expected_detail="set_op_rhs not implemented for use with OPERATOR. Given: [Token(HAS, 'HAS'), Token(OPERATOR, '<'), 1]",
         )
-        # expected_ids = ["mpf_3819"]
-        # self._check_response(request, expected_ids, len(expected_ids))
 
-        request = '/structures?filter=elements HAS ALL "Re","Ti"'
+    def test_has_any_operator(self):
+        request = "/structures?filter=dimension_types HAS ANY > 1"
         self._check_error_response(
-            request, expected_status=501, expected_title="NotImplementedError"
+            request,
+            expected_status=501,
+            expected_title="NotImplementedError",
+            expected_detail="OPERATOR > inside value_list [Token(OPERATOR, '>'), 1] not implemented.",
         )
-        # expected_ids = ["mpf_3819"]
-        # self._check_response(request, expected_ids, len(expected_ids))
+
+    def test_list_has_all(self):
+        request = '/structures?filter=elements HAS ALL "Ba","F","H","Mn","O","Re","Si"'
+        expected_ids = ["mpf_3819"]
+        self._check_response(request, expected_ids, len(expected_ids))
+
+        request = '/structures?filter=elements HAS ALL "Re","Ti"'
+        expected_ids = ["mpf_3819"]
+        self._check_response(request, expected_ids, len(expected_ids))
 
     def test_list_has_any(self):
         request = '/structures?filter=elements HAS ANY "Re","Ti"'
-        self._check_error_response(
-            request, expected_status=501, expected_title="NotImplementedError"
-        )
-        # expected_ids = ["mpf_3819"]
-        # self._check_response(request, expected_ids, len(expected_ids))
+        expected_ids = ["mpf_3819", "mpf_3803"]
+        self._check_response(request, expected_ids, len(expected_ids))
 
     def test_list_length_basic(self):
         request = "/structures?filter=elements LENGTH = 9"
@@ -386,12 +395,22 @@ def test_list_length(self):
         # self._check_response(request, expected_ids, len(expected_ids))
 
     def test_list_has_only(self):
+        """ Test HAS ONLY query on elements.
+
+        This test fails with mongomock<=3.1.9 when $size is 1, but works with a real mongo.
+
+        TODO: this text should be removed once mongomock>3.1.9 has been released, which should
+        contain the bugfix for this: https://github.com/mongomock/mongomock/pull/597.
+
+        """
+
+        request = '/structures?filter=elements HAS ONLY "Ac", "Mg"'
+        expected_ids = ["mpf_23"]
+        self._check_response(request, expected_ids, len(expected_ids))
+
         request = '/structures?filter=elements HAS ONLY "Ac"'
-        self._check_error_response(
-            request, expected_status=501, expected_title="NotImplementedError"
-        )
-        # expected_ids = ["mpf_1"]
-        # self._check_response(request, expected_ids, len(expected_ids))
+        expected_ids = ["mpf_1"]
+        self._check_response(request, expected_ids, len(expected_ids))
 
     def test_list_correlated(self):
         request = '/structures?filter=elements:elements_ratios HAS "Ag":"0.2"'