Overhaul of the validator

- Split validator into validator and utils submodules
- Add validator specific-coverage report
- Disable parsing filter examples from specification for validation
- Removed `optimade.validator.data` submodule
- Removed corresponding invoke task
- Removed tests from validator
- Minor tweaks, function renames and update docs
- Added dynamic filters of every field based on type reported in entry info
- Add another test client that fakes HTTP exceptions
- Created optimade.validator.config module
- Updates to @test_case decorator, and test for MUST fields in entry info
- Added test to check that all MUST properties are present in entry info
- Allow test cases to return None to indicate that the test should be
  ignored. The use case here is e.g. if an optional field is missing,
  then we don't care and the test can be ignored
- Added `multistage` option to `@test_case` that suppresses output and counters of success on multistage tests
- Added hard filter to running optional tests in @test_case itself
- Made deserialization of multi-entry endpoints optional

.github/workflows/deps_lint.yml

@@ -199,12 +199,25 @@ jobs:
     - name: Run previously skipped tests for adapter conversion
       run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
 
+    - name: Run tests for validator only to assess coverage
+      if: matrix.python-version == 3.8
+      run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml tests/server/test_server_validation.py
+
     - name: Upload coverage to Codecov
       if: matrix.python-version == 3.8 && github.repository == 'Materials-Consortia/optimade-python-tools'
       uses: codecov/codecov-action@v1
       with:
+        name: project
         file: ./coverage.xml
-        flags: unittests
+        flags: project
+
+    - name: Upload validator coverage to Codecov
+      if: matrix.python-version == 3.8 && github.repository == 'Materials-Consortia/optimade-python-tools'
+      uses: codecov/codecov-action@v1
+      with:
+        name: validator
+        file: ./validator_cov.xml
+        flags: validator
 
   docs:
     runs-on: ubuntu-latest

docs/api_reference/validator/config.md

@@ -0,0 +1,3 @@
+# config
+
+::: optimade.validator.config

docs/api_reference/validator/utils.md

@@ -0,0 +1,3 @@
+# utils
+
+::: optimade.validator.utils

optimade/validator/__init__.py

@@ -104,7 +104,7 @@ def validate():
     )
 
     try:
-        validator.main()
+        validator.validate_implementation()
     # catch and print internal exceptions, exiting with non-zero error code
     except Exception:
         traceback.print_exc()

optimade/validator/config.py

@@ -0,0 +1,205 @@
+from typing import Dict, Any, Set
+from pydantic import BaseSettings, Field
+
+from optimade.models import InfoResponse, IndexInfoResponse, DataType
+from optimade.validator.utils import (
+    ValidatorLinksResponse,
+    ValidatorReferenceResponseOne,
+    ValidatorReferenceResponseMany,
+    ValidatorStructureResponseOne,
+    ValidatorStructureResponseMany,
+)
+
+_NON_ENTRY_ENDPOINTS = ("info", "links", "versions")
+
+
+_RESPONSE_CLASSES = {
+    "references": ValidatorReferenceResponseMany,
+    "references/": ValidatorReferenceResponseOne,
+    "structures": ValidatorStructureResponseMany,
+    "structures/": ValidatorStructureResponseOne,
+    "info": InfoResponse,
+    "links": ValidatorLinksResponse,
+}
+
+_RESPONSE_CLASSES_INDEX = {
+    "info": IndexInfoResponse,
+    "links": ValidatorLinksResponse,
+}
+
+# This dictionary will eventually be set by getting the fields from response_classes directly
+_PROPERTIES = {
+    "structures": {
+        "MUST": (
+            "id",
+            "type",
+            "structure_features",
+        ),
+        "SHOULD": (
+            "last_modified",
+            "elements",
+            "nelements",
+            "elements_ratios",
+            "chemical_formula_descriptive",
+            "chemical_formula_reduced",
+            "chemical_formula_anonymous",
+            "dimension_types",
+            "nperiodic_dimensions",
+            "lattice_vectors",
+            "cartesian_site_positions",
+            "nsites",
+            "species_at_sites",
+            "species",
+        ),
+        "OPTIONAL": (
+            "immutable_id",
+            "chemical_formula_hill",
+            "assemblies",
+        ),
+    },
+    "references": {
+        "MUST": ("id", "type"),
+        "SHOULD": (),
+        "OPTIONAL": (
+            "last_modified",
+            "immutable_id",
+            "address",
+            "annote",
+            "booktitle",
+            "chapter",
+            "crossref",
+            "edition",
+            "howpublished",
+            "institution",
+            "journal",
+            "key",
+            "month",
+            "note",
+            "number",
+            "organization",
+            "pages",
+            "publisher",
+            "school",
+            "series",
+            "title",
+            "volume",
+            "year",
+            "bib_type",
+            "authors",
+            "editors",
+            "doi",
+            "url",
+        ),
+    },
+}
+
+_UNIQUE_PROPERTIES = ("id", "immutable_id")
+
+_INCLUSIVE_OPERATORS = {
+    DataType.STRING: (
+        "=",
+        "<=",
+        ">=",
+        "CONTAINS",
+        "STARTS WITH",
+        "STARTS",
+        "ENDS WITH",
+        "ENDS",
+    ),
+    DataType.TIMESTAMP: (
+        "=",
+        "<=",
+        ">=",
+        "CONTAINS",
+        "STARTS WITH",
+        "STARTS",
+        "ENDS WITH",
+        "ENDS",
+    ),
+    DataType.INTEGER: (
+        "=",
+        "<=",
+        ">=",
+    ),
+    DataType.FLOAT: (
+        "=",
+        "<=",
+        ">=",
+    ),
+    DataType.LIST: ("HAS", "HAS ALL", "HAS ANY"),
+}
+
+exclusive_ops = ("!=", "<", ">")
+
+_EXCLUSIVE_OPERATORS = {
+    DataType.STRING: exclusive_ops,
+    DataType.TIMESTAMP: exclusive_ops,
+    DataType.FLOAT: exclusive_ops,
+    DataType.INTEGER: exclusive_ops,
+    DataType.LIST: (),
+}
+
+
+class ValidatorConfig(BaseSettings):
+    """This class stores validator config parameters in a way that
+    can be easily modified for testing niche implementations. Many
+    of these fields are determined by the specification directly,
+    but it may be desirable to modify them in certain cases.
+
+    """
+
+    response_classes: Dict[str, Any] = Field(
+        _RESPONSE_CLASSES,
+        description="Dictionary containing the mapping between endpoints and response classes for the main database",
+    )
+
+    response_classes_index: Dict[str, Any] = Field(
+        _RESPONSE_CLASSES_INDEX,
+        description="Dictionary containing the mapping between endpoints and response classes for the index meta-database",
+    )
+
+    entry_properties: Dict[str, Dict[str, Set[str]]] = Field(
+        _PROPERTIES,
+        description=(
+            "Nested dictionary of endpoints -> support level -> list of field names. "
+            "This field will be deprecated once the machinery to read the support levels of fields from their models directly is implemented."
+        ),
+    )
+
+    unique_properties: Set[str] = Field(
+        _UNIQUE_PROPERTIES,
+        description=(
+            "Fields that should be treated as unique indexes for all endpoints, "
+            "i.e. fields on which filters should return at most one entry."
+        ),
+    )
+
+    inclusive_operators: Dict[DataType, Set[str]] = Field(
+        _INCLUSIVE_OPERATORS,
+        description=(
+            "Dictionary mapping OPTIMADE `DataType`s to a list of operators that are 'inclusive', "
+            "i.e. those that should return entries with the matching value from the filter."
+        ),
+    )
+
+    exclusive_operators: Dict[DataType, Set[str]] = Field(
+        _EXCLUSIVE_OPERATORS,
+        description=(
+            "Dictionary mapping OPTIMADE `DataType`s to a list of operators that are 'exclusive', "
+            "i.e. those that should not return entries with the matching value from the filter."
+        ),
+    )
+
+    links_endpoint: str = Field("links", description="The name of the links endpoint")
+    versions_endpoint: str = Field(
+        "versions", description="The name of the versions endpoint"
+    )
+
+    info_endpoint: str = Field("info", description="The name of the info endpoint")
+    non_entry_endpoints: Set[str] = Field(
+        _NON_ENTRY_ENDPOINTS,
+        description="The list specification-mandated endpoint names that do not contain entries",
+    )
+
+
+VALIDATOR_CONFIG = ValidatorConfig()