Merged changes from master.

.github/workflows/ci.yml

@@ -60,6 +60,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Run pre-commit
@@ -85,6 +86,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Pass generated OpenAPI schemas through validator.swagger.io
@@ -194,25 +196,29 @@ jobs:
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
 
-    - name: Run all tests (using `mongomock`)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/
+    - name: Run non-server tests
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ --ignore tests/server
+
+    - name: Install latest server dependencies
+      run: pip install -r requirements-server.txt
+
+    - name: Run server tests (using `mongomock`)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongomock'
 
-
     - name: Run server tests (using a real MongoDB)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongodb'
 
     - name: Run server tests (using Elasticsearch)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'elastic'
 
     - name: Install adapter conversion dependencies
-      run: |
-        pip install -r requirements-client.txt
+      run: pip install -r requirements-client.txt
 
     - name: Setup environment for AiiDA
       env:
@@ -305,6 +311,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
         pip install -r requirements-docs.txt

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       name: Blacken
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     - id: check-symlinks
     - id: check-yaml
@@ -24,7 +24,7 @@ repos:
       args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/pycqa/flake8
-    rev: '5.0.4'
+    rev: '6.0.0'
     hooks:
     - id: flake8
 
@@ -43,3 +43,16 @@ repos:
       pass_filenames: false
       files: ^optimade/.*\.py$
       description: Update the API Reference documentation whenever a Python file is touched in the code base.
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+      - id: mypy
+        name: "MyPy"
+        additional_dependencies: ["types-all", "pydantic~=1.10"]
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: "isort"

CHANGELOG.md

@@ -2,20 +2,60 @@
 
 ## [Unreleased](https://github.com/Materials-Consortia/optimade-python-tools/tree/HEAD)
 
-[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...HEAD)
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.20.0...HEAD)
 
-This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version.
+This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages.
+
+Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku.
+
+## [v0.20.0](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) (2022-11-29)
+
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...v0.20.0)
+
+This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages.
+
+Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku.
+
+**Implemented enhancements:**
+
+- Add support for Python 3.11 [\#1361](https://github.com/Materials-Consortia/optimade-python-tools/issues/1361)
+- Improve test diagnostics and fix deprecated Elasticsearch calls [\#1373](https://github.com/Materials-Consortia/optimade-python-tools/pull/1373) ([ml-evs](https://github.com/ml-evs))
+- Support Python 3.11 [\#1362](https://github.com/Materials-Consortia/optimade-python-tools/pull/1362) ([ml-evs](https://github.com/ml-evs))
 
 **Fixed bugs:**
 
-- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335)
+- Elasticsearch pytest oneliner in the docs is no longer working [\#1377](https://github.com/Materials-Consortia/optimade-python-tools/issues/1377)
+- Remote swagger validator has changed output format [\#1370](https://github.com/Materials-Consortia/optimade-python-tools/issues/1370)
+
+**Closed issues:**
+
+- Fully isolate server code from other submodules [\#1403](https://github.com/Materials-Consortia/optimade-python-tools/issues/1403)
+- Replace https://gitlab.com/pycqa/flake8 with https://github.com/pycqa/flake8 [\#1388](https://github.com/Materials-Consortia/optimade-python-tools/issues/1388)
+- OpenAPI schema should not enforce recommended constraint on `page_number` [\#1372](https://github.com/Materials-Consortia/optimade-python-tools/issues/1372)
+- Pydantic models docs are broken on the mkdocs site with new renderer [\#1353](https://github.com/Materials-Consortia/optimade-python-tools/issues/1353)
+- Migrate away from Heroku for demo server [\#1307](https://github.com/Materials-Consortia/optimade-python-tools/issues/1307)
+- FastAPI should not be a core dependency [\#1198](https://github.com/Materials-Consortia/optimade-python-tools/issues/1198)
+
+**Merged pull requests:**
+
+- Move exceptions and warnings out of server code and separate deps [\#1405](https://github.com/Materials-Consortia/optimade-python-tools/pull/1405) ([ml-evs](https://github.com/ml-evs))
+- Complete migration from Heroku to Fly [\#1400](https://github.com/Materials-Consortia/optimade-python-tools/pull/1400) ([ml-evs](https://github.com/ml-evs))
+- Add GH actions for deploying example server to Fly [\#1396](https://github.com/Materials-Consortia/optimade-python-tools/pull/1396) ([ml-evs](https://github.com/ml-evs))
+- Support new remote swagger.io validator format [\#1371](https://github.com/Materials-Consortia/optimade-python-tools/pull/1371) ([ml-evs](https://github.com/ml-evs))
+- Do not enforce minimum value of `page_number` at model level [\#1369](https://github.com/Materials-Consortia/optimade-python-tools/pull/1369) ([ml-evs](https://github.com/ml-evs))
+- Enable `mypy` and `isort` in pre-commit & CI [\#1346](https://github.com/Materials-Consortia/optimade-python-tools/pull/1346) ([ml-evs](https://github.com/ml-evs))
+- Remove randomness from structure utils tests [\#1338](https://github.com/Materials-Consortia/optimade-python-tools/pull/1338) ([ml-evs](https://github.com/ml-evs))
+- Demote FastAPI to a server dep only [\#1199](https://github.com/Materials-Consortia/optimade-python-tools/pull/1199) ([ml-evs](https://github.com/ml-evs))
 
 ## [v0.19.4](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) (2022-09-19)
 
 [Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.3...v0.19.4)
 
+This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version.
+
 **Fixed bugs:**
 
+- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335)
 - Ensure `chunk_size` is properly set when chunking responses [\#1336](https://github.com/Materials-Consortia/optimade-python-tools/pull/1336) ([ml-evs](https://github.com/ml-evs))
 
 ## [v0.19.3](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.3) (2022-09-06)

docs/api_reference/exceptions.md

@@ -0,0 +1,3 @@
+# exceptions
+
+::: optimade.exceptions

docs/api_reference/utils.md

@@ -0,0 +1,3 @@
+# utils
+
+::: optimade.utils

docs/api_reference/warnings.md

@@ -0,0 +1,3 @@
+# warnings
+
+::: optimade.warnings

docs/static/default_config.json

@@ -13,7 +13,7 @@
     "base_url": null,
     "implementation": {
         "name": "OPTIMADE Python Tools",
-        "version": "0.19.4",
+        "version": "0.20.0",
         "source_url": "https://github.com/Materials-Consortia/optimade-python-tools",
         "maintainer": {"email": "dev@optimade.org"}
     },

openapi/index_openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.2",
   "info": {
     "title": "OPTIMADE API - Index meta-database",
-    "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\nThis is the \"special\" index meta-database.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) v0.19.4.",
+    "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\nThis is the \"special\" index meta-database.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) v0.20.0.",
     "version": "1.1.0"
   },
   "paths": {

openapi/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.2",
   "info": {
     "title": "OPTIMADE API",
-    "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) v0.19.4.",
+    "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) v0.20.0.",
     "version": "1.1.0"
   },
   "paths": {
@@ -3359,9 +3359,9 @@
               "type": "number"
             },
             "description": "If present MUST be a list of floats expressed in a.m.u.\nElements denoting vacancies MUST have masses equal to 0.",
-            "x-optimade-unit": "a.m.u.",
             "x-optimade-support": "optional",
-            "x-optimade-queryable": "optional"
+            "x-optimade-queryable": "optional",
+            "x-optimade-unit": "a.m.u."
           },
           "original_name": {
             "title": "Original Name",
@@ -3639,9 +3639,9 @@
             },
             "description": "The three lattice vectors in Cartesian coordinates, in \u00e5ngstr\u00f6m (\u00c5).\n\n- **Type**: list of list of floats or unknown values.\n\n- **Requirements/Conventions**:\n    - **Support**: SHOULD be supported by all implementations, i.e., SHOULD NOT be `null`.\n    - **Query**: Support for queries on this property is OPTIONAL.\n      If supported, filters MAY support only a subset of comparison operators.\n    - MUST be a list of three vectors *a*, *b*, and *c*, where each of the vectors MUST BE a list of the vector's coordinates along the x, y, and z Cartesian coordinates.\n      (Therefore, the first index runs over the three lattice vectors and the second index runs over the x, y, z Cartesian coordinates).\n    - For databases that do not define an absolute Cartesian system (e.g., only defining the length and angles between vectors), the first lattice vector SHOULD be set along *x* and the second on the *xy*-plane.\n    - MUST always contain three vectors of three coordinates each, independently of the elements of property `dimension_types`.\n      The vectors SHOULD by convention be chosen so the determinant of the `lattice_vectors` matrix is different from zero.\n      The vectors in the non-periodic directions have no significance beyond fulfilling these requirements.\n    - The coordinates of the lattice vectors of non-periodic dimensions (i.e., those dimensions for which `dimension_types` is `0`) MAY be given as a list of all `null` values.\n        If a lattice vector contains the value `null`, all coordinates of that lattice vector MUST be `null`.\n\n- **Examples**:\n    - `[[4.0,0.0,0.0],[0.0,4.0,0.0],[0.0,1.0,4.0]]` represents a cell, where the first vector is `(4, 0, 0)`, i.e., a vector aligned along the `x` axis of length 4 \u00c5; the second vector is `(0, 4, 0)`; and the third vector is `(0, 1, 4)`.",
             "nullable": true,
-            "x-optimade-unit": "\u00c5",
             "x-optimade-support": "should",
-            "x-optimade-queryable": "optional"
+            "x-optimade-queryable": "optional",
+            "x-optimade-unit": "\u00c5"
           },
           "cartesian_site_positions": {
             "title": "Cartesian Site Positions",
@@ -3656,9 +3656,9 @@
             },
             "description": "Cartesian positions of each site in the structure.\nA site is usually used to describe positions of atoms; what atoms can be encountered at a given site is conveyed by the `species_at_sites` property, and the species themselves are described in the `species` property.\n\n- **Type**: list of list of floats\n\n- **Requirements/Conventions**:\n    - **Support**: SHOULD be supported by all implementations, i.e., SHOULD NOT be `null`.\n    - **Query**: Support for queries on this property is OPTIONAL.\n      If supported, filters MAY support only a subset of comparison operators.\n    - It MUST be a list of length equal to the number of sites in the structure, where every element is a list of the three Cartesian coordinates of a site expressed as float values in the unit angstrom (\u00c5).\n    - An entry MAY have multiple sites at the same Cartesian position (for a relevant use of this, see e.g., the property `assemblies`).\n\n- **Examples**:\n    - `[[0,0,0],[0,0,2]]` indicates a structure with two sites, one sitting at the origin and one along the (positive) *z*-axis, 2 \u00c5 away from the origin.",
             "nullable": true,
-            "x-optimade-unit": "\u00c5",
             "x-optimade-support": "should",
-            "x-optimade-queryable": "optional"
+            "x-optimade-queryable": "optional",
+            "x-optimade-unit": "\u00c5"
           },
           "nsites": {
             "title": "Nsites",

optimade/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.19.4"
+__version__ = "0.20.0"
 __api_version__ = "1.1.0"

optimade/adapters/__init__.py

@@ -3,5 +3,4 @@
 from .references import *  # noqa: F403
 from .structures import *  # noqa: F403
 
-
-__all__ = exceptions.__all__ + references.__all__ + structures.__all__  # noqa: F405
+__all__ = exceptions.__all__ + references.__all__ + structures.__all__  # type: ignore[name-defined]  # noqa: F405

optimade/adapters/base.py

@@ -19,13 +19,12 @@
 and [`StructureResource`][optimade.models.structures.StructureResource]s, respectively.
 """
 import re
-from typing import Union, Dict, Callable, Any, Tuple, List, Type
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 
 from pydantic import BaseModel  # pylint: disable=no-name-in-module
 
-from optimade.models import EntryResource
-
 from optimade.adapters.logger import LOGGER
+from optimade.models.entries import EntryResource
 
 
 class EntryAdapter:
@@ -49,15 +48,17 @@ def __init__(self, entry: dict) -> None:
         Parameters:
             entry (dict): A JSON OPTIMADE single resource entry.
         """
-        self._entry = None
-        self._converted = {}
+        self._entry: Optional[EntryResource] = None
+        self._converted: Dict[str, Any] = {}
 
-        self.entry = entry
+        self.entry: EntryResource = entry  # type: ignore[assignment]
 
         # Note that these return also the default values for otherwise non-provided properties.
         self._common_converters = {
-            "json": self.entry.json,  # Return JSON serialized string, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeljson
-            "dict": self.entry.dict,  # Return Python dict, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict
+            # Return JSON serialized string, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeljson
+            "json": self.entry.json,  # type: ignore[attr-defined]
+            # Return Python dict, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict
+            "dict": self.entry.dict,  # type: ignore[attr-defined]
         }
 
     @property
@@ -68,7 +69,7 @@ def entry(self) -> EntryResource:
             The entry resource.
 
         """
-        return self._entry
+        return self._entry  # type: ignore[return-value]
 
     @entry.setter
     def entry(self, value: dict) -> None:
@@ -172,12 +173,12 @@ def __getattr__(self, name: str) -> Any:
             return res
 
         # Non-valid attribute
-        entry_resource_name = re.match(
+        _entry_resource_name = re.match(
             r"(<class ')([a-zA-Z_]+\.)*([a-zA-Z_]+)('>)", str(self.ENTRY_RESOURCE)
         )
         entry_resource_name = (
-            entry_resource_name.group(3)
-            if entry_resource_name is not None
+            _entry_resource_name.group(3)
+            if _entry_resource_name is not None
             else "UNKNOWN RESOURCE"
         )
         raise AttributeError(

optimade/adapters/references/__init__.py

@@ -1,4 +1,3 @@
 from .adapter import Reference
 
-
 __all__ = ("Reference",)

optimade/adapters/references/adapter.py

@@ -1,5 +1,7 @@
-from optimade.models import ReferenceResource
+from typing import Type
+
 from optimade.adapters.base import EntryAdapter
+from optimade.models import ReferenceResource
 
 
 class Reference(EntryAdapter):
@@ -19,4 +21,4 @@ class Reference(EntryAdapter):
 
     """
 
-    ENTRY_RESOURCE: ReferenceResource = ReferenceResource
+    ENTRY_RESOURCE: Type[ReferenceResource] = ReferenceResource

optimade/adapters/structures/__init__.py

@@ -1,4 +1,3 @@
 from .adapter import Structure
 
-
 __all__ = ("Structure",)

optimade/adapters/structures/adapter.py

@@ -1,13 +1,14 @@
 from typing import Callable, Dict, Type
-from optimade.models import StructureResource
+
 from optimade.adapters.base import EntryAdapter
+from optimade.models import StructureResource
 
 from .aiida import get_aiida_structure_data
 from .ase import get_ase_atoms
 from .cif import get_cif
-from .proteindatabank import get_pdb, get_pdbx_mmcif
-from .pymatgen import get_pymatgen, from_pymatgen
 from .jarvis import get_jarvis_atoms
+from .proteindatabank import get_pdb, get_pdbx_mmcif
+from .pymatgen import from_pymatgen, get_pymatgen
 
 
 class Structure(EntryAdapter):