From 93b198ed3bbae2c521e1bb9a15b164c41e4c98c7 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 24 May 2022 17:58:15 +0100 Subject: [PATCH] Added instructions and user guide for `OptimadeClient` - Added config for tabbed code blocks - Limit all code blocks to 20em before overflow --- INSTALL.md | 2 +- docs/api_reference/client/.pages | 1 + docs/api_reference/client/cli.md | 3 + docs/api_reference/client/client.md | 3 + docs/api_reference/client/utils.md | 3 + docs/css/reference.css | 4 + docs/getting_started/.pages | 1 + docs/getting_started/client.md | 258 ++++++++++++++++++++++++++++ mkdocs.yml | 3 +- 9 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 docs/api_reference/client/.pages create mode 100644 docs/api_reference/client/cli.md create mode 100644 docs/api_reference/client/client.md create mode 100644 docs/api_reference/client/utils.md create mode 100644 docs/getting_started/client.md diff --git a/INSTALL.md b/INSTALL.md index 9f386cfc54..a24f85ac5d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -2,7 +2,7 @@ This package can be installed from PyPI, or by cloning the repository, depending on your use-case. -1. To use the `optimade` Python package as a library, (e.g., using the models for validation, parsing filters with the grammar, or using the command-line tool `optimade-validator` tool), it is recommended that you install the latest release of the package from PyPI with `pip install optimade`. +1. To use the `optimade` Python package as a library, (e.g., using the models for validation, parsing filters with the grammar, or using the command-line tool `optimade-validator` tool), it is recommended that you install the latest release of the package from PyPI with `pip install optimade`. If you also want to use the OPTIMADE client to query OPTIMADE APIs, you should install with the additional dependencies: `pip install optimade[http_client]`. 2. If you want to run, use or modify the reference server implementation, then it is recommended that you clone this repository and install it from your local files (with `pip install .`, or `pip install -e .` for an editable installation). As an alternative, you can run the `optimade` container image (see the [Container image](#container-image) section below). diff --git a/docs/api_reference/client/.pages b/docs/api_reference/client/.pages new file mode 100644 index 0000000000..411a2b4756 --- /dev/null +++ b/docs/api_reference/client/.pages @@ -0,0 +1 @@ +title: "client" diff --git a/docs/api_reference/client/cli.md b/docs/api_reference/client/cli.md new file mode 100644 index 0000000000..0fbcffe8d4 --- /dev/null +++ b/docs/api_reference/client/cli.md @@ -0,0 +1,3 @@ +# cli + +::: optimade.client.cli diff --git a/docs/api_reference/client/client.md b/docs/api_reference/client/client.md new file mode 100644 index 0000000000..8b41d88799 --- /dev/null +++ b/docs/api_reference/client/client.md @@ -0,0 +1,3 @@ +# client + +::: optimade.client.client diff --git a/docs/api_reference/client/utils.md b/docs/api_reference/client/utils.md new file mode 100644 index 0000000000..17eb5f68db --- /dev/null +++ b/docs/api_reference/client/utils.md @@ -0,0 +1,3 @@ +# utils + +::: optimade.client.utils diff --git a/docs/css/reference.css b/docs/css/reference.css index 511fdd5d66..a226d9ec33 100644 --- a/docs/css/reference.css +++ b/docs/css/reference.css @@ -7,3 +7,7 @@ div.doc-contents:not(.first) { td code { word-break: normal !important; } + +code { + max-height: 20em; +} diff --git a/docs/getting_started/.pages b/docs/getting_started/.pages index d1e0b3e977..93dcaa6a2b 100644 --- a/docs/getting_started/.pages +++ b/docs/getting_started/.pages @@ -1,4 +1,5 @@ title: "Getting started" nav: + - client.md - setting_up_an_api.md - use_cases.md diff --git a/docs/getting_started/client.md b/docs/getting_started/client.md new file mode 100644 index 0000000000..0205d44fae --- /dev/null +++ b/docs/getting_started/client.md @@ -0,0 +1,258 @@ +# Using the OPTIMADE client + +This package includes a Python client that can be used to query multiple OPTIMADE APIs simultaneously. +The client can be used from the command-line (`optimade-get`), or called in Python code. + +The client requires some extra dependencies that can be installed with the PyPI package with + +```shell +pip install optimade[http_client] +``` +or from a local copy of the repository with +```shell +pip install -e .[http_client] +``` + +By default, the client will query all OPTIMADE API URLs that it can find via the [Providers list](https://providers.optimade.org): + + +=== "Command line" + ```shell + optimade-get --filter 'elements HAS "Ag"' + ``` + +=== "Python" + ```python + from optimade.client import OptimadeClient + client = OptimadeClient() + results = client.get() + ``` + +We can refine the search by manually specifying some URLs: + +=== "Command line" + ```shell + optimade-get https://optimade.herokuapp.com https://optimade.odbx.science + ``` + +=== "Python" + ```python + from optimade.client import OptimadeClient + client = OptimadeClient( + base_urls=["https://optimade.herokuapp.com", "https://optimade.odbx.science"] + ) + client.get() + ``` + +By default, the command-line interface will use an example filter, and the Python interface will use an empty filter. +You can specify your desired filter as follows (note the quotation marks): + +=== "Command line" + ```shell + optimade-get --filter 'elements HAS "Ag" AND nsites < 2' + ``` + +=== "Python" + ```python + from optimade.client import OptimadeClient + client = OptimadeClient() + client.get('elements HAS "Ag" AND nsites < 2') + ``` + +The filter will be validated against the `optimade-python-tools` reference grammar before it is sent to the underlying servers. + +## Accessing the results + +At the command-line, the results of the query will be printed to `stdout` to be redirected to a file or piped into another program. +For example: + +```shell +optimade-get --filter 'nsites = 1' https://optimade.herokuapp.com +``` + +has the followng (truncated) output: + +```json +{ + // The endpoint that was queried + "structures": { + // The filter applied to that endpointk + "nsites = 1": { + // The base URL of the OPTIMADE API + "https://optimade.herokuapp.com": { + // The OPTIMADE API response as if called with an infinite `page_limit` + "data": [ + { + "id": "mpf_1", + "type": "structures", + "attributes": { + ... + } + "relationships": { + ... + } + }, + ... + ], + "errors": [], + "links": { + "next": null, + "prev": null + }, + "included": [ + ... + ], + "meta": { + ... + } + } + } + } +} +``` + +The response is broken down by queried endpoint, filter and then base URL so that the query URL can be easily reconstructed. +This is the same format as the cached results of the Python client: + +```python +from optimade.client import OptimadeClient +import json +client = OptimadeClient(base_urls="https://optimade.herokuapp.com") +client.get('nsites = 1') +client.get('nsites = 2') +print(json.dumps(client.all_results, indent=2)) +``` + +will return a dictionary with top-level keys: +```json +{ + "structures": { + "nsites = 1": { + "https://optimade.herokuapp.com": {...} + }, + "nsites = 2": { + "https://optimade.herokuapp.com": {...} + } + } +} +``` + +For a given session, this cache can be written and reloaded into an OPTIMADE client object to avoid needing to repeat queries. + +!!! info + In a future release, this cache will be automatically restored from disk and will obey defined cache lifetimes. + +### Counting entries and limiting results + +Querying all OPTIMADE APIs without limiting the number of entries can result in a lot of data. +The client will limit the number of results returned per database to the value of `max_results_per_provider` (defaults: 1000 for Python, 100 for CLI). +This limit will be enforced up to a difference of the default page limit for the underlying OPTIMADE API (which is used everywhere). +This parameter can be controlled via the `--max-results-per-provider 10` at the CLI, or as an argument to `OptimadeClient(max_results_per_provider=10)`. + +Downloading all the results for a given query can require hundreds or thousands of requests, depending on the number of results and the database's page limit. +It is possible to just count the number of results before downloading the entries themselves, which only requires 1 request per database. +This is achieved via the `--count` flag in the CLI, or the `.count()` method in the Python interface. +We can use this to repeat the queries from the [OPTIMADE paper](https://doi.org/10.1038/s41597-021-00974-z): + +=== "Command line" + ```shell + optimade-get \ + --count \ + --filter 'elements HAS ANY "C", "Si", "Ge", "Sn", "Pb"' \ + --filter 'elements HAS ANY "C", "Si", "Ge", "Sn", "Pb" AND nelements=2' \ + --filter 'elements HAS ANY "C", "Si", "Ge", "Sn" AND NOT elements HAS "Pb" AND elements LENGTH 3' + ``` + +=== "Python" + ```python + from optimade.client import OptimadeClient + client = OptimadeClient() + filters = [ + 'elements HAS ANY "C", "Si", "Ge", "Sn", "Pb"', + 'elements HAS ANY "C", "Si", "Ge", "Sn", "Pb" AND nelements=2' + 'elements HAS ANY "C", "Si", "Ge", "Sn" AND NOT elements HAS "Pb" AND elements LENGTH 3' + ] + for f in filters: + client.count(f) + ``` + +which, at the timing of writing, yields the results: + +```json +{ + "structures": { + "elements HAS ANY \"C\", \"Si\", \"Ge\", \"Sn\", \"Pb\"": { + "http://aflow.org/API/optimade/": null, + "https://www.crystallography.net/cod/optimade": 436062, + "https://aiida.materialscloud.org/sssplibrary/optimade": 487, + "https://aiida.materialscloud.org/2dstructures/optimade": 1427, + "https://aiida.materialscloud.org/2dtopo/optimade": 0, + "https://aiida.materialscloud.org/tc-applicability/optimade": 3719, + "https://aiida.materialscloud.org/3dd/optimade": null, + "https://aiida.materialscloud.org/mc3d-structures/optimade": 9592, + "https://aiida.materialscloud.org/autowannier/optimade": 1093, + "https://aiida.materialscloud.org/curated-cofs/optimade": 4395, + "https://aiida.materialscloud.org/stoceriaitf/optimade": 0, + "https://aiida.materialscloud.org/pyrene-mofs/optimade": 348, + "https://aiida.materialscloud.org/tin-antimony-sulfoiodide/optimade": 503, + "https://optimade.materialsproject.org": 30351, + "https://api.mpds.io": null, + "https://nomad-lab.eu/prod/rae/optimade/": 4451056, + "https://optimade.odbx.science": 55, + "http://optimade.openmaterialsdb.se": 58718, + "http://oqmd.org/optimade/": null, + "https://jarvis.nist.gov/optimade/jarvisdft": null, + "https://www.crystallography.net/tcod/optimade": 2632, + "http://optimade.2dmatpedia.org": 1172 + }, + "elements HAS ANY \"C\", \"Si\", \"Ge\", \"Sn\", \"Pb\" AND nelements=2": { + "http://aflow.org/API/optimade/": 63011, + "https://www.crystallography.net/cod/optimade": 3968, + "https://aiida.materialscloud.org/sssplibrary/optimade": 2, + "https://aiida.materialscloud.org/2dstructures/optimade": 779, + "https://aiida.materialscloud.org/2dtopo/optimade": 0, + "https://aiida.materialscloud.org/tc-applicability/optimade": 334, + "https://aiida.materialscloud.org/3dd/optimade": null, + "https://aiida.materialscloud.org/mc3d-structures/optimade": 1566, + "https://aiida.materialscloud.org/autowannier/optimade": 276, + "https://aiida.materialscloud.org/curated-cofs/optimade": 24, + "https://aiida.materialscloud.org/stoceriaitf/optimade": 0, + "https://aiida.materialscloud.org/pyrene-mofs/optimade": 0, + "https://aiida.materialscloud.org/tin-antimony-sulfoiodide/optimade": 0, + "https://optimade.materialsproject.org": 3728, + "https://api.mpds.io": null, + "https://nomad-lab.eu/prod/rae/optimade/": 587923, + "https://optimade.odbx.science": 54, + "http://optimade.openmaterialsdb.se": 690, + "http://oqmd.org/optimade/": null, + "https://jarvis.nist.gov/optimade/jarvisdft": null, + "https://www.crystallography.net/tcod/optimade": 296, + "http://optimade.2dmatpedia.org": 739 + }, + "elements HAS ANY \"C\", \"Si\", \"Ge\", \"Sn\" AND NOT elements HAS \"Pb\" AND elements LENGTH 3": { + "http://aflow.org/API/optimade/": null, + "https://www.crystallography.net/cod/optimade": 33776, + "https://aiida.materialscloud.org/sssplibrary/optimade": 0, + "https://aiida.materialscloud.org/2dstructures/optimade": 378, + "https://aiida.materialscloud.org/2dtopo/optimade": 0, + "https://aiida.materialscloud.org/tc-applicability/optimade": 144, + "https://aiida.materialscloud.org/3dd/optimade": null, + "https://aiida.materialscloud.org/mc3d-structures/optimade": 4398, + "https://aiida.materialscloud.org/autowannier/optimade": 74, + "https://aiida.materialscloud.org/curated-cofs/optimade": 1447, + "https://aiida.materialscloud.org/stoceriaitf/optimade": 0, + "https://aiida.materialscloud.org/pyrene-mofs/optimade": 0, + "https://aiida.materialscloud.org/tin-antimony-sulfoiodide/optimade": 0, + "https://optimade.materialsproject.org": 11559, + "https://api.mpds.io": null, + "https://nomad-lab.eu/prod/rae/optimade/": 2092989, + "https://optimade.odbx.science": 0, + "http://optimade.openmaterialsdb.se": 7428, + "http://oqmd.org/optimade/": null, + "https://jarvis.nist.gov/optimade/jarvisdft": null, + "https://www.crystallography.net/tcod/optimade": 661, + "http://optimade.2dmatpedia.org": 255 + } + } +} +``` diff --git a/mkdocs.yml b/mkdocs.yml index 2287af6883..e0dd91e528 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,8 @@ markdown_extensions: - pymdownx.highlight - pymdownx.superfences - pymdownx.inlinehilite - - pymdownx.tabbed + - pymdownx.tabbed: + alternate_style: true - pymdownx.snippets - toc: permalink: true