From 62ed6437bd041b7fb5808566cf942f1b3865d37f Mon Sep 17 00:00:00 2001 From: ramirezfranciscof Date: Wed, 11 Nov 2020 12:02:50 +0100 Subject: [PATCH] REST API: Add full_types_count as new entry point This feature returns a namespace tree of the available node types in the database (data node_types + process process_types) with the addition of a count at each leaf / branch. It also has the option of doing so for a single user, if the pk is provided as an option. --- aiida/restapi/api.py | 1 + aiida/restapi/common/config.py | 2 +- aiida/restapi/common/identifiers.py | 51 +++++++++++++++++++---- aiida/restapi/common/utils.py | 4 +- aiida/restapi/resources.py | 14 +++++-- aiida/restapi/translator/nodes/node.py | 4 +- tests/restapi/conftest.py | 28 +++++++++++++ tests/restapi/test_identifiers.py | 2 +- tests/restapi/test_routes.py | 23 ++++++----- tests/restapi/test_statistics.py | 56 ++++++++++++++++++++++++++ 10 files changed, 159 insertions(+), 26 deletions(-) create mode 100644 tests/restapi/test_statistics.py diff --git a/aiida/restapi/api.py b/aiida/restapi/api.py index 7b2661efcb..796e9a074f 100644 --- a/aiida/restapi/api.py +++ b/aiida/restapi/api.py @@ -131,6 +131,7 @@ def __init__(self, app=None, **kwargs): '/nodes/projectable_properties/', '/nodes/statistics/', '/nodes/full_types/', + '/nodes/full_types_count/', '/nodes/download_formats/', '/nodes/page/', '/nodes/page//', diff --git a/aiida/restapi/common/config.py b/aiida/restapi/common/config.py index 382e334ea4..117cc95db4 100644 --- a/aiida/restapi/common/config.py +++ b/aiida/restapi/common/config.py @@ -16,7 +16,7 @@ 'LIMIT_DEFAULT': 400, # default records total 'PERPAGE_DEFAULT': 20, # default records per page 'PREFIX': '/api/v4', # prefix for all URLs - 'VERSION': '4.0.1', + 'VERSION': '4.1.0', } APP_CONFIG = { diff --git a/aiida/restapi/common/identifiers.py b/aiida/restapi/common/identifiers.py index f3d38b0924..eb7ea85207 100644 --- a/aiida/restapi/common/identifiers.py +++ b/aiida/restapi/common/identifiers.py @@ -209,14 +209,15 @@ def __str__(self): import json return json.dumps(self.get_description(), sort_keys=True, indent=4) - def __init__(self, namespace, path=None, label=None, full_type=None, is_leaf=True): + def __init__(self, namespace, path=None, label=None, full_type=None, counter=None, is_leaf=True): """Construct a new node class namespace.""" - # pylint: disable=super-init-not-called + # pylint: disable=super-init-not-called, too-many-arguments self._namespace = namespace self._path = path if path else namespace self._full_type = self._infer_full_type(full_type) self._subspaces = {} self._is_leaf = is_leaf + self._counter = counter try: self._label = label if label is not None else self.mapping_path_to_label[path] @@ -293,7 +294,15 @@ def get_description(self): } for _, port in self._subspaces.items(): - result['subspaces'].append(port.get_description()) + subspace_result = port.get_description() + result['subspaces'].append(subspace_result) + if 'counter' in subspace_result: + if self._counter is None: + self._counter = 0 + self._counter = self._counter + subspace_result['counter'] + + if self._counter is not None: + result['counter'] = self._counter return result @@ -352,15 +361,20 @@ def create_namespace(self, name, **kwargs): return self[port_name] -def get_node_namespace(): +def get_node_namespace(user_pk=None, count_nodes=False): """Return the full namespace of all available nodes in the current database. :return: complete node `Namespace` """ + # pylint: disable=too-many-branches from aiida import orm from aiida.plugins.entry_point import is_valid_entry_point_string, parse_entry_point_string - builder = orm.QueryBuilder().append(orm.Node, project=['node_type', 'process_type']).distinct() + filters = {} + if user_pk is not None: + filters['user_id'] = user_pk + + builder = orm.QueryBuilder().append(orm.Node, filters=filters, project=['node_type', 'process_type']).distinct() # All None instances of process_type are turned into '' unique_types = {(node_type, process_type if process_type else '') for node_type, process_type in builder.all()} @@ -371,6 +385,7 @@ def get_node_namespace(): for node_type, process_type in unique_types: label = None + counter = None namespace = None if process_type: @@ -393,12 +408,32 @@ def get_node_namespace(): except IndexError: continue + if count_nodes: + builder = orm.QueryBuilder() + concat_filters = [{'node_type': {'==': node_type}}] + + if node_type.startswith('process.'): + if process_type: + concat_filters.append({'process_type': {'==': process_type}}) + else: + concat_filters.append({'process_type': {'or': [{'==': ''}, {'==': None}]}}) + + if user_pk: + concat_filters.append({'user_id': {'==': user_pk}}) + + if len(concat_filters) == 1: + builder.append(orm.Node, filters=concat_filters[0]) + else: + builder.append(orm.Node, filters={'and': concat_filters}) + + counter = builder.count() + full_type = construct_full_type(node_type, process_type) - namespaces.append((namespace, label, full_type)) + namespaces.append((namespace, label, full_type, counter)) node_namespace = Namespace('node') - for namespace, label, full_type in sorted(namespaces, key=lambda x: x[0], reverse=False): - node_namespace.create_namespace(namespace, label=label, full_type=full_type) + for namespace, label, full_type, counter in sorted(namespaces, key=lambda x: x[0], reverse=False): + node_namespace.create_namespace(namespace, label=label, full_type=full_type, counter=counter) return node_namespace diff --git a/aiida/restapi/common/utils.py b/aiida/restapi/common/utils.py index f0ed01f6cc..bd9e259f53 100644 --- a/aiida/restapi/common/utils.py +++ b/aiida/restapi/common/utils.py @@ -208,8 +208,8 @@ def parse_path(self, path_string, parse_pk_uuid=None): return (resource_type, page, node_id, query_type) if path[0] in [ - 'projectable_properties', 'statistics', 'full_types', 'download', 'download_formats', 'report', 'status', - 'input_files', 'output_files' + 'projectable_properties', 'statistics', 'full_types', 'full_types_count', 'download', 'download_formats', + 'report', 'status', 'input_files', 'output_files' ]: query_type = path.pop(0) if path: diff --git a/aiida/restapi/resources.py b/aiida/restapi/resources.py index 4391970578..18572264e9 100644 --- a/aiida/restapi/resources.py +++ b/aiida/restapi/resources.py @@ -264,15 +264,23 @@ def get(self, id=None, page=None): # pylint: disable=redefined-builtin,invalid- elif query_type == 'statistics': headers = self.utils.build_headers(url=request.url, total_count=0) if filters: - usr = filters['user']['=='] + user_pk = filters['user']['=='] else: - usr = None - results = self.trans.get_statistics(usr) + user_pk = None + results = self.trans.get_statistics(user_pk) elif query_type == 'full_types': headers = self.utils.build_headers(url=request.url, total_count=0) results = self.trans.get_namespace() + elif query_type == 'full_types_count': + headers = self.utils.build_headers(url=request.url, total_count=0) + if filters: + user_pk = filters['user']['=='] + else: + user_pk = None + results = self.trans.get_namespace(user_pk=user_pk, count_nodes=True) + # TODO improve the performance of tree endpoint by getting the data from database faster # TODO add pagination for this endpoint (add default max limit) elif query_type == 'tree': diff --git a/aiida/restapi/translator/nodes/node.py b/aiida/restapi/translator/nodes/node.py index 6c68ee3a6e..69eaedb0dd 100644 --- a/aiida/restapi/translator/nodes/node.py +++ b/aiida/restapi/translator/nodes/node.py @@ -577,12 +577,12 @@ def get_statistics(self, user_pk=None): return qmanager.get_creation_statistics(user_pk=user_pk) @staticmethod - def get_namespace(): + def get_namespace(user_pk=None, count_nodes=False): """ return full_types of the nodes """ - return get_node_namespace().get_description() + return get_node_namespace(user_pk=user_pk, count_nodes=count_nodes).get_description() def get_io_tree(self, uuid_pattern, tree_in_limit, tree_out_limit): # pylint: disable=too-many-statements,too-many-locals diff --git a/tests/restapi/conftest.py b/tests/restapi/conftest.py index d4ae82bae0..f710feaab4 100644 --- a/tests/restapi/conftest.py +++ b/tests/restapi/conftest.py @@ -55,3 +55,31 @@ def restrict_sqlalchemy_queuepool(aiida_profile): backend_manager = get_manager().get_backend_manager() backend_manager.reset_backend_environment() backend_manager.load_backend_environment(aiida_profile, pool_timeout=1, max_overflow=0) + + +@pytest.fixture +def populate_restapi_database(clear_database_before_test): + """Populates the database with a considerable set of nodes to test the restAPI""" + # pylint: disable=unused-argument + from aiida import orm + + struct_forcif = orm.StructureData().store() + orm.StructureData().store() + orm.StructureData().store() + + orm.Dict().store() + orm.Dict().store() + + orm.CifData(ase=struct_forcif.get_ase()).store() + + orm.KpointsData().store() + + orm.FolderData().store() + + orm.CalcFunctionNode().store() + orm.CalcJobNode().store() + orm.CalcJobNode().store() + + orm.WorkFunctionNode().store() + orm.WorkFunctionNode().store() + orm.WorkChainNode().store() diff --git a/tests/restapi/test_identifiers.py b/tests/restapi/test_identifiers.py index e66c4a872b..c7144c88b6 100644 --- a/tests/restapi/test_identifiers.py +++ b/tests/restapi/test_identifiers.py @@ -10,8 +10,8 @@ """Tests for the `aiida.restapi.common.identifiers` module.""" from threading import Thread -import pytest import requests +import pytest from aiida import orm from aiida.restapi.common.identifiers import get_full_type_filters, FULL_TYPE_CONCATENATOR, LIKE_OPERATOR_CHARACTER diff --git a/tests/restapi/test_routes.py b/tests/restapi/test_routes.py index 8a7c5a43d0..ca6eada052 100644 --- a/tests/restapi/test_routes.py +++ b/tests/restapi/test_routes.py @@ -1067,15 +1067,20 @@ def test_node_namespace(self): """ Test the rest api call to get list of available node namespace """ - url = f'{self.get_url_prefix()}/nodes/full_types' - with self.app.test_client() as client: - rv_obj = client.get(url) - response = json.loads(rv_obj.data) - expected_data_keys = ['path', 'namespace', 'subspaces', 'label', 'full_type'] - response_keys = response['data'].keys() - for dkay in expected_data_keys: - self.assertIn(dkay, response_keys) - RESTApiTestCase.compare_extra_response_data(self, 'nodes', url, response) + endpoint_datakeys = { + '/nodes/full_types': ['path', 'namespace', 'subspaces', 'label', 'full_type'], + '/nodes/full_types_count': ['path', 'namespace', 'subspaces', 'label', 'full_type', 'counter'], + } + + for endpoint_suffix, expected_data_keys in endpoint_datakeys.items(): + url = f'{self.get_url_prefix()}{endpoint_suffix}' + with self.app.test_client() as client: + rv_obj = client.get(url) + response = json.loads(rv_obj.data) + response_keys = response['data'].keys() + for dkay in expected_data_keys: + self.assertIn(dkay, response_keys) + RESTApiTestCase.compare_extra_response_data(self, 'nodes', url, response) def test_comments(self): """ diff --git a/tests/restapi/test_statistics.py b/tests/restapi/test_statistics.py new file mode 100644 index 0000000000..3d3e3700a5 --- /dev/null +++ b/tests/restapi/test_statistics.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Unit tests for REST API statistics.""" +from threading import Thread + +import requests +import pytest + + +def linearize_namespace(tree_namespace, linear_namespace=None): + """Linearize a branched namespace with full_type, count, and subspaces""" + if linear_namespace is None: + linear_namespace = {} + + full_type = tree_namespace['full_type'] + while full_type[-1] != '.': + full_type = full_type[0:-1] + + counter = tree_namespace['counter'] + subspaces = tree_namespace['subspaces'] + + linear_namespace[full_type] = counter + for subspace in subspaces: + linearize_namespace(subspace, linear_namespace) + + return linear_namespace + + +@pytest.mark.usefixtures('populate_restapi_database') +def test_count_consistency(restapi_server, server_url): + """ + Test the consistency in values between full_type_count and statistics + """ + server = restapi_server() + server_thread = Thread(target=server.serve_forever) + + try: + server_thread.start() + type_count_response = requests.get(f'{server_url}/nodes/full_types_count', timeout=10) + statistics_response = requests.get(f'{server_url}/nodes/statistics', timeout=10) + finally: + server.shutdown() + + type_count_dict = linearize_namespace(type_count_response.json()['data']) + statistics_dict = statistics_response.json()['data']['types'] + + for full_type, count in statistics_dict.items(): + if full_type in type_count_dict: + assert count == type_count_dict[full_type]