REST API: Add full_types_count as new entry point

This feature returns a namespace tree of the available node types in the
database (data node_types + process process_types) with the addition of
a count at each leaf / branch. It also has the option of doing so for a
single user, if the pk is provided as an option.

aiida/restapi/api.py

@@ -131,6 +131,7 @@ def __init__(self, app=None, **kwargs):
             '/nodes/projectable_properties/',
             '/nodes/statistics/',
             '/nodes/full_types/',
+            '/nodes/full_types_count/',
             '/nodes/download_formats/',
             '/nodes/page/',
             '/nodes/page/<int:page>/',

aiida/restapi/common/config.py

@@ -16,7 +16,7 @@
     'LIMIT_DEFAULT': 400,  # default records total
     'PERPAGE_DEFAULT': 20,  # default records per page
     'PREFIX': '/api/v4',  # prefix for all URLs
-    'VERSION': '4.0.1',
+    'VERSION': '4.1.0',
 }
 
 APP_CONFIG = {

aiida/restapi/common/identifiers.py

@@ -209,14 +209,15 @@ def __str__(self):
         import json
         return json.dumps(self.get_description(), sort_keys=True, indent=4)
 
-    def __init__(self, namespace, path=None, label=None, full_type=None, is_leaf=True):
+    def __init__(self, namespace, path=None, label=None, full_type=None, counter=None, is_leaf=True):
         """Construct a new node class namespace."""
-        # pylint: disable=super-init-not-called
+        # pylint: disable=super-init-not-called, too-many-arguments
         self._namespace = namespace
         self._path = path if path else namespace
         self._full_type = self._infer_full_type(full_type)
         self._subspaces = {}
         self._is_leaf = is_leaf
+        self._counter = counter
 
         try:
             self._label = label if label is not None else self.mapping_path_to_label[path]
@@ -293,7 +294,15 @@ def get_description(self):
         }
 
         for _, port in self._subspaces.items():
-            result['subspaces'].append(port.get_description())
+            subspace_result = port.get_description()
+            result['subspaces'].append(subspace_result)
+            if 'counter' in subspace_result:
+                if self._counter is None:
+                    self._counter = 0
+                self._counter = self._counter + subspace_result['counter']
+
+        if self._counter is not None:
+            result['counter'] = self._counter
 
         return result
 
@@ -352,15 +361,20 @@ def create_namespace(self, name, **kwargs):
         return self[port_name]
 
 
-def get_node_namespace():
+def get_node_namespace(user_pk=None, count_nodes=False):
     """Return the full namespace of all available nodes in the current database.
 
     :return: complete node `Namespace`
     """
+    # pylint: disable=too-many-branches
     from aiida import orm
     from aiida.plugins.entry_point import is_valid_entry_point_string, parse_entry_point_string
 
-    builder = orm.QueryBuilder().append(orm.Node, project=['node_type', 'process_type']).distinct()
+    filters = {}
+    if user_pk is not None:
+        filters['user_id'] = user_pk
+
+    builder = orm.QueryBuilder().append(orm.Node, filters=filters, project=['node_type', 'process_type']).distinct()
 
     # All None instances of process_type are turned into ''
     unique_types = {(node_type, process_type if process_type else '') for node_type, process_type in builder.all()}
@@ -371,6 +385,7 @@ def get_node_namespace():
     for node_type, process_type in unique_types:
 
         label = None
+        counter = None
         namespace = None
 
         if process_type:
@@ -393,12 +408,32 @@ def get_node_namespace():
             except IndexError:
                 continue
 
+        if count_nodes:
+            builder = orm.QueryBuilder()
+            concat_filters = [{'node_type': {'==': node_type}}]
+
+            if node_type.startswith('process.'):
+                if process_type:
+                    concat_filters.append({'process_type': {'==': process_type}})
+                else:
+                    concat_filters.append({'process_type': {'or': [{'==': ''}, {'==': None}]}})
+
+            if user_pk:
+                concat_filters.append({'user_id': {'==': user_pk}})
+
+            if len(concat_filters) == 1:
+                builder.append(orm.Node, filters=concat_filters[0])
+            else:
+                builder.append(orm.Node, filters={'and': concat_filters})
+
+            counter = builder.count()
+
         full_type = construct_full_type(node_type, process_type)
-        namespaces.append((namespace, label, full_type))
+        namespaces.append((namespace, label, full_type, counter))
 
     node_namespace = Namespace('node')
 
-    for namespace, label, full_type in sorted(namespaces, key=lambda x: x[0], reverse=False):
-        node_namespace.create_namespace(namespace, label=label, full_type=full_type)
+    for namespace, label, full_type, counter in sorted(namespaces, key=lambda x: x[0], reverse=False):
+        node_namespace.create_namespace(namespace, label=label, full_type=full_type, counter=counter)
 
     return node_namespace

aiida/restapi/common/utils.py

@@ -208,8 +208,8 @@ def parse_path(self, path_string, parse_pk_uuid=None):
             return (resource_type, page, node_id, query_type)
 
         if path[0] in [
-            'projectable_properties', 'statistics', 'full_types', 'download', 'download_formats', 'report', 'status',
-            'input_files', 'output_files'
+            'projectable_properties', 'statistics', 'full_types', 'full_types_count', 'download', 'download_formats',
+            'report', 'status', 'input_files', 'output_files'
         ]:
             query_type = path.pop(0)
             if path:

aiida/restapi/resources.py

@@ -264,15 +264,23 @@ def get(self, id=None, page=None):  # pylint: disable=redefined-builtin,invalid-
         elif query_type == 'statistics':
             headers = self.utils.build_headers(url=request.url, total_count=0)
             if filters:
-                usr = filters['user']['==']
+                user_pk = filters['user']['==']
             else:
-                usr = None
-            results = self.trans.get_statistics(usr)
+                user_pk = None
+            results = self.trans.get_statistics(user_pk)
 
         elif query_type == 'full_types':
             headers = self.utils.build_headers(url=request.url, total_count=0)
             results = self.trans.get_namespace()
 
+        elif query_type == 'full_types_count':
+            headers = self.utils.build_headers(url=request.url, total_count=0)
+            if filters:
+                user_pk = filters['user']['==']
+            else:
+                user_pk = None
+            results = self.trans.get_namespace(user_pk=user_pk, count_nodes=True)
+
         # TODO improve the performance of tree endpoint by getting the data from database faster
         # TODO add pagination for this endpoint (add default max limit)
         elif query_type == 'tree':

aiida/restapi/translator/nodes/node.py

@@ -577,12 +577,12 @@ def get_statistics(self, user_pk=None):
         return qmanager.get_creation_statistics(user_pk=user_pk)
 
     @staticmethod
-    def get_namespace():
+    def get_namespace(user_pk=None, count_nodes=False):
         """
         return full_types of the nodes
         """
 
-        return get_node_namespace().get_description()
+        return get_node_namespace(user_pk=user_pk, count_nodes=count_nodes).get_description()
 
     def get_io_tree(self, uuid_pattern, tree_in_limit, tree_out_limit):
         # pylint: disable=too-many-statements,too-many-locals

tests/restapi/conftest.py

@@ -55,3 +55,31 @@ def restrict_sqlalchemy_queuepool(aiida_profile):
     backend_manager = get_manager().get_backend_manager()
     backend_manager.reset_backend_environment()
     backend_manager.load_backend_environment(aiida_profile, pool_timeout=1, max_overflow=0)
+
+
+@pytest.fixture
+def populate_restapi_database(clear_database_before_test):
+    """Populates the database with a considerable set of nodes to test the restAPI"""
+    # pylint: disable=unused-argument
+    from aiida import orm
+
+    struct_forcif = orm.StructureData().store()
+    orm.StructureData().store()
+    orm.StructureData().store()
+
+    orm.Dict().store()
+    orm.Dict().store()
+
+    orm.CifData(ase=struct_forcif.get_ase()).store()
+
+    orm.KpointsData().store()
+
+    orm.FolderData().store()
+
+    orm.CalcFunctionNode().store()
+    orm.CalcJobNode().store()
+    orm.CalcJobNode().store()
+
+    orm.WorkFunctionNode().store()
+    orm.WorkFunctionNode().store()
+    orm.WorkChainNode().store()

tests/restapi/test_identifiers.py

@@ -10,8 +10,8 @@
 """Tests for the `aiida.restapi.common.identifiers` module."""
 from threading import Thread
 
-import pytest
 import requests
+import pytest
 
 from aiida import orm
 from aiida.restapi.common.identifiers import get_full_type_filters, FULL_TYPE_CONCATENATOR, LIKE_OPERATOR_CHARACTER

tests/restapi/test_routes.py

@@ -1067,15 +1067,20 @@ def test_node_namespace(self):
         """
         Test the rest api call to get list of available node namespace
         """
-        url = f'{self.get_url_prefix()}/nodes/full_types'
-        with self.app.test_client() as client:
-            rv_obj = client.get(url)
-            response = json.loads(rv_obj.data)
-            expected_data_keys = ['path', 'namespace', 'subspaces', 'label', 'full_type']
-            response_keys = response['data'].keys()
-            for dkay in expected_data_keys:
-                self.assertIn(dkay, response_keys)
-            RESTApiTestCase.compare_extra_response_data(self, 'nodes', url, response)
+        endpoint_datakeys = {
+            '/nodes/full_types': ['path', 'namespace', 'subspaces', 'label', 'full_type'],
+            '/nodes/full_types_count': ['path', 'namespace', 'subspaces', 'label', 'full_type', 'counter'],
+        }
+
+        for endpoint_suffix, expected_data_keys in endpoint_datakeys.items():
+            url = f'{self.get_url_prefix()}{endpoint_suffix}'
+            with self.app.test_client() as client:
+                rv_obj = client.get(url)
+                response = json.loads(rv_obj.data)
+                response_keys = response['data'].keys()
+                for dkay in expected_data_keys:
+                    self.assertIn(dkay, response_keys)
+                RESTApiTestCase.compare_extra_response_data(self, 'nodes', url, response)
 
     def test_comments(self):
         """

tests/restapi/test_statistics.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Unit tests for REST API statistics."""
+from threading import Thread
+
+import requests
+import pytest
+
+
+def linearize_namespace(tree_namespace, linear_namespace=None):
+    """Linearize a branched namespace with full_type, count, and subspaces"""
+    if linear_namespace is None:
+        linear_namespace = {}
+
+    full_type = tree_namespace['full_type']
+    while full_type[-1] != '.':
+        full_type = full_type[0:-1]
+
+    counter = tree_namespace['counter']
+    subspaces = tree_namespace['subspaces']
+
+    linear_namespace[full_type] = counter
+    for subspace in subspaces:
+        linearize_namespace(subspace, linear_namespace)
+
+    return linear_namespace
+
+
+@pytest.mark.usefixtures('populate_restapi_database')
+def test_count_consistency(restapi_server, server_url):
+    """
+    Test the consistency in values between full_type_count and statistics
+    """
+    server = restapi_server()
+    server_thread = Thread(target=server.serve_forever)
+
+    try:
+        server_thread.start()
+        type_count_response = requests.get(f'{server_url}/nodes/full_types_count', timeout=10)
+        statistics_response = requests.get(f'{server_url}/nodes/statistics', timeout=10)
+    finally:
+        server.shutdown()
+
+    type_count_dict = linearize_namespace(type_count_response.json()['data'])
+    statistics_dict = statistics_response.json()['data']['types']
+
+    for full_type, count in statistics_dict.items():
+        if full_type in type_count_dict:
+            assert count == type_count_dict[full_type]