From 7611de2e9dce1a1f435037e576e005d52be6e977 Mon Sep 17 00:00:00 2001 From: Ilias Koutsakis Date: Mon, 26 Apr 2021 12:29:46 +0200 Subject: [PATCH] schemas: add config fixture, update serializer * closes #2215 Signed-off-by: Ilias Koutsakis --- cap/modules/schemas/cli.py | 12 +- cap/modules/schemas/configs/config.json | 16 + .../schemas/configs/notifications.json | 573 ++++++++++++++++++ cap/modules/schemas/configs/repositories.json | 182 ++++++ cap/modules/schemas/configs/reviewable.json | 4 + cap/modules/schemas/utils.py | 38 ++ cap/modules/schemas/views.py | 13 +- tests/integration/test_schemas_views.py | 43 ++ 8 files changed, 879 insertions(+), 2 deletions(-) create mode 100644 cap/modules/schemas/configs/config.json create mode 100644 cap/modules/schemas/configs/notifications.json create mode 100644 cap/modules/schemas/configs/repositories.json create mode 100644 cap/modules/schemas/configs/reviewable.json diff --git a/cap/modules/schemas/cli.py b/cap/modules/schemas/cli.py index 9855668601..1245e279d1 100644 --- a/cap/modules/schemas/cli.py +++ b/cap/modules/schemas/cli.py @@ -42,7 +42,7 @@ from cap.modules.schemas.models import Schema from cap.modules.schemas.resolvers import resolve_schema_by_url,\ resolve_schema_by_name_and_version, schema_name_to_url -from cap.modules.schemas.utils import is_later_version +from cap.modules.schemas.utils import is_later_version, validate_schema_config DEPOSIT_REQUIRED_FIELDS = [ '_buckets', @@ -253,6 +253,16 @@ def add_schema_from_json(data, replace=None, force_version=None): allow_all = data.pop("allow_all", False) version = data['version'] name = data['name'] + config = data.get('config') + + if config: + errors = validate_schema_config(config) + if errors: + click.secho(errors, fg='red') + click.secho( + f'Configuration is invalid. ' + f'Aborting update for schema {name}.', fg='red') + return try: with db.session.begin_nested(): diff --git a/cap/modules/schemas/configs/config.json b/cap/modules/schemas/configs/config.json new file mode 100644 index 0000000000..0ef64a9641 --- /dev/null +++ b/cap/modules/schemas/configs/config.json @@ -0,0 +1,16 @@ +{ + "type": "object", + "title": "General Configuration", + "additionalProperties":false, + "properties": { + "notifications": { + "$ref": "/cap/modules/schemas/configs/configs/notifications.json" + }, + "reviewable": { + "$ref": "/cap/modules/schemas/configs/reviewable.json" + }, + "repositories": { + "$ref": "/cap/modules/schemas/configs/repositories.json" + } + } +} diff --git a/cap/modules/schemas/configs/notifications.json b/cap/modules/schemas/configs/notifications.json new file mode 100644 index 0000000000..d7b56bcdda --- /dev/null +++ b/cap/modules/schemas/configs/notifications.json @@ -0,0 +1,573 @@ +{ + "type": "object", + "title": "Notification Configuration", + "definitions": { + "ctx": { + "type": "array", + "title": "Context (ctx) Options", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Option", + "properties": { + "type": { + "type": "string", + "enum": ["path", "method"], + "title": "Context variable type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["path"]}, + "name": { + "type": "string", + "title": "Variable name" + }, + "path": { + "type": "string", + "title": "Variable Path" + } + }, + "required": ["type", "name", "path"] + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "type": "string", + "title": "Variable Method" + } + }, + "required": ["type", "method"] + }] + } + } + } + }, + "mails": { + "type": "object", + "title": "Mails", + "properties": { + "default": { + "type": "array", + "title": "Default", + "uniqueItems": true, + "items": { + "type":"string", + "title": "Mail" + } + }, + "formatted": { + "type": "array", + "title": "Formatted", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Mail", + "properties": { + "template": { + "type": "string", + "title": "Template" + }, + "ctx": { + "$ref": "#/definitions/ctx" + } + } + } + } + } + }, + "checks": { + "type": "array", + "title": "Checks", + "items": { + "type": "object", + "title": "Add Check", + "properties": { + "path": { + "type": "string", + "title": "Path" + }, + "if": { + "type": "string", + "title": "Condition" + }, + "value": { + "type": "string", + "title": "Value" + } + } + } + }, + "template_type": { + "oneOf": [{ + "properties": { + "template_type": {"enum": ["string"]}, + "template": { + "type": "string", + "title": "Template string" + } + } + },{ + "properties": { + "template_type": {"enum": ["path"]}, + "template_file": { + "type": "string", + "title": "Template path" + } + } + }] + } + }, + "properties": { + "actions": { + "type": "object", + "title": "Notification Actions", + "properties": { + "publish": { + "type": "array", + "title": "Publish Options", + "items": { + "type": "object", + "title": "Mail Config/Options", + "properties": { + "subject": { + "type": "object", + "title": "Mail Subject", + "properties": { + "template_type": { + "type": "string", + "enum": ["string", "path"], + "title": "Template type (string/path)" + }, + "ctx": { + "$ref": "#/definitions/ctx" + } + }, + "dependencies": { + "template_type": { + "$ref": "#/definitions/template_type" + } + } + }, + "message": { + "type": "object", + "title": "Mail Message", + "properties": { + "template_type": { + "type": "string", + "enum": ["string", "path"], + "title": "Template type (string/path)" + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "base_template": { + "type": "string", + "title": "Base Template" + }, + "plain": { + "type": "boolean", + "title": "Plain text or HTML" + } + }, + "dependencies": { + "template_type": { + "$ref": "#/definitions/template_type" + } + } + }, + "recipients": { + "type": "object", + "title": "Mail Recipients", + "properties": { + "bcc": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + }, + "cc": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + }, + "recipients": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + } + } + } + } + } + }, + "review": { + "type": "array", + "title": "Publish Options", + "items": { + "type": "object", + "title": "Mail Config/Options", + "properties": { + "subject": { + "type": "object", + "title": "Mail Subject", + "properties": { + "template_type": { + "type": "string", + "enum": ["string", "path"], + "title": "Template type (string/path)" + }, + "ctx": { + "$ref": "#/definitions/ctx" + } + }, + "dependencies": { + "template_type": { + "$ref": "#/definitions/template_type" + } + } + }, + "message": { + "type": "object", + "title": "Mail Message", + "properties": { + "template_type": { + "type": "string", + "enum": ["string", "path"], + "title": "Template type (string/path)" + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "base_template": { + "type": "string", + "title": "Base Template" + }, + "plain": { + "type": "boolean", + "title": "Plain text or HTML" + } + }, + "dependencies": { + "template_type": { + "$ref": "#/definitions/template_type" + } + } + }, + "recipients": { + "type": "object", + "title": "Mail Recipients", + "properties": { + "bcc": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + }, + "cc": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + }, + "recipients": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": true, + "items": { + "type": "object", + "title": "Add config", + "properties": { + "type": { + "type": "string", + "enum": ["default", "method", "condition"], + "title": "Config type" + } + }, + "dependencies": { + "type": { + "oneOf": [{ + "properties": { + "type": {"enum": ["default"]}, + "mails": { + "$ref": "#/definitions/mails" + } + } + }, { + "properties": { + "type": {"enum": ["method"]}, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": ["type", "method"] + }, { + "properties": { + "type": {"enum": ["condition"]}, + "op": { + "type": "string", + "enum": ["and", "or"], + "title": "Conditional operator" + }, + "checks": { + "$ref": "#/definitions/checks" + }, + "mails": { + "$ref": "#/definitions/mails" + } + }, + "required": ["type", "checks", "mails"] + }] + } + } + } + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/cap/modules/schemas/configs/repositories.json b/cap/modules/schemas/configs/repositories.json new file mode 100644 index 0000000000..f1a78e6717 --- /dev/null +++ b/cap/modules/schemas/configs/repositories.json @@ -0,0 +1,182 @@ +{ + "type": "object", + "title": "Repositories", + "properties": { + "github": { + "type": "object", + "title": "Github configuration", + "properties": { + "admin": { + "type": "string", + "title": "Admin key of the experiment" + }, + "org_name": { + "type": "string", + "title": "Organization Name" + }, + "private": { + "type": "boolean", + "private": "Private Repo" + }, + "license": { + "type": "string", + "title": "License" + }, + "repo_name": { + "type": "object", + "title": "Repository Name", + "properties": { + "template": { + "type": "string", + "title": "Repo name template path" + }, + "ctx": { + "type": "array", + "title": "Repo name context options", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Context Option", + "properties": { + "type": { + "type": "string", + "enum": ["path", "method"] + }, + "method": { + "type": "string", + "title": "The method that retrieves the value" + }, + "path": { + "type": "string", + "title": "The path that retrieves the value" + } + } + } + } + } + }, + "description": { + "type": "object", + "title": "Repository Description", + "properties": { + "template": { + "type": "string", + "title": "Repo description template path" + }, + "ctx": { + "type": "array", + "title": "Repo description context options", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Context Option", + "properties": { + "type": { + "type": "string", + "enum": ["path", "method"] + }, + "method": { + "type": "string", + "title": "The method that retrieves the value" + }, + "path": { + "type": "string", + "title": "The path that retrieves the value" + } + } + } + } + } + } + } + }, + "gitlab": { + "type": "object", + "title": "Gitlab configuration", + "properties": { + "admin": { + "type": "string", + "title": "Admin key of the experiment" + }, + "org_name": { + "type": "string", + "title": "Organization Name" + }, + "private": { + "type": "boolean", + "private": "Private Repo" + }, + "license": { + "type": "string", + "title": "License" + }, + "repo_name": { + "type": "object", + "title": "Repository Name", + "properties": { + "template": { + "type": "string", + "title": "Repo name template path" + }, + "ctx": { + "type": "array", + "title": "Repo name context options", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Context Option", + "properties": { + "type": { + "type": "string", + "enum": ["path", "method"] + }, + "method": { + "type": "string", + "title": "The method that retrieves the value" + }, + "path": { + "type": "string", + "title": "The path that retrieves the value" + } + } + } + } + } + }, + "description": { + "type": "object", + "title": "Repository Description", + "properties": { + "template": { + "type": "string", + "title": "Repo description template path" + }, + "ctx": { + "type": "array", + "title": "Repo description context options", + "uniqueItems": true, + "items": { + "type":"object", + "title": "Context Option", + "properties": { + "type": { + "type": "string", + "enum": ["path", "method"] + }, + "method": { + "type": "string", + "title": "The method that retrieves the value" + }, + "path": { + "type": "string", + "title": "The path that retrieves the value" + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/cap/modules/schemas/configs/reviewable.json b/cap/modules/schemas/configs/reviewable.json new file mode 100644 index 0000000000..f435c00f25 --- /dev/null +++ b/cap/modules/schemas/configs/reviewable.json @@ -0,0 +1,4 @@ +{ + "type": "boolean", + "title": "Reviewable Configuration" +} \ No newline at end of file diff --git a/cap/modules/schemas/utils.py b/cap/modules/schemas/utils.py index dfb11b992d..2d7e3f1e3c 100644 --- a/cap/modules/schemas/utils.py +++ b/cap/modules/schemas/utils.py @@ -23,9 +23,17 @@ # as an Intergovernmental Organization or submit itself to any jurisdiction. """Utils for Schemas module.""" +import os +import json import re +import pathlib from itertools import groupby +from jsonschema import Draft4Validator, RefResolver +from invenio_rest.errors import FieldError + +from cap.modules.records.errors import get_error_path + from .models import Schema from .permissions import ReadSchemaPermission @@ -104,3 +112,33 @@ def is_later_version(version1, version2): return False elif patch1 == patch2: return False + + +def validate_schema_config(config_data): + cwd = pathlib.Path(__file__).parent.absolute() + with open(cwd.joinpath('configs/config.json')) as json_: + schema = json.load(json_) + + schema_store = {} + schema_search_path = 'cap/modules/schemas/configs' + fnames = os.listdir(schema_search_path) + + for fname in fnames: + fpath = os.path.join(schema_search_path, fname) + if fpath.endswith(".json") and not fpath.endswith("config.json"): + with open(fpath, "r") as schema_fd: + _schema = json.load(schema_fd) + schema_store[f'file:///{fpath}'] = _schema + + resolver = RefResolver( + "file:///cap/modules/schemas/configs/config.json", + schema, schema_store + ) + validator = Draft4Validator(schema, resolver=resolver) + + errors = [ + FieldError(get_error_path(error), str(error.message)).res + for error in validator.iter_errors(config_data) + ] + + return errors diff --git a/cap/modules/schemas/views.py b/cap/modules/schemas/views.py index b1d409570b..a12ded2d30 100644 --- a/cap/modules/schemas/views.py +++ b/cap/modules/schemas/views.py @@ -36,7 +36,8 @@ from .models import Schema from .permissions import AdminSchemaPermission, ReadSchemaPermission from .serializers import schema_serializer, update_schema_serializer -from .utils import get_indexed_schemas_for_user, get_schemas_for_user +from .utils import get_indexed_schemas_for_user, get_schemas_for_user, \ + validate_schema_config blueprint = Blueprint( 'cap_schemas', @@ -83,6 +84,7 @@ def post(self): """Create new schema.""" data = request.get_json() + self._validate_config(data) serialized_data, errors = schema_serializer.load(data) if errors: @@ -110,6 +112,8 @@ def put(self, name, version): with AdminSchemaPermission(schema).require(403): data = request.get_json() + + self._validate_config(data) serialized_data, errors = update_schema_serializer.load( data, partial=True) @@ -134,6 +138,13 @@ def delete(self, name, version): return 'Schema deleted.', 204 + def _validate_config(self, data): + config = data.get('config') + if config: + errors = validate_schema_config(config) + if errors: + raise abort(400, errors) + schema_view_func = SchemaAPI.as_view('schemas') diff --git a/tests/integration/test_schemas_views.py b/tests/integration/test_schemas_views.py index 06d06d938e..c39f33c61a 100644 --- a/tests/integration/test_schemas_views.py +++ b/tests/integration/test_schemas_views.py @@ -826,6 +826,49 @@ def test_put_when_not_an_schema_owner_returns_403( assert resp.status_code == 403 +def test_put_schema_with_valid_config(client, db, auth_headers_for_user, users, json_headers): + owner = users['cms_user'] + schema = dict( + name='new-schema', + version='1.0.0', + deposit_schema={'title': 'deposit_schema'}, + config={'reviewable': True}, + is_indexed=True, + use_deposit_as_record=True, + ) + + resp = client.post( + '/jsonschemas/', + data=json.dumps(schema), + headers=json_headers + auth_headers_for_user(owner), + ) + + assert resp.status_code == 200 + + +def test_put_schema_with_invalid_config(client, db, auth_headers_for_user, users, json_headers): + owner = users['cms_user'] + schema = dict( + name='new-schema', + version='1.0.0', + deposit_schema={'title': 'deposit_schema'}, + config={'reviewable': 123}, # INVALID, SHOULD FAIL + is_indexed=True, + use_deposit_as_record=True, + ) + + resp = client.post( + '/jsonschemas/', + data=json.dumps(schema), + headers=json_headers + auth_headers_for_user(owner), + ) + + assert resp.status_code == 400 + assert resp.json['message'] == [{ + 'field': ['reviewable'], + 'message': "123 is not of type 'boolean'" + }] + ##################################### # api/jsonschemas/{id}/{version} [DELETE] #####################################