diff --git a/cap/config.py b/cap/config.py index 7b54ecd387..64817deded 100644 --- a/cap/config.py +++ b/cap/config.py @@ -106,6 +106,8 @@ def _(x): CAP_SEND_MAIL = True MAIL_DEFAULT_SENDER = SUPPORT_EMAIL +CAP_MAIL_HOST_URL = "https://analysispreservation.cern.ch" +CAP_MAIL_HOST_API_URL = "https://analysispreservation.cern.ch/api" # For Flask-mail variables, the defaults are used, found here: # https://pythonhosted.org/Flask-Mail/#configuring-flask-mail diff --git a/cap/modules/fixtures/schemas/cms-questionnaire.json b/cap/modules/fixtures/schemas/cms-questionnaire.json index 5ac804d541..8b7d485b3b 100644 --- a/cap/modules/fixtures/schemas/cms-questionnaire.json +++ b/cap/modules/fixtures/schemas/cms-questionnaire.json @@ -4,10 +4,244 @@ "fullname":"CMS Statistics Questionnaire", "experiment":"CMS", "is_indexed":true, - "use_deposit_as_record":true, "config": { - "reviewable": true + "reviewable": true, + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + "bcc": [{ + "method": "get_cms_stat_recipients" + }, { + "method": "get_owner" + }, { + "method": "get_submitter" + }, { + "checks": [{ + "path": "parton_distribution_functions", + "condition": "exists" + }], + "mails": { + "default": ["pdf-forum-placeholder@cern.ch"] + } + }, { + "checks": [{ + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes" + }], + "mails": { + "default": ["cms-conveners-placeholder@cern.ch"] + } + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [{ + "checks": [{ + "path": "analysis_context.cadi_id", + "condition": "exists" + }], + "mails": { + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }] + } + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [{ + "op": "or", + "checks": [{ + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes" + }], + "mails": { + "default": ["cms-conveners-jira-placeholder@cern.ch"] + } + }] + } + } + ], + "review": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_review.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "working_url" + }, { + "method": "creator_email" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + "bcc": [{ + "method": "get_owner" + }, { + "method": "get_submitter" + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_review_plain.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "working_url" + }, { + "method": "creator_email" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [ + { + "checks": [{ + "path": "analysis_context.cadi_id", + "condition": "exists" + }, { + "path": "parton_distribution_functions", + "condition": "is_egroup_member", + "value": "comittee-mail" + }], + "mails": { + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }] + } + } + ] + } + } + ] + } + } }, + "use_deposit_as_record":true, "deposit_schema":{ "additionalProperties":false, "$schema":"http://json-schema.org/draft-04/schema#", diff --git a/cap/modules/fixtures/schemas/cms-questionnaire_v2.json b/cap/modules/fixtures/schemas/cms-questionnaire_v2.json index 021bb14cf1..31da966593 100644 --- a/cap/modules/fixtures/schemas/cms-questionnaire_v2.json +++ b/cap/modules/fixtures/schemas/cms-questionnaire_v2.json @@ -4,10 +4,287 @@ "fullname":"CMS Statistics Questionnaire", "experiment":"CMS", "is_indexed":true, - "use_deposit_as_record":true, "config": { - "reviewable": true + "reviewable": true, + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + "bcc": [{ + "method": "get_cms_stat_recipients" + }, { + "method": "get_owner" + }, { + "method": "get_submitter" + }, { + "checks": [{ + "path": "parton_distribution_functions", + "condition": "exists" + }], + "mails": { + "default": ["pdf-forum-placeholder@cern.ch"] + } + }, { + "op": "or", + "checks": [ + { + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes" + }, { + "path": "ml_app_use", + "condition": "exists" + }, { + "path": "ml_survey.options", + "condition": "equals", + "value": "Yes" + }, { + "op": "and", + "checks": [ + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "exists" + }, { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "is_not_in", + "value": "No" + } + ] + } + ], + "mails": { + "default": ["cms-conveners-placeholder@cern.ch"] + } + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [{ + "checks": [{ + "path": "analysis_context.cadi_id", + "condition": "exists" + }], + "mails": { + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }] + } + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id if cadi_id else \"\" }} {{ published_id }} - {{ \"New Version of Published Analysis\" if revision > 0 else \"New Published Analysis\" }} | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "published_url" + }, { + "method": "cms_stats_committee_by_pag" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [{ + "op": "or", + "checks": [ + { + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes" + }, { + "path": "ml_app_use", + "condition": "exists" + }, { + "path": "ml_survey.options", + "condition": "equals", + "value": "Yes" + }, { + "op": "and", + "checks": [ + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "exists" + }, { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "is_not_in", + "value": "No" + } + ] + } + ], + "mails": { + "default": ["cms-conveners-jira-placeholder@cern.ch"] + } + }] + } + } + ], + "review": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_review.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "working_url" + }, { + "method": "creator_email" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + "bcc": [{ + "method": "get_owner" + }, { + "method": "get_submitter" + }] + } + }, { + "subject": { + "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_review_plain.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + },{ + "method": "working_url" + }, { + "method": "creator_email" + }, { + "method": "submitter_email" + }], + "base_template": "mail/analysis_plain_text.html", + "plain": true + }, + "recipients": { + "bcc": [ + { + "checks": [{ + "path": "analysis_context.cadi_id", + "condition": "exists" + }, { + "path": "parton_distribution_functions", + "condition": "is_egroup_member", + "value": "comittee-mail" + }], + "mails": { + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }] + } + } + ] + } + } + ] + } + } }, + "use_deposit_as_record":true, "deposit_schema":{ "additionalProperties":false, "$schema":"http://json-schema.org/draft-04/schema#", diff --git a/cap/modules/mail/attributes.py b/cap/modules/mail/attributes.py new file mode 100644 index 0000000000..b99fc3fd24 --- /dev/null +++ b/cap/modules/mail/attributes.py @@ -0,0 +1,312 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +from flask import current_app + +from .conditions import CONDITION_METHODS +from .custom import body as custom_body +from .custom import recipients as custom_recipients +from .custom import subject as custom_subjects +from .utils import ( + EMAIL_REGEX, + get_config_default, + populate_template_from_ctx, + update_mail_list, +) + + +def check_condition(record, condition, default_ctx={}): + """ + Iterates conditions and return true/false. + + For each condition we have a group of checks to perform, which should + return True/False + + Each condition has the following: + - `checks`: the checks to be evaluated + - `op`: the operation to be used for all the checks (and/or) + - `mails`: the mail list + + Each check has their own attributes: + - `condition`: the type of check (e.g. exists, is_in, etc) + - `value`: the result + - `path`: the path that gets the field to be evaluated + + An example of conditions: + { + "checks": [{ + "path": "parton_distribution_functions", + "condition": "exists", + "value": true + }], + "mails": { + "default": ["pdf-forum-placeholder@cern.ch"] + } + } + + Nested checks are also allowed. + """ + # for each condition we have a group of checks to perform + # all of the checks should give a True/False result + operator = condition.get("op", "and") + checks = condition.get("checks", []) + check_results = [] + + for check in checks: + # get the method to apply, and use it on the required path/value + # for malformed conditions, we assume False + if check.get("checks"): + check_results.append(check_condition(record, check, default_ctx)) + elif check.get("condition"): + try: + method = CONDITION_METHODS[check["condition"]] + path = check.get("path") + value = check.get("value", True) + # TODO: maybe update with kwargs + check_results.append( + method(record, path, value, default_ctx=default_ctx) + ) + except (KeyError, Exception): + check_results.append(False) + else: + check_results.append(False) + + # due to all([]) == True, we make sure the list is not empty + if check_results: + # we check the validity of the condition depending on the operator: + # - if 'and', then we need all the items to be true + # - if 'or' we need at least 1 item to be true + if operator == "and": + return all(check_results) + elif operator == "or": + return any(check_results) + return False + + +def get_recipients_from_config(record, config, default_ctx={}): + """ + Get recipients from configuration. + + The `recipients` field differentiates 3 categories: + - recipients + - cc + - bcc + All 3 can be used to send a mail, and have their own mails and rules about + how they will be added. + + The rules are in 3 categories: + - `default`: the mails in the list will be added + - `method`: a method that returns a list of mail (for complicated options) + - `conditions`: mails will be added if a certain condition is true + + An example config of recipients: + "recipients": { + "bcc": [ + { + "method": "get_owner" + }, { + "mails": { + "default": ["some-recipient-placeholder@cern.ch"], + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", # noqa + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }] + } + }, { + "checks": {...} + } + ] + } + """ + if not config: + return [] + + mails = [] + for item in config: + if isinstance(item, str): + mails.append(item) + elif isinstance(item, dict): + if item.get("checks"): + if not check_condition(record, item, default_ctx=default_ctx): + continue + + if item.get("method"): + if isinstance(item.get("method"), list): + methods = item.get("method") + else: + methods = [item.get("method")] + + for _method in methods: + try: + method = getattr(custom_recipients, _method) + result = method( + record, config=item, default_ctx=default_ctx + ) + if isinstance(result, list): + mails += result + else: + mails += [result] + except AttributeError as exc: + current_app.logger.error( + f"Recipients function not found. Skipping.\n" + f"Error: {exc.args[0]}" + ) + + if item.get("mails"): + mail_config = item.get("mails", {}) + update_mail_list( + record, mail_config, mails, default_ctx=default_ctx + ) + else: + continue + + # remove duplicates and possible empty values + _mails = [] + for mail in set(mails): + if isinstance(mail, str) and EMAIL_REGEX.match(mail): + _mails.append(mail) + + return _mails + + +def generate_recipients(record, config, default_ctx={}): + """ + Recipients generator for notification action. + + Using the `get_recipients_from_config` function, it retrieves and returns + 3 possible lists of mails: recipients, bcc, cc. + """ + re_config = config.get("recipients") + if not re_config: + return [], [], [] + + recipients = get_recipients_from_config( + record, re_config.get("recipients"), default_ctx=default_ctx + ) + cc = get_recipients_from_config( + record, re_config.get("cc"), default_ctx=default_ctx + ) + bcc = get_recipients_from_config( + record, re_config.get("bcc"), default_ctx=default_ctx + ) + + return recipients, cc, bcc + + +def generate_body(record, config, default_ctx={}): + """ + Body generator for notification action. + + It requires a template and a context (dict of vars-values), to populate it. + If no template is found, the default one will be used. + + This function will retrieve the template and context for the body (message) + as well as the base template and `plain` parameter. + + Example of message config: + "body": { + "body_template_file": "mail/message/questionnaire_message_published.html", + "ctx": [{ + "name": "title", + "path": "general_title" + }, { + "method": "submitter_email" + }], + "base_template_file": "mail/analysis_plain_text.html", + "plain": false + } + + In case of `method`, then the message will be retrieved from the result of + the method. It's implementation should always be in the mail.custom.messages.py file # noqa + """ + body_config = config.get("body", {}) + + body = populate_template_from_ctx( + record, + body_config, + module=custom_body, + type="body", + default_ctx=default_ctx, + ) + + base = "mail/base.html" + + if body_config.get("plain"): + base = "mail/base_plain.html" + + if body_config.get("base_template"): + base = body_config.get("base_template") + + return body, base + + +def generate_subject(record, config, default_ctx={}): + """ + Subject generator for notification action. + + It requires a template and a context (dict of vars-values), to populate it. + If no template is found, the default one will be used. + + Example of subject config: + "subject": { + "template": "Subject with {{ title }} and id {{ published_id }}", + "ctx": [{ + "name": "title", + "path": "general_title" + }, { + "method": "published_id" + }] + } + + In case of `method`, then the subject will be retrieved from the result of + the method. It's implementation should always be in the mail.custom.subjects.py file # noqa + """ + subj_config = config.get("subject") + action = default_ctx.get("action", "") + + if not subj_config: + return get_config_default(action, "subject") + + func = subj_config.get("method") + if func: + try: + custom_subject_func = getattr(custom_subjects, func) + subject = custom_subject_func(record, config) + return subject + except AttributeError as exc: + current_app.logger.error( + f"Subject function not found. Providing default subject.\n" + f"Error: {exc.args[0]}" + ) + return get_config_default(action, "subject") + + return populate_template_from_ctx( + record, + subj_config, + module=custom_subjects, + default_ctx=default_ctx, + type="subject", + ) diff --git a/cap/modules/mail/conditions.py b/cap/modules/mail/conditions.py new file mode 100644 index 0000000000..201c12ebc6 --- /dev/null +++ b/cap/modules/mail/conditions.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2016 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +from flask import current_app +from flask_principal import RoleNeed +from invenio_access.permissions import Permission +from invenio_oauthclient.models import RemoteAccount + +from .utils import path_value_equals + + +def equals(record, path, value, **kwargs): + data = path_value_equals(path, record) + return True if data and data == value else False + + +def not_equals(record, path, value, **kwargs): + data = path_value_equals(path, record) + return True if data and data != value else False + + +def exists(record, path, value, **kwargs): + data = path_value_equals(path, record) + return True if data else False + + +def is_in(record, path, value, **kwargs): + data = path_value_equals(path, record) + return True if data and value in data else False + + +def is_not_in(record, path, value, **kwargs): + data = path_value_equals(path, record) + return True if data and value not in data else False + + +def is_egroup_member(record, path, value, **kwargs): + submitter_id = kwargs.get('default_ctx', {}).get('submitter_id') + groups = get_cern_extra_data_egroups(submitter_id) + + return True if value in groups else False + + +CONDITION_METHODS = { + # path/metadata conditions + 'equals': equals, + 'not_equals': not_equals, + 'exists': exists, + 'is_in': is_in, + 'is_not_in': is_not_in, + # mail/permission conditions + 'is_egroup_member': is_egroup_member, +} + + +def get_cern_extra_data_egroups(user_id): + client_id = current_app.config['CERN_APP_CREDENTIALS']['consumer_key'] + account = RemoteAccount.get( + user_id=user_id, + client_id=client_id, + ) + groups = [] + + if account: + groups = account.extra_data.get('groups', []) + + return groups diff --git a/cap/modules/mail/custom/__init__.py b/cap/modules/mail/custom/__init__.py new file mode 100644 index 0000000000..c50096b814 --- /dev/null +++ b/cap/modules/mail/custom/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +"""Mail custom methods package.""" + +from .body import * +from .recipients import * +from .subject import * diff --git a/cap/modules/mail/custom/body.py b/cap/modules/mail/custom/body.py new file mode 100644 index 0000000000..6f13885a01 --- /dev/null +++ b/cap/modules/mail/custom/body.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +from flask import current_app +from invenio_accounts.models import User + + +def draft_url(record, **kwargs): + """Get the draft html url of the analysis.""" + base_url = current_app.config['CAP_MAIL_HOST_URL'] + return f'{base_url}/drafts/{record["_deposit"]["id"]}' + + +def published_url(record, **kwargs): + """Get the published html url of the analysis.""" + base_url = current_app.config['CAP_MAIL_HOST_URL'] + if record.get("control_number"): + return f'{base_url}/published/{record["control_number"]}' + return None + + +def working_url(record, **kwargs): + """Get the working html url of the analysis.""" + status = record.get("_deposit", {}).get("status") + + if status == "draft": + return draft_url(record) + elif status == "published": + return published_url(record) + else: + return None + + +def submitter_email(record, **kwargs): + """Returns the submitter of the analysis, aka the current user.""" + submitter_user_id = kwargs.get('default_ctx', {}).get('submitter_id') + submitter_user = User.query.filter_by(id=submitter_user_id).one() + + return submitter_user.email + + +def creator_email(record, **kwargs): + """Returns the owner of the analysis.""" + owner_list = record.get("_deposit", {}).get("owners") + if owner_list: + return User.query.filter_by(id=owner_list[0]).one().email + return None + + +def cms_stats_committee_by_pag(record, **kwargs): + """Retrieve reviewer parameters according to the working group.""" + committee_pags = current_app.config.get("CMS_STATS_COMMITEE_AND_PAGS") + working_group = record.get("analysis_context", {}).get("wg") + + if working_group and committee_pags: + return committee_pags.get(working_group, {}).get("params", {}) + return {} diff --git a/cap/modules/mail/custom/recipients.py b/cap/modules/mail/custom/recipients.py new file mode 100644 index 0000000000..731be6ebff --- /dev/null +++ b/cap/modules/mail/custom/recipients.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +from flask import current_app +from invenio_accounts.models import User + + +def get_submitter(record, **kwargs): + """Returns the submitter of the analysis, aka the current user.""" + submitter_user_id = kwargs.get('default_ctx', {}).get('submitter_id') + submitter_user = User.query.filter_by(id=submitter_user_id).one() + + return [submitter_user.email] if submitter_user else [] + + +def get_owner(record, **kwargs): + """Returns the owner of the analysis.""" + owner_list = record.get('_deposit', {}).get('owners') + if owner_list: + return [User.query.filter_by(id=owner_list[0]).one().email] + return [] + + +def get_cms_stat_recipients(record, **kwargs): + """Adds PAGS committee data from JSON file.""" + committee_pags = current_app.config.get("CMS_STATS_COMMITEE_AND_PAGS") + working_group = record.get('analysis_context', {}).get('wg') + if working_group and committee_pags: + return committee_pags.get(working_group, {}).get("contacts", []) + return [] diff --git a/cap/modules/mail/custom/subject.py b/cap/modules/mail/custom/subject.py new file mode 100644 index 0000000000..664653a620 --- /dev/null +++ b/cap/modules/mail/custom/subject.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + + +def published_id(record, **kwargs): + try: + return record.get('_deposit', {}).get('pid', {}).get('value') + except: + return None + + +def draft_id(record, **kwargs): + try: + return record.get('_deposit', {}).get('id') + except: + return None + + +def revision(deposit, **kwargs): + try: + _, record = deposit.fetch_published() + return record.revision_id + except KeyError: + return None + + +def draft_revision(deposit, **kwargs): + try: + return deposit.revision_id + except KeyError: + return None diff --git a/cap/modules/mail/receivers.py b/cap/modules/mail/receivers.py index 67e53a3ebf..a1451c7726 100644 --- a/cap/modules/mail/receivers.py +++ b/cap/modules/mail/receivers.py @@ -21,40 +21,111 @@ # In applying this license, CERN does not # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. -from flask import request +from celery import shared_task +from flask import current_app +from flask_login import current_user -from .utils import NOTIFICATION_RECEPIENT, generate_notification_attrs, \ - send_mail_on_review, send_mail_on_publish, create_analysis_url +from cap.modules.deposit.api import CAPDeposit + +from .attributes import generate_body, generate_recipients, generate_subject +from .custom.body import working_url +from .custom.subject import draft_id, published_id, revision +from .tasks import create_and_send +from .utils import UnsuccessfulMail, generate_ctx, is_review_request def post_action_notifications(sender, action=None, pid=None, deposit=None): - """Method to run after a deposit action .""" - schema = deposit.get("$schema") - recipients_config = NOTIFICATION_RECEPIENT.get(schema, {}).get(action) - host_url = request.host_url - - if recipients_config: - subject, message, recipients = generate_notification_attrs( - deposit, host_url, recipients_config) - - if recipients: - if action == "publish": - recid, record = deposit.fetch_published() - - send_mail_on_publish( - recid.pid_value, - record.revision_id, - host_url, - recipients, - message, - subject_prefix=subject) - - if action == "review": - analysis_url = create_analysis_url(deposit) - - send_mail_on_review( - analysis_url, - host_url, - recipients, - message, - subject_prefix=subject) + """ + Notification through mail, after specified deposit actions. + + The procedure followed to get the mail attrs will be described here: + + - Get the config for the action that triggered the receiver. + - Through the configuration, retrieve the recipients, subject, body, and + base template, and render them when needed. + - Create the message and mail contexts (attributes), and pass them to + the `create_and_send` task. + """ + if not is_review_request(): + return + + action_notifications_config = ( + deposit.schema.config.get('notifications', {}) + .get('actions', {}) + .get(action, []) + ) + current_user_id = current_user.id + if action_notifications_config: + send_deposit_notifications(str(deposit.id), current_user_id, action) + + +@shared_task(ignore_result=True) +def send_deposit_notifications(record_uuid, user_id, action): + """ + Notification through mail, after specified deposit actions. + + The procedure followed to get the mail attrs will be described here: + + - Get the config for the action that triggered the receiver. + - Through the configuration, retrieve the recipients, subject, body, and + base template, and render them when needed. + - Create the message and mail contexts (attributes), and pass them to + the `create_and_send` task. + """ + deposit = CAPDeposit.get_record(record_uuid) + + mail_sender = current_app.config.get('MAIL_DEFAULT_SENDER') + if not mail_sender: + current_app.logger.info('Mail Error: Sender not found.') + return + + action_configs = ( + deposit.schema.config.get('notifications', {}) + .get('actions', {}) + .get(action, []) + ) + + # retrieve the most common Deposit/Record attributes, used in messages + # try: + msg_ctx = { + 'action': action, + 'base_url': current_app.config.get('CAP_MAIL_HOST_URL', ''), + 'base_api_url': current_app.config.get('CAP_MAIL_HOST_API_URL', ''), + 'submitter_id': user_id, + 'published_id': published_id(deposit), + 'draft_id': draft_id(deposit), + 'revision': revision(deposit), + 'working_url': working_url(deposit), + } + + for config in action_configs: + ctx_config = config.get("ctx", []) + default_ctx = generate_ctx(ctx_config, record=deposit) + _ctx = {**default_ctx, **msg_ctx} + recipients, cc, bcc = generate_recipients( + deposit, config, default_ctx=_ctx + ) + if not any([recipients, cc, bcc]): + continue + + try: + subject = generate_subject(deposit, config, default_ctx=_ctx) + body, base = generate_body(deposit, config, default_ctx=_ctx) + except UnsuccessfulMail as err: + current_app.logger.error( + f"UnsuccessfulMail | Rec_id:{err.rec_uuid} - {err.msg}" + ) + continue + plain = config.get('body', {}).get('plain') + + mail_ctx = { + 'sender': mail_sender, + 'subject': subject, + 'recipients': recipients, + 'cc': cc, + 'bcc': bcc, + } + + _ctx.update({'mail_body': body}) + + create_and_send(base, _ctx, mail_ctx, plain=plain) diff --git a/cap/modules/mail/tasks.py b/cap/modules/mail/tasks.py index 58b7f38c92..81db3d93bf 100644 --- a/cap/modules/mail/tasks.py +++ b/cap/modules/mail/tasks.py @@ -22,39 +22,47 @@ # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. -from celery import shared_task from flask import current_app from invenio_mail.api import TemplatedMessage +from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError -@shared_task(autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 3, - 'countdown': 10}) -def create_and_send(template, ctx, subject, recipients, - sender=None, type=None): - if not current_app.config['CAP_SEND_MAIL']: +def create_and_send(template, ctx, mail_ctx, plain=False): + """Creates the mail using the invenio-mail template, and sends it.""" + if not current_app.config["CAP_SEND_MAIL"]: + current_app.logger.info("Mail Error: Notifications disabled.") return - sender = sender or current_app.config.get('MAIL_DEFAULT_SENDER') - try: - assert recipients + if not any([mail_ctx["recipients"], mail_ctx["bcc"], mail_ctx["cc"]]): + current_app.logger.error( + f"Mail Error for analysis with the following information: {ctx}:\n" + f"Empty recipient list." + ) + return - if type == "plain": - msg = TemplatedMessage(template_body=template, - ctx=ctx, - **dict(sender=sender, - recipients=recipients, - subject=subject)) - else: - msg = TemplatedMessage(template_html=template, - ctx=ctx, - **dict(sender=sender, - recipients=recipients, - subject=subject)) - current_app.extensions['mail'].send(msg) + try: + msg = ( + TemplatedMessage(template_body=template, ctx=ctx, **mail_ctx) + if plain + else TemplatedMessage(template_html=template, ctx=ctx, **mail_ctx) + ) - except AssertionError: - current_app.logger.error( - f'Mail Error from {sender} with subject: {subject}.\n' - f'Empty recipient list.') - raise AssertionError + current_app.extensions["mail"].send(msg) + except TemplateNotFound as ex: + _msg = ( + f"Mail::create_and_send - Template {ex.name} not found." + f" Notification procedure aborted." + ) + current_app.logger.error(_msg) + except TemplateSyntaxError as ex: + _msg = ( + f"Mail::create_and_send - Template error: {ex.message}." + f" Notification procedure aborted." + ) + current_app.logger.error(_msg) + except TypeError: + _msg = ( + "Mail::create_and_send - Context for template is empty." + "Notification procedure aborted." + ) + current_app.logger.error(_msg) diff --git a/cap/modules/mail/templates/mail/analysis_published_new.html b/cap/modules/mail/templates/mail/analysis_published_new.html deleted file mode 100644 index 448085c6a3..0000000000 --- a/cap/modules/mail/templates/mail/analysis_published_new.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "mail/base.html" %} - -{% block paragraph %} -

A new document has been published, with id {{ recid }}.

-

You can check it here.

- - {% if message %} -
-

{{message}}

- {% endif %} -{% endblock %} - -{% block button_url %}{{ url + "published/" + recid }}{% endblock %} - -{% block button_text %}Check document{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/analysis_published_revision.html b/cap/modules/mail/templates/mail/analysis_published_revision.html deleted file mode 100644 index beb2b41ab2..0000000000 --- a/cap/modules/mail/templates/mail/analysis_published_revision.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "mail/base.html" %} - -{% block paragraph %} -

An newer version of document {{ recid }} has been published.

-

You can check it here.

- - {% if message %} -
-

{{message}}

- {% endif %} -{% endblock %} - -{% block button_url %}{{ url + "published/" + recid }}{% endblock %} - -{% block button_text %}Check new version{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/analysis_review.html b/cap/modules/mail/templates/mail/analysis_review.html deleted file mode 100644 index 76821ad869..0000000000 --- a/cap/modules/mail/templates/mail/analysis_review.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "mail/base.html" %} - -{% block paragraph %} -

A document has been reviewed.

-

You can check it here.

- - {% if message %} -
-

{{message}}

- {% endif %} -{% endblock %} - -{% block button_url %}{{ url + analysis_url }}{% endblock %} - -{% block button_text %}Check review{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/base.html b/cap/modules/mail/templates/mail/base.html index 9cd421e852..c8f8d3f385 100644 --- a/cap/modules/mail/templates/mail/base.html +++ b/cap/modules/mail/templates/mail/base.html @@ -1,357 +1,26 @@ - + - - - - - - - - - - - - - - - +{% include "mail/partials/head.html" %} - - - - - - - - - + + + + + + + - + + \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/base_plain.html b/cap/modules/mail/templates/mail/base_plain.html new file mode 100644 index 0000000000..8214073cb4 --- /dev/null +++ b/cap/modules/mail/templates/mail/base_plain.html @@ -0,0 +1 @@ +{{ mail_body | safe }} diff --git a/cap/modules/mail/templates/mail/analysis_plain_text.html b/cap/modules/mail/templates/mail/body/analysis_plain_text.html similarity index 100% rename from cap/modules/mail/templates/mail/analysis_plain_text.html rename to cap/modules/mail/templates/mail/body/analysis_plain_text.html diff --git a/cap/modules/mail/templates/mail/body/analysis_published.html b/cap/modules/mail/templates/mail/body/analysis_published.html new file mode 100644 index 0000000000..bdaf4bb02c --- /dev/null +++ b/cap/modules/mail/templates/mail/body/analysis_published.html @@ -0,0 +1,21 @@ +{% extends "mail/body/body_with_buttons.html" %} + +{% block content %} + {% if revision and revision > 0 %} +

An newer version of document {{ published_id }} has been published.

+ {% else %} +

A new document has been published, with id {{ published_id }}.

+ {% endif %} +

You can check it here.

+ + {% if message %} +
+

{{ message | safe }}

+ {% endif %} +{% endblock %} + +{% block button_url %}{{ working_url }}{% endblock %} + +{% block button_text %} + {{ "Check new version" if revision and revision > 0 else "Check document"}} +{% endblock %} diff --git a/cap/modules/mail/templates/mail/body/analysis_published_plain.html b/cap/modules/mail/templates/mail/body/analysis_published_plain.html new file mode 100644 index 0000000000..956bb1d909 --- /dev/null +++ b/cap/modules/mail/templates/mail/body/analysis_published_plain.html @@ -0,0 +1,16 @@ +{% block content %} +{% if revision and revision > 0 %} +An newer version of document {{ published_id }} has been published. +You can check it here <{{ working_url }}>. +{% else %} +A new document has been published, with id {{ published_id }}. +You can check it here <{{ working_url }}>. +{% endif %} + +{% if message %} +{{ message | safe }} +{% endif %} +{% endblock %} + +{{ "Check new version" if revision and revision > 0 else "Check document"}} <{{ working_url }}> + diff --git a/cap/modules/mail/templates/mail/body/analysis_review.html b/cap/modules/mail/templates/mail/body/analysis_review.html new file mode 100644 index 0000000000..a4e5d551ec --- /dev/null +++ b/cap/modules/mail/templates/mail/body/analysis_review.html @@ -0,0 +1,15 @@ +{% extends "mail/body/body_with_buttons.html" %} + +{% block content %} +

A document has been reviewed.

+

You can check it here.

+ + {% if message %} +
+

{{ message | safe }}

+ {% endif %} +{% endblock %} + +{% block button_url %}{{ working_url }}{% endblock %} + +{% block button_text %}Check review{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/body/body_with_buttons.html b/cap/modules/mail/templates/mail/body/body_with_buttons.html new file mode 100644 index 0000000000..baa4e82a56 --- /dev/null +++ b/cap/modules/mail/templates/mail/body/body_with_buttons.html @@ -0,0 +1,67 @@ +
+
+
+ + +
+
+ +
+ + +
+
+ +

{% block content %}{% endblock %}

+ +
+
+ + +
+ +
+
+ + +
+
+
+
+
+
+ + +
+
+ + + +
+
+ + +
+
+
diff --git a/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published.html b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published.html new file mode 100644 index 0000000000..a2d851739e --- /dev/null +++ b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published.html @@ -0,0 +1,16 @@ +{% extends "mail/body/analysis_published.html" %} +{% block content %} +{{super()}} +
+

+{% if cadi_id %} +CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode={{ cadi_id }}
+{% endif %} +Title: {{ title }}
+Questionnaire URL: {{ published_url }}
+{% if cms_stats_committee_by_pag %} +Statistics Committee assignee: {{ cms_stats_committee_by_pag.get('primary', '-') }} (primary), {{ cms_stats_committee_by_pag.get('secondary', '-') }} (secondary).
+{% endif %} + +Submitted by {{ submitter_email }}.

+{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published_plain.html b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published_plain.html new file mode 100644 index 0000000000..6f28c9ff7d --- /dev/null +++ b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_published_plain.html @@ -0,0 +1,13 @@ +{% block content %} + + {% if cadi_id %} + CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode={{ cadi_id }} + {% endif %} + Title: {{ title }} + Questionnaire URL: {{ published_url }} + {% if cms_stats_committee_by_pag %} + Statistics Committee assignee: {{ cms_stats_committee_by_pag.get('primary', '-') }} (primary), {{ cms_stats_committee_by_pag.get('secondary', '-') }} (secondary). + {% endif %} + + Submitted by {{ submitter_email }}. +{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review.html b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review.html new file mode 100644 index 0000000000..29de4ca9d7 --- /dev/null +++ b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review.html @@ -0,0 +1,9 @@ +{% block paragraph %} + {% if cadi_id %} +

CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode={{ cadi_id }}

+ {% endif %} +

Title: {{ title }}

+

Questionnaire URL: {{ working_url }}

+
+

Submitted by {{ submitter_email }}.

+{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review_plain.html b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review_plain.html new file mode 100644 index 0000000000..ce01645689 --- /dev/null +++ b/cap/modules/mail/templates/mail/body/experiments/cms/questionnaire_message_review_plain.html @@ -0,0 +1,9 @@ +{% block paragraph %} + {% if cadi_id %} + CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode={{ cadi_id }} + {% endif %} + Title: {{ title }} + Questionnaire URL: {{ working_url }} + + Submitted by {{ submitter_email }}. +{% endblock %} \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/partials/divider.html b/cap/modules/mail/templates/mail/partials/divider.html new file mode 100644 index 0000000000..20090c80ce --- /dev/null +++ b/cap/modules/mail/templates/mail/partials/divider.html @@ -0,0 +1,48 @@ +
+
+
+ + +
+
+ +
+ + + + + + + + + +
+ +
+
+ + +
+
+
\ No newline at end of file diff --git a/cap/modules/mail/templates/mail/partials/footer.html b/cap/modules/mail/templates/mail/partials/footer.html new file mode 100644 index 0000000000..ab47400e7e --- /dev/null +++ b/cap/modules/mail/templates/mail/partials/footer.html @@ -0,0 +1,98 @@ +
+
+
+ + +
+
+ +
+ + + {% block footer %}{% endblock %} +
+
+

+ This email was sent automatically by CERN Analysis Preservation. +

+

+ For tutorials, use cases, FAQ and more, please refer to our documentation + here.

+

+ If you need help or have a question, please open a ticket through the CERN Service Portal. 

+ +

+  

+

+  

+
+
+ + +
+
+

+ Copyright © 2014-2021 CERN

+
+
+ + + + + + + + + +
+ +
+
+ + +
+
+
\ No newline at end of file diff --git a/cap/modules/mail/templates/mail/partials/head.html b/cap/modules/mail/templates/mail/partials/head.html new file mode 100644 index 0000000000..44dfcf5a6a --- /dev/null +++ b/cap/modules/mail/templates/mail/partials/head.html @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + diff --git a/cap/modules/mail/templates/mail/partials/header.html b/cap/modules/mail/templates/mail/partials/header.html new file mode 100644 index 0000000000..5d038d8468 --- /dev/null +++ b/cap/modules/mail/templates/mail/partials/header.html @@ -0,0 +1,85 @@ +
+
+
+ + +
+
+ +
+ + + + + + + + +
+ Alternate text + +
+ + + + + + + + +
+ +
+
+ + +
+
+
\ No newline at end of file diff --git a/cap/modules/mail/templates/mail/subject/questionnaire_subject_published.html b/cap/modules/mail/templates/mail/subject/questionnaire_subject_published.html new file mode 100644 index 0000000000..350e578c7e --- /dev/null +++ b/cap/modules/mail/templates/mail/subject/questionnaire_subject_published.html @@ -0,0 +1 @@ +Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - {{ "New Version of Published Analysis" if revision > 0 else "New Published Analysis" }} | CERN Analysis Preservation \ No newline at end of file diff --git a/cap/modules/mail/templates/mail/subject/questionnaire_subject_review.html b/cap/modules/mail/templates/mail/subject/questionnaire_subject_review.html new file mode 100644 index 0000000000..7122647d1f --- /dev/null +++ b/cap/modules/mail/templates/mail/subject/questionnaire_subject_review.html @@ -0,0 +1 @@ +Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation \ No newline at end of file diff --git a/cap/modules/mail/utils.py b/cap/modules/mail/utils.py index 54ab50a2fa..d6245b290e 100644 --- a/cap/modules/mail/utils.py +++ b/cap/modules/mail/utils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of CERN Analysis Preservation Framework. -# Copyright (C) 2016 CERN. +# Copyright (C) 2021 CERN. # # CERN Analysis Preservation Framework is free software; you can redistribute # it and/or modify it under the terms of the GNU General Public License as @@ -21,285 +21,244 @@ # In applying this license, CERN does not # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. - import re -from flask import current_app -from flask_login import current_user -from flask_principal import RoleNeed +from flask import current_app, request +from jinja2 import Environment, PackageLoader, select_autoescape +from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError +from werkzeug.exceptions import BadRequest -from invenio_accounts.models import User -from invenio_access.permissions import Permission +from . import custom as custom_methods -from cap.modules.mail.tasks import create_and_send +# Creating new environment and new loader for the Jinja templates, to avoid +# injection with Flask context passed in render_template +_template_loader = PackageLoader("cap.modules.mail", "templates") +_mail_jinja_env = Environment( + loader=_template_loader, autoescape=select_autoescape() +) +EMAIL_REGEX = re.compile(r"(?!.*\.\.)(^[^.][^@\s]+@[^@\s]+\.[^@\s.]+$)") -def path_value_equals(element, JSON): - paths = element.split(".") - data = JSON - try: - for i in range(0, len(paths)): - data = data[paths[i]] - except KeyError: - return None - - return data +CONFIG_DEFAULTS = { + "review": { + "subject": "New review on document | CERN Analysis Preservation", + "body": { + "plain": ("mail/body/analysis_review.html", None), + "html": ("mail/body/analysis_review.html", None), + }, + }, + "publish": { + "subject": "New published document | CERN Analysis Preservation", + "body": { + "plain": ("mail/body/analysis_published_plain.html", None), + "html": ("mail/body/analysis_published.html", None), + }, + }, +} -def create_analysis_url(deposit): - status = deposit['_deposit']['status'] - return f'drafts/{deposit["_deposit"]["id"]}' if status == 'draft'\ - else f'published/{deposit["control_number"]}' +class UnsuccessfulMail(Exception): + """Error during sending email.""" + def __init__(self, **kwargs): + """Initialize exception.""" + super(UnsuccessfulMail, self).__init__() + self.rec_uuid = kwargs.get("rec_uuid", "") + self.msg = kwargs.get("msg", "") + self.params = kwargs.get("params", {}) -def create_base_message(deposit, host_url, params=None): - cadi = f"CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=" \ - f"{deposit.get('analysis_context', {}).get('cadi_id')}\n" - title = f"Title: {deposit.get('general_title')}\n" - quest = f"Questionnaire URL : {host_url}{create_analysis_url(deposit)}\n" - msg = f"{title}{cadi}{quest}" +def get_config_default(action, type, plain=False): + """Get defaults for different actions/mail info.""" + return CONFIG_DEFAULTS.get(action, {}).get(type) - if params: - stats_assignee = f"Statistics Committee assignee: " \ - f"{params.get('primary', '-')} (primary), " \ - f"{params.get('secondary', '-')} (secondary)\n" - msg += stats_assignee - return msg +def path_value_equals(path, record): + """Given a string path, retrieve the JSON item.""" + # TODO: better handle data iterations + paths = path.split(".") + data = record.dumps() + try: + for i in range(0, len(paths)): + data = data[paths[i]] + return data + # TODO: better error handling + except: + return None -def create_base_subject(config, cadi_id, recid=None): - if not recid: - return f"Questionnaire for {cadi_id} - " \ - if cadi_id else config.get("email_subject") +def populate_template_from_ctx( + record, config, module=None, type=None, default_ctx={} +): + """ + Render a template according to the context provided in the schema. + + Args: + record: The analysis record that has the necessary fields. + config: The analysis config, provided in the `schema`. + action: THe action that triggered the notification (e.g. `publish`). + module: The file that will hold the custom created functions. + type: The specific attribute that triggers this template, e.g. + subject, message, etc + + Returns: The rendered string, using the required context values. + """ + config_ctx = config.get("ctx", []) + template = config.get("template") + template_file = config.get("template_file") + action = default_ctx.get("action", "") + + render = render_template + if template: + render = render_template_string + template_to_render = template + elif template_file: + template_to_render = template_file else: - return f"Questionnaire for {cadi_id} {recid} - " \ - if cadi_id else f"Questionnaire for {recid} - " - - -def add_hypernews_mail_to_recipients(recipients, cadi_id): - # mail for reviews - Hypernews - # should be sent to hn-cms-@cern.ch if well-formed - cadi_regex = current_app.config.get("CADI_REGEX") - hypernews_mail = current_app.config.get("CMS_HYPERNEWS_EMAIL_FORMAT") - - if re.match(cadi_regex, cadi_id) and hypernews_mail: - recipients.append(hypernews_mail.format(cadi_id)) - + if type == "body": + mime_type = "plain" if config.get("plain") else "html" + template_to_render = ( + CONFIG_DEFAULTS.get(action, {}).get(type, {}).get(mime_type) + ) + else: + template_to_render = CONFIG_DEFAULTS.get(action, {}).get(type, {}) + + if not template_to_render: + # if there is no `template_to_render` then abort + # TODO: except if it is for recepients continue with next + msg = ( + "Not template passed and no default templates found. " + "Notification procedure aborted." + ) + current_app.logger.error(msg) + raise UnsuccessfulMail( + rec_uuid=record.id, msg=msg, params={"config": config} + ) -def get_review_recipients(deposit, host_url, config): - # mail of reviewer - reviewer_mail = current_user.email - recipients = [reviewer_mail, ] + ctx = {**default_ctx} + gen_ctx = generate_ctx(config_ctx, record=record, default_ctx=default_ctx) + ctx = {**ctx, **gen_ctx} - # mail of owner - # owners = deposit.get("_deposit", {}).get("owners") - owner_mail = "-" try: - owner = deposit["_deposit"]["owners"][0] - owner_mail = User.query.filter_by(id=owner).one().email - recipients.append(owner_mail) - except IndexError: - pass - - cadi_id = deposit.get("analysis_context", {}).get("cadi_id") - if cadi_id: - # if cadi mail Hypernews if review from admin reviewer (stat committee) - cms_stats_commitee_email = current_app.config.get( - "CMS_STATS_QUESTIONNAIRE_ADMIN_ROLES" + return render(template_to_render, **ctx) + except TemplateNotFound as ex: + msg = f"Template {ex.name} not found. Notification procedure aborted." + raise UnsuccessfulMail( + rec_uuid=record.id, msg=msg, params={"config": config} + ) + except TemplateSyntaxError as ex: + msg = f"Template error: {ex.message}. Notification procedure aborted." + raise UnsuccessfulMail( + rec_uuid=record.id, msg=msg, params={"config": config} + ) + except TypeError: + msg = "Context for template is empty. Notification procedure aborted." + raise UnsuccessfulMail( + rec_uuid=record.id, msg=msg, params={"config": config} ) - # check that current user is an admin reviewer - if (cms_stats_commitee_email and - Permission(RoleNeed(cms_stats_commitee_email)).can()): - add_hypernews_mail_to_recipients(recipients, cadi_id) - - subject = create_base_subject(config, cadi_id) - message = create_base_message(deposit, host_url) - message += f"Submitted by {owner_mail}, and reviewed by {reviewer_mail}." - - return subject, message, recipients - - -def get_cms_stat_recipients(record, host_url, config): - data = current_app.config.get("CMS_STATS_COMMITEE_AND_PAGS") - key = path_value_equals(config.get("path", ""), record) - key = key if key else "other" - recipients = data.get(key, {}).get("contacts", []) - params = data.get(key, {}).get("params", {}) - - # submitter email - recipients.append(current_user.email) - - # mail for PDF forum - pdf_mail = current_app.config.get("PDF_FORUM_MAIL") - if pdf_mail and record.get("parton_distribution_functions", None): - recipients.append(pdf_mail) - - # mail for ML surveys - CMS conveners - conveners_ml_mail = current_app.config.get("CONVENERS_ML_MAIL") - conveners_ml_jira_mail = current_app.config.get("CONVENERS_ML_JIRA_MAIL") - - # Some extra info for the CMS conveners recipients: - # 1. if uses centralized CMS ML applications (3.3) (not empty) - # 2. if 3.4 = Yes (mva_use = Yes) - # 3. if the user adds an app to 3.a (ml_app_use - not empty) - # 4. or if the user answers to 3.b (ml_survey.options = Yes) - centralized_apps = ( - record.get("multivariate_discriminants", {}) - .get("use_of_centralized_cms_apps", {}) - .get("options", []) - ) - mva_use = record.get("multivariate_discriminants", {}).get("mva_use") - ml_app_use = record.get("ml_app_use", []) - ml_survey = record.get("ml_survey", {}).get("options") - - if conveners_ml_mail and ( - (centralized_apps and 'No' not in centralized_apps) or mva_use == 'Yes' or # noqa - ml_app_use or ml_survey == 'Yes' # noqa - ): - recipients += [conveners_ml_mail, conveners_ml_jira_mail] - - cadi_id = record.get("analysis_context", {}).get("cadi_id") - if cadi_id: - add_hypernews_mail_to_recipients(recipients, cadi_id) - - recid = record['_deposit']['pid']['value'] - - subject = create_base_subject(config, cadi_id, recid) - message = create_base_message(record, host_url, params) - message += f"Submitted by {current_user.email}." - - return subject, message, recipients - - -GENERATE_RECIPIENT_METHODS = { - "path_value_equals": path_value_equals, - "get_cms_stat_recipients": get_cms_stat_recipients, - "get_review_recipients": get_review_recipients, -} -NOTIFICATION_RECEPIENT = { - "https://analysispreservation.cern.ch/schemas/deposits/" - "records/cms-stats-questionnaire-v0.0.1.json": { - "publish": { - "type": "method", - "method": "get_cms_stat_recipients", - "path": "analysis_context.wg", - "email_subject": "CMS Statistics Committee - " - }, - "review": { - "type": "method", - "method": "get_review_recipients", - "email_subject": "CMS Statistics Questionnaire - " - } - }, - "https://analysispreservation.cern.ch/schemas/deposits/" - "records/cms-stats-questionnaire-v0.0.2.json": { - "publish": { - "type": "method", - "method": "get_cms_stat_recipients", - "path": "analysis_context.wg", - "email_subject": "CMS Statistics Committee - " - }, - "review": { - "type": "method", - "method": "get_review_recipients", - "email_subject": "CMS Statistics Questionnaire - " - } +def generate_ctx(config_ctx, record=None, default_ctx={}): + ctx = {**default_ctx} + for attrs in config_ctx: + if attrs.get("path"): + name = attrs["name"] + val = path_value_equals(attrs["path"], record) + ctx.update({name: val}) + elif attrs.get("method"): + try: + name = attrs["method"] + custom_func = getattr(custom_methods, name) + val = custom_func(record, default_ctx=default_ctx) + ctx.update({name: val}) + except AttributeError: + continue + return ctx + + +def update_mail_list(record, config, mails, default_ctx={}): + """ + Adds mails (default or formatted) in tha mails collection. + + An example of the expected config is this: + "mails": { + "default": [default1, default2], + "formatted": [{ + "template": "template-mail-with-{{ var }}", + "ctx": [{"name": "var", "path": "fields.field1"}] + }] } -} - - -def generate_notification_attrs(record, host_url, config): - _type = config.get("type") - if _type == "method": - func = GENERATE_RECIPIENT_METHODS.get(config.get("method")) - if func: - return func(record, host_url, config) - elif _type == "list": - return config.get("message", ""), config.get("data") - else: - return "", "", None - - -def send_mail_on_publish(recid, revision, - host_url='https://analysispreservation.cern.ch/', - recipients=None, - message='', - subject_prefix=''): - if revision > 0: - subject = subject_prefix + \ - "New Version of Published Analysis | CERN Analysis Preservation" - template = "mail/analysis_published_revision.html" - else: - subject = subject_prefix + \ - "New Published Analysis | CERN Analysis Preservation" - template = "mail/analysis_published_new.html" - - send_mail_on_hypernews(recipients, subject, message) - send_mail_on_jira(recid, host_url, recipients, message, subject) - - current_app.logger.info( - f'Publish mail: {recid} - {", ".join(recipients)}.') - create_and_send.delay( - template, - dict(recid=recid, url=host_url, message=message), - subject, - recipients - ) - + """ + try: + mails_list = config.get("default", []) + formatted_list = config.get("formatted", []) + except AttributeError: + current_app.logger.error( + "Mail configuration is not a dict with 'default', 'formatted' keys" + ) + return + + if mails_list: + mails += mails_list + if formatted_list: + for formatted in formatted_list: + try: + _formatted_email = populate_template_from_ctx( + record, formatted, type="recipient", default_ctx=default_ctx + ) + mails += [_formatted_email] + except UnsuccessfulMail: + continue + + +def is_review_request(): + """ + On a review action, make sure that it is an actual review. + + On a review action, make sure that it is an actual review and not one + of the accompanying actions (delete review, resolve review. + """ + # TODO: check how to better handle, maybe send different messages + not_permitted = ["resolve", "delete"] -def send_mail_on_review(analysis_url, - host_url='https://analysispreservation.cern.ch/', - recipients=None, - message='', - subject_prefix=''): - subject = subject_prefix + \ - "New Review on Analysis | CERN Analysis Preservation" - template = "mail/analysis_review.html" + try: + req = request.get_json() + if isinstance(req, dict) and req.get("action") in not_permitted: + return False + except BadRequest: + return False - send_mail_on_hypernews(recipients, subject, message) + return True - create_and_send.delay( - template, - dict(analysis_url=analysis_url, url=host_url, message=message), - subject, - recipients - ) +def _render(template, context): + """Renders the template and fires the signal.""" + rv = template.render(context) + return rv -def send_mail_on_hypernews(recipients, subject, message): - # differentiate between hypernews mail and the others - r = re.compile('hn-cms-.+') - hypernews_list = [rec for rec in recipients if r.match(rec)] - if hypernews_list: - template = "mail/analysis_plain_text.html" - recipients.remove(hypernews_list[0]) +def render_template(template_name_or_list, **context): + """Renders a template from the template folder with context. - create_and_send.delay( - template, - dict(message=message), - subject, - hypernews_list, - type="plain" - ) + :param template_name_or_list: the name of the template to be + rendered, or an iterable with template names + the first one existing will be rendered + :param context: the variables that should be available in the + context of the template. + """ + return _render( + _mail_jinja_env.get_or_select_template(template_name_or_list), context + ) -def send_mail_on_jira(recid, host_url, recipients, message, subject): - # only JIRA ML mail - conveners_ml_jira_mail = current_app.config.get("CONVENERS_ML_JIRA_MAIL") +def render_template_string(source, **context): + """Renders a template from the template string with context. - if conveners_ml_jira_mail in recipients: - template = "mail/analysis_plain_text.html" - recipients.remove(conveners_ml_jira_mail) + Template variables will be autoescaped. - create_and_send.delay( - template, - dict(recid=recid, url=host_url, message=message), - subject, - [conveners_ml_jira_mail], - type="plain" - ) + :param source: the source code of the template to be + rendered + :param context: the variables that should be available in the + context of the template. + """ + return _render(_mail_jinja_env.from_string(source), context) diff --git a/cap/modules/schemas/cli.py b/cap/modules/schemas/cli.py index 20a3f7fb3b..74bc4bbd0d 100644 --- a/cap/modules/schemas/cli.py +++ b/cap/modules/schemas/cli.py @@ -30,10 +30,10 @@ import requests from flask import current_app from flask_cli import with_appcontext -from invenio_db import db from invenio_accounts.models import User -from invenio_oauth2server.models import Token +from invenio_db import db from invenio_jsonschemas.errors import JSONSchemaNotFound +from invenio_oauth2server.models import Token from invenio_search import current_search_client from sqlalchemy.exc import IntegrityError @@ -41,9 +41,13 @@ from cap.modules.deposit.errors import DepositValidationError from cap.modules.fixtures.cli import fixtures from cap.modules.records.api import CAPRecord +from cap.modules.schemas.helpers import ValidationError from cap.modules.schemas.models import Schema -from cap.modules.schemas.resolvers import resolve_schema_by_url,\ - resolve_schema_by_name_and_version, schema_name_to_url +from cap.modules.schemas.resolvers import ( + resolve_schema_by_name_and_version, + resolve_schema_by_url, + schema_name_to_url, +) from cap.modules.schemas.utils import is_later_version DEPOSIT_REQUIRED_FIELDS = [ @@ -53,7 +57,7 @@ '_experiment', '_fetched_from', '_user_edited', - '_access' + '_access', ] @@ -86,9 +90,7 @@ def _get_or_create_token(token_name, user_email): token_ = Token.query.filter_by(user_id=user.id).first() if not token_: token_ = Token.create_personal( - token_name, - user.id, - scopes=['deposit:write'] + token_name, user.id, scopes=['deposit:write'] ) db.session.add(token_) db.session.commit() @@ -98,33 +100,52 @@ def _get_or_create_token(token_name, user_email): @fixtures.command() -@click.option('--schema-url', '-u', - cls=MutuallyExclusiveOption, - mutually_exclusive=["ana_type", "ana_version"], - help='The url of the schema used for validation.') -@click.option('--ana-type', '-a', - cls=MutuallyExclusiveOption, - mutually_exclusive=["schema_url"], - help='The analysis type of the schema used for validation.') -@click.option('--ana-version', '-v', - help='The analysis version of the records.') -@click.option('--compare-with', '-c', - help='The schema version, that the ' - 'records should be compared to.') -@click.option('--status', '-s', - default='draft', - type=click.Choice(['draft', 'published']), - help='The metadata type that will be used for validation.') -@click.option('--export', '-e', - type=click.Path(), - help='A file where, the validation errors can be saved.') -@click.option('--export-type', '-et', - default=None, - type=click.Choice(['md']), - help='The export type that will be used for output.') +@click.option( + '--schema-url', + '-u', + cls=MutuallyExclusiveOption, + mutually_exclusive=["ana_type", "ana_version"], + help='The url of the schema used for validation.', +) +@click.option( + '--ana-type', + '-a', + cls=MutuallyExclusiveOption, + mutually_exclusive=["schema_url"], + help='The analysis type of the schema used for validation.', +) +@click.option( + '--ana-version', '-v', help='The analysis version of the records.' +) +@click.option( + '--compare-with', + '-c', + help='The schema version, that the ' 'records should be compared to.', +) +@click.option( + '--status', + '-s', + default='draft', + type=click.Choice(['draft', 'published']), + help='The metadata type that will be used for validation.', +) +@click.option( + '--export', + '-e', + type=click.Path(), + help='A file where, the validation errors can be saved.', +) +@click.option( + '--export-type', + '-et', + default=None, + type=click.Choice(['md']), + help='The export type that will be used for output.', +) @with_appcontext -def validate(schema_url, ana_type, ana_version, compare_with, - status, export, export_type): +def validate( + schema_url, ana_type, ana_version, compare_with, status, export, export_type +): """Validate deposit or record metadata based on their schema. Provide the schema url OR ana-type and version, as well as the schema version @@ -144,16 +165,18 @@ def validate(schema_url, ana_type, ana_version, compare_with, schema = resolve_schema_by_name_and_version(ana_type, ana_version) else: raise click.UsageError( - 'You need to provide the ana-type or the schema-url.') + 'You need to provide the ana-type or the schema-url.' + ) except JSONSchemaNotFound: - raise click.UsageError( - 'Schema not found.') + raise click.UsageError('Schema not found.') except ValueError: raise click.UsageError( - 'Version has to be passed as string ...') + 'Version has to be passed as string ...' + ) # differentiate between drafts/published from cap.modules.deposit.api import CAPDeposit + if status == 'draft': search_path = 'deposits-records' cap_record_class = CAPDeposit @@ -165,13 +188,14 @@ def validate(schema_url, ana_type, ana_version, compare_with, records = current_search_client.search( search_path, q=f'_deposit.status: {status} AND ' - f'$schema: "{schema_name_to_url(schema.name, schema.version)}"', - size=5000 + f'$schema: "{schema_name_to_url(schema.name, schema.version)}"', + size=5000, )['hits']['hits'] pids = [rec['_id'] for rec in records] click.secho( - f'{len(records)} record(s) of {schema.name} found.\n', fg='green') + f'{len(records)} record(s) of {schema.name} found.\n', fg='green' + ) total_errors = [] for pid in pids: @@ -182,15 +206,18 @@ def validate(schema_url, ana_type, ana_version, compare_with, # get the url of the schema version, used for validation if compare_with: cap_record['$schema'] = schema_name_to_url( - schema.name, compare_with) + schema.name, compare_with + ) try: cap_record.validate() click.secho(f'No errors found in record {pid}', fg='green') except DepositValidationError as exc: if export_type == 'md': - msg = '- [ ] Errors in **CADI ID:** ' + \ - f'{cap_record_cadi_id or "?"}' + \ - f' - **[link]({cap_host}/{cap_record_pid})** :\n' + msg = ( + '- [ ] Errors in **CADI ID:** ' + + f'{cap_record_cadi_id or "?"}' + + f' - **[link]({cap_host}/{cap_record_pid})** :\n' + ) msg += "\n| Field Path | Error | \n| ---------- | ----- | \n" for err in exc.errors: _err = err.res @@ -199,9 +226,11 @@ def validate(schema_url, ana_type, ana_version, compare_with, msg += "----\n" else: error_list = '\n'.join(str(err.res) for err in exc.errors) - msg = f'Errors in {pid} - CADI ' + \ - f'id: {cap_record_cadi_id or "?"}' + \ - f' - {cap_host}/{cap_record_pid} :\n{error_list}' + msg = ( + f'Errors in {pid} - CADI ' + + f'id: {cap_record_cadi_id or "?"}' + + f' - {cap_host}/{cap_record_pid} :\n{error_list}' + ) click.secho(msg, fg='red') @@ -216,20 +245,33 @@ def validate(schema_url, ana_type, ana_version, compare_with, @fixtures.command() -@click.option('--dir', '-d', - type=click.Path(exists=True), - help='The directory of the schemas to be added to the db.') -@click.option('--url', '-u', - help='The url of the schema to be added to the db.') -@click.option('--force', '-f', - is_flag=True, - help='Force add the schema without validation.') -@click.option('--force-version', '-fv', - is_flag=True, - help='Force disable version checking.') -@click.option('--replace', '-r', - is_flag=True, - help='It this schema/version exists, update it.') +@click.option( + '--dir', + '-d', + type=click.Path(exists=True), + help='The directory of the schemas to be added to the db.', +) +@click.option( + '--url', '-u', help='The url of the schema to be added to the db.' +) +@click.option( + '--force', + '-f', + is_flag=True, + help='Force add the schema without validation.', +) +@click.option( + '--force-version', + '-fv', + is_flag=True, + help='Force disable version checking.', +) +@click.option( + '--replace', + '-r', + is_flag=True, + help='It this schema/version exists, update it.', +) @with_appcontext def schemas(dir, url, force, force_version, replace): """Add schemas to CAP. @@ -248,19 +290,25 @@ def schemas(dir, url, force, force_version, replace): if url: resp = requests.get(url) if not resp.ok: - click.secho(f'Please provide a public url to a json file. ' - f'Error {resp.status_code}.', fg='red') + click.secho( + f'Please provide a public url to a json file. ' + f'Error {resp.status_code}.', + fg='red', + ) return try: data = resp.json() if not has_all_required_fields(data) and not force: - click.secho(f'Missing required fields. Make sure that all of: ' - f'{DEPOSIT_REQUIRED_FIELDS} are in the json.', fg='red') # noqa + click.secho( + f'Missing required fields. Make sure that all of: ' + f'{DEPOSIT_REQUIRED_FIELDS} are in the json.', + fg='red', + ) # noqa return - add_schema_from_json(data, - replace=replace, - force_version=force_version) + add_schema_from_json( + data, replace=replace, force_version=force_version + ) except (json.JSONDecodeError, ValueError): click.secho('Error in decoding the returned json.', fg='red') @@ -269,24 +317,26 @@ def schemas(dir, url, force, force_version, replace): add_fixtures_from_path( current_app.config.get('SCHEMAS_DEFAULT_PATH'), replace=replace, - force_version=True) + force_version=True, + ) def add_fixtures_from_path(dir, replace=None, force_version=None): """Add fixtures from a specified path to CAP.""" for root, dirs, files in os.walk(dir): json_filepaths = [ - os.path.join(root, f) for f in files - if f.endswith(".json") + os.path.join(root, f) for f in files if f.endswith(".json") ] for filepath in json_filepaths: with open(filepath, 'r') as f: try: json_content = json.load(f) - add_schema_from_json(json_content, - replace=replace, - force_version=force_version) + add_schema_from_json( + json_content, + replace=replace, + force_version=force_version, + ) except ValueError: click.secho(f'Not valid json in {filepath} file', fg='red') @@ -310,12 +360,16 @@ def add_schema_from_json(data, replace=None, force_version=None): except JSONSchemaNotFound: try: _schema = Schema.get_latest(name=name) - if is_later_version(_schema.version, - version) and not force_version: + if ( + is_later_version(_schema.version, version) + and not force_version + ): click.secho( f'A later version ({_schema.version}) of ' f'{name} is already in the db. Error while ' - f'adding {name}-{version}', fg='red') + f'adding {name}-{version}', + fg='red', + ) return else: _schema = Schema(**data) @@ -330,6 +384,13 @@ def add_schema_from_json(data, replace=None, force_version=None): db.session.commit() click.secho(f'{name} has been added.', fg='green') + except ValidationError as err: + click.secho(err, fg='red') + click.secho( + f'Configuration is invalid. ' f'Aborting update for schema {name}.', + fg='red', + ) + return except IntegrityError: click.secho(f'An db error occurred while adding {name}.', fg='red') @@ -337,9 +398,11 @@ def add_schema_from_json(data, replace=None, force_version=None): def has_all_required_fields(data): """Check if there are any missing required fields in a json schema.""" deposit_as_record = data.get('use_deposit_as_record') - schemas_to_check = [data['deposit_schema']] \ - if deposit_as_record \ + schemas_to_check = ( + [data['deposit_schema']] + if deposit_as_record else [data['deposit_schema'], data['record_schema']] + ) for _schema in schemas_to_check: schema_fields = _schema.get('properties', {}).keys() diff --git a/cap/modules/schemas/helpers.py b/cap/modules/schemas/helpers.py new file mode 100644 index 0000000000..3dba8816a9 --- /dev/null +++ b/cap/modules/schemas/helpers.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2016 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +"""Utils for Schemas module.""" + + +class ValidationError(Exception): + """Schema validation error.""" + + errors = None + description = "Validation Error" + + def __init__(self, errors=None, description=None, **kwargs): + """Initialize RESTException.""" + super(ValidationError, self).__init__(**kwargs) + if errors is not None: + self.errors = errors + + if description: + self.description = description diff --git a/cap/modules/schemas/jsonschemas/definitions.py b/cap/modules/schemas/jsonschemas/definitions.py new file mode 100644 index 0000000000..ef7797439a --- /dev/null +++ b/cap/modules/schemas/jsonschemas/definitions.py @@ -0,0 +1,77 @@ +definitions_schema = { + "ctx": { + "type": "array", + "title": "Context (ctx) Options", + "uniqueItems": True, + "items": { + "anyOf": [ + { + "properties": { + "name": {"type": "string", "title": "Variable name"}, + "path": {"type": "string", "title": "Variable Path"}, + }, + "required": ["name", "path"], + }, + { + "properties": { + "method": {"type": "string", "title": "Variable Method"} + }, + "required": ["method"], + }, + ] + }, + }, + "mails": { + "type": "object", + "title": "Mails", + "properties": { + "default": { + "type": "array", + "title": "Default", + "uniqueItems": True, + "items": {"type": "string", "title": "Mail"}, + }, + "formatted": { + "type": "array", + "title": "Formatted", + "uniqueItems": True, + "items": { + "type": "object", + "title": "Mail", + "properties": { + "template": {"type": "string", "title": "Template"}, + "ctx": {"$ref": "#/definitions/ctx"}, + }, + }, + }, + }, + }, + "checks": { + "title": "Checks", + "type": "object", + "additionalProperties": False, + "properties": { + "op": {"type": "string", "title": "AND/OR"}, + "checks": { + "type": "array", + "items": { + "oneOf": [ + {"$ref": "#/definitions/checks"}, + {"$ref": "#/definitions/condition"}, + ] + }, + }, + }, + }, + "condition": { + "type": "object", + "title": "Add Check", + "additionalProperties": False, + "properties": { + "path": {"type": "string", "title": "Path"}, + "condition": {"type": "string", "title": "Condition"}, + "value": {"type": "string", "title": "Value"}, + "params": {"type": "object"}, + }, + }, +} diff --git a/cap/modules/schemas/jsonschemas/notifications_schema.py b/cap/modules/schemas/jsonschemas/notifications_schema.py new file mode 100644 index 0000000000..813fc5ce77 --- /dev/null +++ b/cap/modules/schemas/jsonschemas/notifications_schema.py @@ -0,0 +1,174 @@ +notifications_schema = { + "type": "object", + "title": "Notification Configuration", + "additionalProperties": False, + "properties": { + "actions": { + "type": "object", + "title": "Notification Actions", + "additionalProperties": False, + "patternProperties": { + "^(publish|review)$": { + "type": "array", + "title": "Publish Options", + "items": { + "type": "object", + "title": "Mail Config/Options", + "required": ["recipients"], + "minItems": 1, + "properties": { + "ctx": {"$ref": "#/definitions/ctx"}, + "subject": { + "anyOf": [ + { + "type": "object", + "title": "Mail String Template Subject", + "additionalProperties": False, + "properties": { + "template": { + "type": "string", + "title": "Template string (Jinja format)", # noqa + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "method": { + "type": "string", + "title": "Method name", + }, + }, + }, + { + "type": "object", + "title": "Mail Template File Subject", + "additionalProperties": False, + "properties": { + "template_file": { + "type": "string", + "title": "Template file (path)", + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "method": { + "type": "string", + "title": "Method name", + }, + }, + }, + ] + }, + "body": { + "anyOf": [ + { + "type": "object", + "title": "Mail Message", + "additionalProperties": False, + "properties": { + "template_file": { + "type": "string", + "title": "Template file (path)", # noqa + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "base_template": { + "type": "string", + "title": "Base Template", + }, + "plain": { + "type": "boolean", + "title": "Plain text or HTML", + }, + }, + }, + { + "type": "object", + "title": "Mail Message", + "additionalProperties": False, + "properties": { + "template": { + "type": "string", + "title": "Template type (string/path)", # noqa + }, + "ctx": { + "$ref": "#/definitions/ctx" + }, + "base_template": { + "type": "string", + "title": "Base Template", + }, + "plain": { + "type": "boolean", + "title": "Plain text or HTML", + }, + }, + }, + ] + }, + "recipients": { + "type": "object", + "title": "Mail Recipients", + "minProperties": 1, + "patternProperties": { + "^(bcc|cc|recipients)$": { + "type": "array", + "title": "List of BCC recipients", + "uniqueItems": True, + "minItems": 1, + "items": { + "oneOf": [ + {"type": "string"}, + { + "additionalProperties": False, # noqa + "type": "object", + "properties": { + "op": { + "type": "string", + "title": "AND/OR", + }, + "checks": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/checks" # noqa + }, + { + "$ref": "#/definitions/condition" # noqa + }, + ] + }, + }, + "mails": { + "$ref": "#/definitions/mails" # noqa + }, + "method": { + "oneOf": [ + { + "title": "Method", # noqa + "type": "string", # noqa + }, + { + "type": "array", # noqa + "items": { + "title": "Method", # noqa + "type": "string", # noqa + }, + }, + ] + }, + }, + }, + ] + }, + } + }, + }, + }, + }, + } + }, + } + }, +} diff --git a/cap/modules/schemas/jsonschemas/repositories_schema.py b/cap/modules/schemas/jsonschemas/repositories_schema.py new file mode 100644 index 0000000000..3b6ad381a0 --- /dev/null +++ b/cap/modules/schemas/jsonschemas/repositories_schema.py @@ -0,0 +1,76 @@ +repositories_schema = { + "type": "object", + "title": "Repository configuration", + "patternProperties": { + "^.*$": { + "title": "Repository Configuration", + "type": "object", + "description": "Add your repository", + "required": [ + "host", + "authentication", + "repo_name", + "repo_description", + "org_name", + "private", + "license", + ], + "dependencies": { + "host": { + "oneOf": [ + {"properties": {"host": {"enum": ["github.com"]}}}, + { + "properties": { + "host": {"enum": ["gitlab.cern.ch"]}, + "org_id": { + "title": "Unique ID of the organisation", # noqa + "type": "string", + }, + }, + "required": ["org_id"], + }, + ] + } + }, + "properties": { + "display_name": {"type": "string"}, + "display_description": {"type": "string"}, + "host": { + "title": "Host name", + "type": "string", + "enum": ["gitlab.cern.ch", "github.com"], + }, + "org_name": { + "title": "Name of the organisation", + "type": "string", + }, + "authentication": { + "title": "Authentication for repository.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["user", "cap"], + } + }, + }, + "repo_name": { + "title": "Name of the repository", + "type": "object", + }, + "repo_description": { + "title": "Description of the repository", + "type": "object", + }, + "private": { + "title": "Visibility of the repository", + "type": "boolean", + }, + "license": { + "title": "License of the repository", + "type": "string", + }, + }, + } + }, +} diff --git a/cap/modules/schemas/jsonschemas/schema_config.py b/cap/modules/schemas/jsonschemas/schema_config.py index 1f068a6c8c..d03980b12b 100644 --- a/cap/modules/schemas/jsonschemas/schema_config.py +++ b/cap/modules/schemas/jsonschemas/schema_config.py @@ -1,7 +1,16 @@ +from cap.modules.schemas.jsonschemas.definitions import definitions_schema +from cap.modules.schemas.jsonschemas.notifications_schema import ( + notifications_schema, +) +from cap.modules.schemas.jsonschemas.repositories_schema import ( + repositories_schema, +) + SCHEMA_CONFIG_JSONSCHEMA_V1 = { "title": "Deposit/Record Schema Configuration", "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "definitions": definitions_schema, "properties": { "pid": { "type": "object", @@ -14,87 +23,9 @@ }, }, "x-cap-permission": {"type": "boolean"}, + "notifications": notifications_schema, "reviewable": {"type": "boolean"}, - "repositories": { - "type": "object", - "title": "Repository configuration", - "patternProperties": { - "^.*$": { - "title": "Repository Configuration", - "type": "object", - "description": "Add your repository", - "required": [ - "host", - "authentication", - "repo_name", - "repo_description", - "org_name", - "private", - "license", - ], - "dependencies": { - "host": { - "oneOf": [ - { - "properties": { - "host": {"enum": ["github.com"]} - } - }, - { - "properties": { - "host": {"enum": ["gitlab.cern.ch"]}, - "org_id": { - "title": "Unique ID of the organisation", # noqa - "type": "string", - }, - }, - "required": ["org_id"], - }, - ] - } - }, - "properties": { - "display_name": {"type": "string"}, - "display_description": {"type": "string"}, - "host": { - "title": "Host name", - "type": "string", - "enum": ["gitlab.cern.ch", "github.com"], - }, - "org_name": { - "title": "Name of the organisation", - "type": "string", - }, - "authentication": { - "title": "Authentication for repository.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["user", "cap"], - } - }, - }, - "repo_name": { - "title": "Name of the repository", - "type": "object", - }, - "repo_description": { - "title": "Description of the repository", - "type": "object", - }, - "private": { - "title": "Visibility of the repository", - "type": "boolean", - }, - "license": { - "title": "License of the repository", - "type": "string", - }, - }, - } - }, - }, + "repositories": repositories_schema, }, "additionalProperties": False, } diff --git a/cap/modules/schemas/models.py b/cap/modules/schemas/models.py index 6ed18f86c4..e16ef3e28d 100644 --- a/cap/modules/schemas/models.py +++ b/cap/modules/schemas/models.py @@ -42,10 +42,10 @@ from sqlalchemy.orm.exc import NoResultFound from werkzeug.utils import import_string -from cap.modules.deposit.errors import WrongJSONSchemaError from cap.modules.records.errors import get_error_path from cap.types import json_type +from .helpers import ValidationError from .jsonschemas import SCHEMA_CONFIG_JSONSCHEMA_V1 from .permissions import SchemaAdminAction, SchemaReadAction from .serializers import ( @@ -209,11 +209,15 @@ def validate_config(self, key, value): for error in validator.iter_errors(value): errors.append( - FieldError(get_error_path(error), str(error.message)) + FieldError( + get_error_path(error), str(error.message) + ).to_dict() ) + if errors: - raise WrongJSONSchemaError( - "ERROR: Invalid 'config' object.", errors=errors + raise ValidationError( + errors=errors, + description="Schema configuration validation error", ) return value diff --git a/cap/modules/schemas/serializers.py b/cap/modules/schemas/serializers.py index 799aea69f2..c7ed120e92 100644 --- a/cap/modules/schemas/serializers.py +++ b/cap/modules/schemas/serializers.py @@ -63,8 +63,6 @@ class SchemaSerializer(Schema): record_options = fields.Dict() record_mapping = fields.Dict() - links = fields.Method('build_links', dump_only=True) - @pre_load def filter_out_fields_that_cannot_be_updated(self, data, **kwargs): """Remove non editable fields from serialized data.""" @@ -72,6 +70,10 @@ def filter_out_fields_that_cannot_be_updated(self, data, **kwargs): raise ValidationError('Empty data') return data + +class SchemaResponseSerializer(SchemaSerializer): + links = fields.Method('build_links', dump_only=True) + def build_links(self, obj): """Construct schema links.""" deposit_path = url_for( @@ -116,7 +118,25 @@ class PatchedSchemaSerializer(Schema): record_mapping = fields.Dict() -class ResolvedSchemaSerializer(SchemaSerializer): +class SchemaPayloadSerializer(SchemaSerializer): + """Schema serializer with resolved jsonschemas.""" + + config = fields.Dict() + + +class UpdateSchemaPayloadSerializer(SchemaPayloadSerializer): + """Schema serializer with resolved jsonschemas.""" + + @pre_load + def filter_out_fields_that_cannot_be_updated(self, data, **kwargs): + """Remove non editable fields from serialized data.""" + data = {k: v for k, v in iteritems(data) if k in EDITABLE_FIELDS} + if not data: + raise ValidationError('Empty data') + return data + + +class ResolvedSchemaResponseSerializer(SchemaResponseSerializer): """Schema serializer with resolved jsonschemas.""" deposit_schema = fields.Method( @@ -139,12 +159,12 @@ def get_resolved_record_schema(self, obj): return copy.deepcopy(schema) # so all the JSONRefs get resoved -class ConfigResolvedSchemaSerializer(ResolvedSchemaSerializer): +class ConfigResolvedSchemaSerializer(ResolvedSchemaResponseSerializer): config = fields.Dict() experiment = fields.Str(required=False) -class CreateConfigPayload(SchemaSerializer): +class CreateConfigPayload(SchemaResponseSerializer): config = fields.Dict() experiment = fields.Str(required=False) @@ -214,11 +234,13 @@ def build_links(self, obj): return links -schema_serializer = SchemaSerializer() +schema_serializer = SchemaResponseSerializer() +resolved_schemas_serializer = ResolvedSchemaResponseSerializer() patched_schema_serializer = PatchedSchemaSerializer() update_schema_serializer = UpdateSchemaSerializer() -resolved_schemas_serializer = ResolvedSchemaSerializer() config_resolved_schemas_serializer = ConfigResolvedSchemaSerializer() create_config_payload = CreateConfigPayload() collection_serializer = CollectionSerializer() link_serializer = LinkSerializer() +schema_payload_serializer = SchemaPayloadSerializer() +update_payload_schema_serializer = UpdateSchemaPayloadSerializer() diff --git a/cap/modules/schemas/views.py b/cap/modules/schemas/views.py index b7135e9306..33ac475f12 100644 --- a/cap/modules/schemas/views.py +++ b/cap/modules/schemas/views.py @@ -40,12 +40,13 @@ from cap.modules.access.permissions import admin_permission_factory from cap.modules.access.utils import login_required +from .helpers import ValidationError from .models import Schema from .permissions import AdminSchemaPermission, ReadSchemaPermission from .serializers import ( create_config_payload, link_serializer, - update_schema_serializer, + update_payload_schema_serializer, ) from .utils import ( check_allowed_patch_operation, @@ -140,6 +141,11 @@ def post(self): except IntegrityError: raise abort(400, 'Error occured during saving schema in the db.') + except ValidationError as err: + return ( + jsonify({"message": err.description, "errors": err.errors}), + 400, + ) return jsonify(schema.config_serialize()) @@ -153,15 +159,23 @@ def put(self, name, version): with AdminSchemaPermission(schema).require(403): data = request.get_json() - serialized_data, errors = update_schema_serializer.load( + + # self._validate_config(data) + serialized_data, errors = update_payload_schema_serializer.load( data, partial=True ) if errors: raise abort(400, errors) - schema.update(**serialized_data) - db.session.commit() + try: + schema.update(**serialized_data) + db.session.commit() + except ValidationError as err: + return ( + jsonify({"message": err.description, "errors": err.errors}), + 400, + ) return jsonify(schema.config_serialize()) @@ -222,7 +236,7 @@ def patch(self, name, version): 400, ) - serialized_data, errors = update_schema_serializer.load( + serialized_data, errors = update_payload_schema_serializer.load( patched_schema, partial=True ) if errors: @@ -249,9 +263,22 @@ def patch(self, name, version): ), 500, ) + except ValidationError as err: + return ( + jsonify({"message": err.description, "errors": err.errors}), + 400, + ) return jsonify(schema.config_serialize()) + # TODO check if config validates with the model decorator + # def _validate_config(self, data): + # config = data.get('config') + # if config: + # errors = validate_schema_config(config) + # if errors: + # raise abort(400, errors) + schema_view_func = SchemaAPI.as_view('schemas') diff --git a/docs/schema.md b/docs/schema.md index 02d20579ff..9f7fb50f8d 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -90,3 +90,176 @@ Each schema is directly provided or created with the support of collaboration ph Every schema change is versioned so that it can adapt to changes in the data or other components provided by the collaborations. This practice also ensures the that the integrity of the older analysis records is maintained. Depending on the preference and work environment of the researcher, analysis information can be created and edited through a [submission form](./tutorials.md#the-cap-form) on the web interface or via the [API](./api.md). + + +# Schema Configuration + +The new versions of CAP schemas support hardcoded configuration options, which can be used either to perform certain tasks, +or certain checks. Right now the supported options are: + +- `reviewable`: shows if the analysis type is reviewable (used to add review options in the UI) +- `notifications`: an extended JSON that provides a lot of options regarding notifications/mails + +As the `notifications` is the most complicated of the two, we will explain its components in depth, and how they should be used. + + +### Notification Config Structure + +The basic structure that is followed is this: + + "config": { + "notifications": { + "actions": { + "publish": {...}, + "review": {...} + } + } + } + +What this means essentially is that when certain actions are triggered on a deposit, in this case when it is `published` or +`reviewed`, the backend will handle the mail/notifications according to what is configured. The fields in each one of those +actions represent a part of the required context of the mail: + +- `subject` +- `body` +- `recipients` + +Let's provide some examples for each one, and explain their usage. + + +##### 1. Subject + +This field provides configuration regarding the `subject` of the mail. It looks like this: + + "subject": { + "template": "Subject with {{ title }} and id {{ published_id }}", + "ctx": [{ + "name": "title", + "path": "general_title" + }, { + "method": "published_id" + }] + } + +The user needs to provide a path to a `template` or `template_file`, that will be populated by the context (`ctx`). +The context variables can be accessed in 2 different ways by their respective key names: + +- `path` uses the deposit path that holds a variable +- `method` uses a custom method that can be created and accessed in the `cap.modules.mail.custom.subject.py` file + +The subject `template_file` should be added in the `cap.modules.mail.templates.mail.subject` folder. +The template can be omitted since there are defaults that can be used for each action. + + +##### 2. Message/Body + +Similar to the subject, we provide a template and a context for the message/body of tha notification, as follows: + + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "title", + "path": "general_title" + }, { + "method": "submitter_email" + }], + "base_template_file": "mail/analysis_plain_text.html", + "plain": false + } + +The same rules are being followed, with the exception that if no message/body is provided, then the notification will only +contain a header with general information, and no specific info. The template file should be added in the +`cap.modules.mail.templates.mail.body` folder, and the custom functions in the `cap.modules.mail.custom.body.py` file. + + +##### 3. Recipients + +Due to the complicated nature and requirements in adding recipients, the configuration follows more steps. Let's take a +look at an example: + + "recipients": { + "bcc": [ + { + "method": "get_owner" + }, { + "mails": { + "default": ["some-recipient-placeholder@cern.ch"], + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "type": "path", + "path": "analysis_context.cadi_id" + }] + }] + } + }, { + "checks": [{ + "path": "parton_distribution_functions", + "if": "exists", + "value": true + }], + "mails": { + "default": ["pdf-forum-placeholder@cern.ch"] + } + } + ] + } + +The `recipients` field differentiates 3 categories: +- `recipients` +- `cc` +- `bcc` + +All 3 can be used to send a mail, and have their own mails and rules about how they will be added. The rules are in 3 categories: + +- `mails`: the mails in the list will be added +- `method`: a method that returns a list of mail (for complicated options) +- `checks`: mails will be added if a certain condition is true + + +Regarding the conditions, each one of them has a collection of checks that need to pass, in order for the mails to be added +in the recipients list. Each condition object contains: + +- `op`: the operation to be done in the check results. If `and`, all the checks need to be true, and if `or`, just one of them is enough +- `mails`: the mails that will be added +- `checks`: a list of checks + +Each `check` object needs to have: + +- `path`: the path of variable, found in the deposit +- `value`: the value that needs to be the result of the method used +- `if`: the method that will be used + +If for example you need to make sure that the deposit has a `general_title`, then the check would be + + { + "path": "general_title", + "if": "exists", + "value": true ## DEFAULT, NOT REQUIRED + } + +if you need to make sure that a certain value does not appear in a list, then + + { + "path": "path.to.some_list", + "if": "is_not_in", + "value": "some value' + } + +Here is a list of currently supported conditions: + +| name (if) | expected value | +| -------------- |:--------------------------:| +| equals | a string | +| exists | None (default true) | +| is_in | an iterable (list/string) | +| is_not_in | an iterable (list/string | +| is_egroup_member | an egroup name | + +The supported conditions and checks can be found in the `cap.modules.mail.conditions.py` file + + + +### Notification Config Examples + diff --git a/scripts/cap.wordlist b/scripts/cap.wordlist index a4100fdcdd..744761d61a 100644 --- a/scripts/cap.wordlist +++ b/scripts/cap.wordlist @@ -12,9 +12,11 @@ cli CMS cms config +configs devops docker docs +docstrings formBuilder GNU GPL @@ -31,6 +33,7 @@ oauthclient OIDC orcid prescale +py readme reana recid diff --git a/tests/conftest.py b/tests/conftest.py index 788a442279..b63f538f91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,13 +126,33 @@ def default_config(): TESTING=True, APP_GITLAB_OAUTH_ACCESS_TOKEN='testtoken', MAIL_DEFAULT_SENDER="analysis-preservation-support@cern.ch", - CMS_STATS_COMMITEE_AND_PAGS={'key': {'contacts': []}}, + CADI_REGEX="^[A-Z]{3}-[0-9]{2}-[0-9]{3}$", + CMS_STATS_COMMITEE_AND_PAGS=dict({ + 'ABC': { + 'params': { + 'primary': 'Admin Rev 1', + 'secondary': 'Admin Rev 2' + }, + 'contacts': [ + 'admin-rev-1@cern0.ch', + 'admin-rev-2@cern0.ch', + 'admin-rev-group-1@cern0.ch', + ] + }, + 'ABD': { + 'contacts': [ + + ] + } + }), PDF_FORUM_MAIL='pdf-forum-test@cern0.ch', CONVENERS_ML_MAIL='ml-conveners-test@cern0.ch', CONVENERS_ML_JIRA_MAIL='ml-conveners-jira-test@cern0.ch', CMS_HYPERNEWS_EMAIL_FORMAT='hn-cms-{}@cern0.ch', GITHUB_CAP_TOKEN="testtokengithub", - GITLAB_CAP_TOKEN="testtokengitlab") + GITLAB_CAP_TOKEN="testtokengitlab", + CMS_STATS_QUESTIONNAIRE_ADMIN_ROLES='cms-admins@cern0.ch') + @pytest.fixture(scope='session') diff --git a/tests/data/mail_configs.py b/tests/data/mail_configs.py new file mode 100644 index 0000000000..ce672881c3 --- /dev/null +++ b/tests/data/mail_configs.py @@ -0,0 +1,1543 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2020 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +"""Config example data.""" + +from cap.modules.mail.custom import body + + +EMPTY_CONFIG = {} + +EMPTY_CONFIG_ASSERTIONS = { + "response": 202, + "outbox": {}, +} + +DEFAULT_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + {"method": "get_submitter"}, + { + "mails": { + "default": [ + "user1@cern0.ch", + "user3@cern0.ch", + "user4@cern0.ch", + ] + } + }, + ] + } + } + ], + "review": [], + } + } +} + +DEFAULT_CONFIG_ASSERTIONS = { + "ctx": {"pid": {"type": "response", "path": "recid"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "New published document")], + "html": [ + ("in", "can check it"), + ( + "in", + 'here', + ), + ], + "recipients": { + "recipients": [ + ("in", "superuser@cern.ch"), + ("in", "user1@cern0.ch"), + ("in", "user4@cern0.ch"), + ] + }, + } + }, +} + +DEFAULT_CONFIG_PLAIN = { + "notifications": { + "actions": { + "publish": [ + { + "body": {"plain": True}, + "recipients": { + "recipients": [ + {"method": "get_submitter"}, + { + "mails": { + "default": [ + "user1@cern0.ch", + "user3@cern0.ch", + "user4@cern0.ch", + ] + } + }, + ] + }, + } + ], + "review": [], + } + } +} + +DEFAULT_CONFIG_PLAIN_ASSERTIONS = { + "ctx": {"pid": {"type": "response", "path": "recid"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "New published document")], + "body": [("in", "can check it"), ("in", "with id {pid}")], + "recipients": { + "recipients": [ + ("in", "superuser@cern.ch"), + ("in", "user1@cern0.ch"), + ("in", "user4@cern0.ch"), + ] + }, + } + }, +} + + +DEFAULT_CONFIG_WITH_ERRORS = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + {"method": "submitter"}, + { + "mails": [ + "user1@cern0.ch", + "user3@cern0.ch", + "user4@cern0.ch", + ] + }, + ] + } + } + ], + "review": [], + } + } +} + +DEFAULT_CONFIG_WITH_ERRORS_ASSERTIONS = { + "validationError": True, + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + # 0: { + # "subject": [( "in", "Questionnaire for {cadi_id} published")], + # "html": [( "in", "Message with cadi id: {cadi_id}.")], + # "recipients": { + # "recipients": [ + # ("in", "cms_user@cern.ch"), + # ("in", "user1@cern0.ch"), + # ("in", "user2@cern0.ch"), + # ("in", "user4@cern0.ch"), + # ] + # }, + # } + }, +} + +SIMPLE_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id }} published.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "body": { + "template": "Message with cadi id: {{ cadi_id }}.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} +SIMPLE_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "Questionnaire for {cadi_id} published")], + "html": [("in", "Message with cadi id: {cadi_id}.")], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ] + }, + } + }, +} + +SIMPLE_GLOBAL_CTX_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "ctx": [{"name": "cadi_id", "path": "analysis_context.cadi_id"}], + "subject": { + "template": "Questionnaire for {{ cadi_id }} published." + }, + "body": {"template": "Message with cadi id: {{ cadi_id }}."}, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} +SIMPLE_GLOBAL_CTX_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "Questionnaire for {cadi_id} published")], + "html": [("in", "Message with cadi id: {cadi_id}.")], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ] + }, + } + }, +} + +SIMPLE_GLOBAL_CTX_2_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "ctx": [ + {"name": "cadi_idd", "path": "analysis_context.cadi_id"}, + { + "name": "cadi_id_error", + "path": "analysis_context.cadi_id_wrong", + }, + ], + "subject": { + "template": "Questionnaire for {{ cadi_id }} published." + }, + "body": {"template": "Message with cadi id: {{ cadi_id }}."}, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} +SIMPLE_GLOBAL_CTX_2_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "Questionnaire for published")], + "html": [("in", "Message with cadi id: .")], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ] + }, + } + }, +} + +NESTED_CONDITION_WITH_ERRORS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "analysis_context.cadi_id", + "condition": "exists", + }, + { + "checks": [ + { + "path": "analysis_context", + "condition": "exists", + }, + { + "path": "test1", + "condition": "error_here", + }, + ] + }, + ], + "mails": {"default": ["test@cern.ch"]}, + } + ] + } + } + ] + } + } +} +NESTED_CONDITION_WITH_ERRORS_CONFIG_ASSERTIONS = { + "response": 202, + "outbox": {}, +} + + +HAS_PERMISSION_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "analysis_context.cadi_id", + "condition": "exists", + }, + { + "condition": "is_egroup_member", + "value": "cms-access", + }, + ], + "mails": {"default": ["test@cern.ch"]}, + } + ] + } + } + ] + } + } +} +HAS_PERMISSION_CONFIG_ASSERTIONS = { + "response": 202, + "outbox": { + 0: { + "recipients": { + "recipients": [ + ("in", "test@cern.ch"), + ] + }, + } + }, +} + + +WRONG_TEMPLATE_FILE_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template_file": "wrong/path/template.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} + +WRONG_TEMPLATE_FILE_CONFIG_ASSERTIONS = { + "response": 202, + "outbox": {}, +} + + +CTX_EXAMPLES_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template": """ + Published analysis + + CADI ID: {{cadi_id}} + ================== + + revision : {{revision}} | + draft_revision : {{draft_revision}} | + draft_id : {{draft_id}} | + published_id : {{published_id}} | + draft_url : {{draft_url}} | + published_url : {{published_url}} | + working_id : {{working_id}} | + working_url : {{working_url}} | + submitter_email : {{submitter_email}} | + creator_email : {{creator_email}} | + cms_stats_committee_by_pag : {{cms_stats_committee_by_pag}} | + get_cms_stat_recipients : {{get_cms_stat_recipients}} | + """, + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "draft_revision"}, + {"method": "draft_id"}, + {"method": "published_id"}, + {"method": "draft_url"}, + {"method": "published_url"}, + {"method": "working_id"}, + {"method": "working_url"}, + {"method": "submitter_email"}, + {"method": "creator_email"}, + {"method": "cms_stats_committee_by_pag"}, + {"method": "get_cms_stat_recipients"}, + ], + "plain": True, + }, + "recipients": { + "recipients": [ + "test-recipient@cern0.ch", + {"method": "get_cms_stat_recipients"}, + ] + }, + } + ], + "review": [], + } + } +} +CTX_EXAMPLES_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "New published document | CERN Analysis Preservation")], + "body": [("in", "CADI ID: {cadi_id}")], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ("in", "admin-rev-1@cern0.ch"), + ("in", "admin-rev-2@cern0.ch"), + ] + }, + } + }, +} + +WRONG_CTX_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template": """ + Published analysis + + CADI ID: {{cadi_id}} + WRONG CADI ID: {{wrong_cadi_id}} + WRONG CADI ID 2: {{wrong_cadi_id_2}} + """, + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + { + "name": "wrong_cadi_id", + "path": "analysis_context.cadi_id.0", + }, + { + "name": "wrong_cadi_id_2", + "path": "analysis_context.cadi_id.[]", + }, + ], + "plain": True, + }, + "recipients": { + "recipients": [ + "test-recipient@cern0.ch", + {"method": "get_cms_stat_recipients"}, + { + "mails": { + "formatted": [ + { + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "path": "analysis_context.cadi_id", + } + ], + }, + { + "template": "{% if cadi_id %}mail-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "path": "analysis_context.cadi_id", + } + ], + }, + ] + } + }, + ] + }, + } + ], + "review": [], + } + } +} +WRONG_CTX_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "New published document | CERN Analysis Preservation")], + "body": [("in", "CADI ID: {cadi_id}")], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ("in", "admin-rev-2@cern0.ch"), + ("in", "hn-cms-ABC-11-111@cern0.ch"), + ("in", "mail-ABC-11-111@cern0.ch"), + ] + }, + } + }, +} + +WRONG_TEMPLATE_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template": """ + Published analysis + + CADI ID: {{cadi_id}} + WRONG CADI ID: {{wrong_cadi_id}} + WRONG CADI ID 2: {{wrong_cadi_id_2}} + """, + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + { + "name": "wrong_cadi_id", + "path": "analysis_context.cadi_id.0", + }, + { + "name": "wrong_cadi_id_2", + "path": "analysis_context.cadi_id.[]", + }, + ], + "plain": True, + }, + "recipients": { + "recipients": [ + "test-recipient@cern0.ch", + # 3, + [ + "test-recipient_error@cern0.ch", + "test-recipient_error2@cern0.ch", + ], + {"method": "get_cms_stat_recipients"}, + { + "mails": { + # "default": ['user7@cern0.ch', 4], + "formatted": [ + { + "templfate": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "path": "analysis_context.cadi_id", + } + ], + }, + { + "template": "{% if cadi_id %}mail-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "path": "analysis_context.cadi_id", + } + ], + }, + ] + } + }, + ] + }, + } + ], + "review": [], + } + } +} +WRONG_TEMPLATE_CONFIG_ASSERTIONS = { + "validationError": True, + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "subject": [("in", "New published document | CERN Analysis Preservation")], + "body": [("in", "CADI ID: {cadi_id}")], + "recipients": { + "recipients": [ + ("in", "user7@cern0.ch"), + ("in", "test-recipient@cern0.ch"), + ("in", "admin-rev-2@cern0.ch"), + ("in", "mail-ABC-11-111@cern0.ch"), + ] + }, + } + }, +} +MUTLIPLE_PUBLISH_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - ' + '{{ "New Version of Published Analysis" if revision > 0 else "New published document" }} ' + "| CERN Analysis Preservation", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "published_id"}, + ], + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "submitter_email"}, + ], + # "base_template": "mail/analysis_plain_text.html", + "plain": True, + }, + "recipients": { + "recipients": [ + "test-recipient@cern0.ch", + { + "checks": [ + { + "path": "analysis_context.cadi_id", + "condition": "exists", + } + ], + "mails": { + "formatted": [ + { + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "type": "path", + "path": "analysis_context.cadi_id", + } + ], + } + ] + }, + }, + ] + }, + }, + { + "subject": { + "template_file": "mail/subject/questionnaire_subject_published.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "published_id"}, + ], + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "submitter_email"}, + ], + }, + "recipients": { + "recipients": [ + {"method": "get_owner"}, + {"method": "get_submitter"}, + ], + "bcc": [ + {"method": "get_cms_stat_recipients"}, + { + "op": "and", + "checks": [ + {"path": "ml_app_use", "condition": "exists"} + ], + "mails": { + "default": [ + "ml-conveners-test@cern0.ch", + "ml-conveners-jira-test@cern0.ch", + ] + }, + }, + ], + }, + }, + ] + } + } +} + +MUTLIPLE_PUBLISH_CONFIG_ASSERTIONS = { + "ctx": { + "cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}, + "published_id": {"type": "response", "path": "recid"}, + }, + "response": 202, + "outbox": { + 0: { + "subject": [ + ("in", "Questionnaire for {cadi_id} {published_id}"), + ("in", "New published document"), + ], + "body": [ + ("in", "ancode={cadi_id}"), + ], + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ] + }, + }, + 1: {}, + }, +} + +NESTED_CONDITIONS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "parton_distribution_functions", + "condition": "exists", + } + ], + "mails": {"default": ["pdf-forum-placeholder@cern.ch"]}, + }, + { + "op": "or", + "checks": [ + { + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes", + }, + # {"path": "ml_app_use", "condition": "exists"}, + { + "path": "ml_survey.options", + "condition": "equals", + "value": "Yes", + }, + { + "op": "and", + "checks": [ + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "exists", + }, + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "is_not_in", + "value": "No", + }, + ], + }, + ], + "mails": { + "default": ["nested-conditions-mail@cern.ch"] + }, + }, + ] + } + }, + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "parton_distribution_functions", + "condition": "exists", + } + ], + "mails": {"default": ["wrong-condition@cern.ch"]}, + }, + { + "checks": [ + { + "path": "parton_distribution_funcions", + "condition": "existds", + } + ], + "mails": {"default": ["wrong-condition@cern.ch"]}, + }, + ] + } + }, + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "parton_distribution_functions", + "condition": "exists", + } + ], + "method": ["get_submitter"], + "mails": { + "default": ["condition-with-methods@cern.ch"] + }, + } + ] + } + }, + ] + } + } +} + + +NESTED_CONDITIONS_CONFIG_ASSERTIONS = { + "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "recipients": { + "recipients": [ + ("in", "pdf-forum-placeholder@cern.ch"), + ("in", "nested-conditions-mail@cern.ch"), + ] + }, + }, + 1: { + "recipients": {"recipients": [("in", "wrong-condition@cern.ch")]}, + }, + 2: { + "recipients": { + "recipients": [ + ("in", "condition-with-methods@cern.ch"), + ("in", "superuser@cern.ch"), + ] + }, + }, + }, +} + +CONDITION_THAT_DOESNT_EXIST_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "test", + "condition": "doesnt_exist", + } + ], + "mails": {"default": ["test@cern.ch"]}, + } + ] + } + } + ] + } + } +} + +NO_RECIPIENTS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id }} published.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "body": { + "template": "Message with cadi id: {{ cadi_id }}.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + } + ] + } + } +} + +SUBJECT_METHOD_DOESNT_EXIST_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "subject": {"method": "get_subject_not_here"}, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} + +SUBJECT_MISSING_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template": "Message with cadi id: {{ cadi_id }}.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} + +BODY_MISSING_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": "Questionnaire for {{ cadi_id }} published.", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"} + ], + }, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} + +MULTIPLE_RECIPIENTS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + {"recipients": {"recipients": ["test-recipient@cern0.ch"]}}, + {"recipients": {"bcc": ["test-recipient-bcc@cern0.ch"]}}, + ] + } + } +} + +CONDITION_RECIPIENTS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "analysis_context.wg", + "condition": "equals", + "value": "ABC", + } + ], + "mails": { + "default": ["test-recipient@cern0.ch"], + }, + } + ] + } + }, + {"recipients": {"bcc": ["test-recipient-bcc@cern0.ch"]}}, + ] + } + } +} + +CONDITION_RECIPIENTS_CONFIG_ASSERTIONS = { + # "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ] + }, + }, + 1: { + "recipients": { + "bcc": [ + ("in", "test-recipient-bcc@cern0.ch"), + ] + }, + }, + }, +} + +WRONG_CONDITION_RECIPIENTS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "analysis_context.wg", + "condition": "equals", + "value": "ABdC", + } + ], + "mails": { + "default": ["test-recipient@cern0.ch"], + }, + } + ] + } + }, + {"recipients": {"bcc": ["test-recipient-bcc@cern0.ch"]}}, + ] + } + } +} + +WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS = { + # "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + # 0: { + # "recipients": { + # "recipients": [ + # # ("in", "test-recipient@cern0.ch"), + # ] + # }, + # }, + 0: { + "recipients": { + "bcc": [ + ("in", "test-recipient-bcc@cern0.ch"), + ] + }, + } + }, +} + +METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "recipients": { + "recipients": [ + { + "checks": [ + { + "path": "analysis_context.wg", + "condition": "equals", + "value": "ABC", + } + ], + "method": ["get_owner", "get_submitter"], + "mails": { + "default": ["test-recipient@cern0.ch"], + }, + } + ] + } + }, + {"recipients": {"bcc": ["test-recipient-bcc@cern0.ch"]}}, + ] + } + } +} + +METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS = { + # "ctx": {"cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}}, + "response": 202, + "outbox": { + 0: { + "recipients": { + "recipients": [ + ("in", "test-recipient@cern0.ch"), + ("in", "cms_user@cern.ch"), + ("in", "superuser@cern.ch"), + ] + }, + }, + 1: { + "recipients": { + "bcc": [ + ("in", "test-recipient-bcc@cern0.ch"), + ] + }, + }, + }, +} + + +CTX_METHOD_MISSING_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template": "Message with cadi id: {{ cadi_id }} and val {{ new_val }}", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "new_val"}, + ], + }, + "recipients": {"recipients": ["test-recipient@cern0.ch"]}, + } + ] + } + } +} + +CMS_STATS_QUESTIONNAIRE = { + "notifications": { + "actions": { + "publish": [ + { + "subject": { + "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - {{ "New Version of Published Analysis" if revision > 0 else "New published document" }} | CERN Analysis Preservation', + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "published_id"}, + ], + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "cms_stats_committee_by_pag"}, + {"method": "submitter_email"}, + ], + }, + "recipients": { + "bcc": [ + {"method": "get_cms_stat_recipients"}, + {"method": "get_owner"}, + {"method": "get_submitter"}, + { + "checks": [ + { + "path": "parton_distribution_functions", + "condition": "exists", + } + ], + "mails": {"default": ["pdf-forum-placeholder@cern.ch"]}, + }, + { + "op": "or", + "checks": [ + { + "path": "multivariate_discriminants.mva_use", + "condition": "equals", + "value": "Yes", + }, + {"path": "ml_app_use", "condition": "exists"}, + { + "path": "ml_survey.options", + "condition": "equals", + "value": "Yes", + }, + { + "op": "and", + "checks": [ + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "exists", + }, + { + "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + "condition": "is_not_in", + "value": "No", + }, + ], + }, + ], + "mails": { + "default": ["cms-conveners-placeholder@cern.ch"] + }, + }, + ] + }, + }, + { + "subject": { + "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - {{ "New Version of Published Analysis" if revision > 0 else "New published document" }} | CERN Analysis Preservation', + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "published_id"}, + ], + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "cms_stats_committee_by_pag"}, + {"method": "submitter_email"}, + ], + "base_template": "mail/body/analysis_plain_text.html", + "plain": True, + }, + "recipients": { + "bcc": [ + { + "checks": [ + { + "path": "analysis_context.cadi_id", + "condition": "exists", + } + ], + "mails": { + "formatted": [ + { + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [ + { + "name": "cadi_id", + "path": "analysis_context.cadi_id", + } + ], + } + ] + }, + } + ] + }, + }, + # { + # "subject": { + # "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - {{ "New Version of Published Analysis" if revision > 0 else "New published document" }} | CERN Analysis Preservation', + # }, + # "body": { + # "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + # "ctx": [ + # {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + # {"name": "title", "path": "general_title"}, + # {"method": "published_url"}, + # {"method": "cms_stats_committee_by_pag"}, + # {"method": "submitter_email"}, + # ], + # "base_template": "mail/analysis_plain_text.html", + # "plain": true, + # }, + # "recipients": { + # "bcc": [ + # { + # "op": "or", + # "checks": [ + # { + # "path": "multivariate_discriminants.mva_use", + # "condition": "equals", + # "value": "Yes", + # }, + # {"path": "ml_app_use", "condition": "exists"}, + # { + # "path": "ml_survey.options", + # "condition": "equals", + # "value": "Yes", + # }, + # { + # "op": "and", + # "checks": [ + # { + # "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + # "condition": "exists", + # }, + # { + # "path": "multivariate_discriminants.use_of_centralized_cms_apps.options", + # "condition": "is_not_in", + # "value": "No", + # }, + # ], + # }, + # ], + # "mails": { + # "default": [ + # "cms-conveners-jira-placeholder@cern.ch" + # ] + # }, + # } + # ] + # }, + # }, + ], + # "review": [ + # { + # "subject": { + # "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + # "ctx": [ + # {"name": "cadi_id", "path": "analysis_context.cadi_id"} + # ], + # }, + # "body": { + # "template_file": "mail/body/experiments/cms/questionnaire_message_review.html", + # "ctx": [ + # {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + # {"name": "title", "path": "general_title"}, + # {"method": "working_url"}, + # {"method": "creator_email"}, + # {"method": "submitter_email"}, + # ], + # }, + # "recipients": { + # "bcc": [{"method": "get_owner"}, {"method": "get_submitter"}] + # }, + # }, + # { + # "subject": { + # "template": "Questionnaire for {{ cadi_id }} - New Review on Analysis | CERN Analysis Preservation", + # "ctx": [ + # {"name": "cadi_id", "path": "analysis_context.cadi_id"} + # ], + # }, + # "body": { + # "template_file": "mail/body/experiments/cms/questionnaire_message_review_plain.html", + # "ctx": [ + # {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + # {"name": "title", "path": "general_title"}, + # {"method": "working_url"}, + # {"method": "creator_email"}, + # {"method": "submitter_email"}, + # ], + # "base_template": "mail/analysis_plain_text.html", + # "plain": true, + # }, + # "recipients": { + # "bcc": [ + # { + # "checks": [ + # { + # "path": "analysis_context.cadi_id", + # "condition": "exists", + # }, + # { + # "path": "parton_distribution_functions", + # "condition": "is_egroup_member", + # "value": "comittee-mail", + # }, + # ], + # "mails": { + # "formatted": [ + # { + # "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern.ch{% endif %}", + # "ctx": [ + # { + # "name": "cadi_id", + # "path": "analysis_context.cadi_id", + # } + # ], + # } + # ] + # }, + # } + # ] + # }, + # }, + # ], + } + } +} + +CMS_STATS_QUESTIONNAIRE_ASSERTIONS = { + "ctx": { + "cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}, + "published_id": {"type": "response", "path": "recid"}, + }, + "response": 202, + "outbox": { + 0: { + "subject": [ + ("in", "Questionnaire for {cadi_id} {published_id}"), + ("in", "New published document"), + ], + "html": [ + ("in", "ancode={cadi_id}"), + ("in", "Admin Rev 1 (primary)"), + ("in", "Admin Rev 2 (secondary)"), + ], + "recipients": { + "bcc": [ + ("in", "admin-rev-1@cern0.ch"), + ("in", "admin-rev-2@cern0.ch"), + ("in", "cms_user@cern.ch"), + ("in", "superuser@cern.ch"), + ] + }, + }, + # 1: { + # "subject": [ + # ("in", "Questionnaire for {cadi_id} {published_id}"), + # ("in", "New published document"), + # ], + # "body": [ + # ("in", "ancode={cadi_id}"), + # ("in", "Admin Rev 1 (primary)"), + # ("in", "Admin Rev 2 (secondary)"), + # ], + # "recipients": { + # "bcc": [ + # ("in", "hn-cms-ABC-11-111@cern0.ch") + # ] + # }, + # }, + }, +} + +WRONG_BASE_TEMPLATE_CONFIG = { + "notifications": { + "actions": { + "publish": [ + { + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "submitter_email"}, + ], + "base_template": "mail/wrong_template.html", + "plain": True, + }, + "recipients": { + "recipients": [ "test-recipient@cern0.ch" ] + }, + }, + { + "subject": { + "template_file": "mail/subject/questionnaire_subject_published.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"method": "revision"}, + {"method": "published_id"}, + ], + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [ + {"name": "cadi_id", "path": "analysis_context.cadi_id"}, + {"name": "title", "path": "general_title"}, + {"method": "published_url"}, + {"method": "submitter_email"}, + ], + }, + "recipients": { + "recipients": [ + {"method": "get_owner"}, + {"method": "get_submitter"}, + ], + "bcc": [ + {"method": "get_cms_stat_recipients"}, + { + "op": "and", + "checks": [ + {"path": "ml_app_use", "condition": "exists"} + ], + "mails": { + "default": [ + "ml-conveners-test@cern0.ch", + "ml-conveners-jira-test@cern0.ch", + ] + }, + }, + ], + }, + }, + ] + } + } +} + +WRONG_BASE_TEMPLATE_CONFIG_ASSERTIONS = { + "ctx": { + "cadi_id": {"type": "deposit", "path": "analysis_context.cadi_id"}, + "published_id": {"type": "response", "path": "recid"}, + }, + "response": 202, + "outbox_length": 1, + "outbox": { + 0: { + "recipients": { + "recipients": [ + ("in", "cms_user@cern.ch"), + ("in", "superuser@cern.ch"), + ], + "bcc": [ + ("in", "ml-conveners-test@cern0.ch"), + ("in", "ml-conveners-jira-test@cern0.ch"), + ] + }, + } + } +} \ No newline at end of file diff --git a/tests/integration/mail/test_post_action.py b/tests/integration/mail/test_post_action.py new file mode 100644 index 0000000000..5fdffcc6be --- /dev/null +++ b/tests/integration/mail/test_post_action.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2020 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +"""Tests for mail.""" + +import json +from pytest import raises, mark +from mock import patch + +from invenio_deposit.signals import post_action +from cap.modules.schemas.helpers import ValidationError + +from data.mail_configs import ( + EMPTY_CONFIG, EMPTY_CONFIG_ASSERTIONS, + DEFAULT_CONFIG, DEFAULT_CONFIG_ASSERTIONS, + DEFAULT_CONFIG_PLAIN, DEFAULT_CONFIG_PLAIN_ASSERTIONS, + DEFAULT_CONFIG_WITH_ERRORS, DEFAULT_CONFIG_WITH_ERRORS_ASSERTIONS, + SIMPLE_CONFIG, SIMPLE_CONFIG_ASSERTIONS, + SIMPLE_GLOBAL_CTX_CONFIG, SIMPLE_GLOBAL_CTX_CONFIG_ASSERTIONS, + SIMPLE_GLOBAL_CTX_2_CONFIG, SIMPLE_GLOBAL_CTX_2_CONFIG_ASSERTIONS, + WRONG_TEMPLATE_FILE_CONFIG, WRONG_TEMPLATE_FILE_CONFIG_ASSERTIONS, + NESTED_CONDITION_WITH_ERRORS_CONFIG, NESTED_CONDITION_WITH_ERRORS_CONFIG_ASSERTIONS, + CTX_EXAMPLES_CONFIG, CTX_EXAMPLES_CONFIG_ASSERTIONS, + WRONG_CTX_CONFIG, WRONG_CTX_CONFIG_ASSERTIONS, + CONDITION_RECIPIENTS_CONFIG, CONDITION_RECIPIENTS_CONFIG_ASSERTIONS, + WRONG_CONDITION_RECIPIENTS_CONFIG, WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS, + METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG, METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS, + MUTLIPLE_PUBLISH_CONFIG, MUTLIPLE_PUBLISH_CONFIG_ASSERTIONS, + NESTED_CONDITIONS_CONFIG, NESTED_CONDITIONS_CONFIG_ASSERTIONS, + WRONG_TEMPLATE_CONFIG, WRONG_TEMPLATE_CONFIG_ASSERTIONS, + HAS_PERMISSION_CONFIG, HAS_PERMISSION_CONFIG_ASSERTIONS, + CMS_STATS_QUESTIONNAIRE, CMS_STATS_QUESTIONNAIRE_ASSERTIONS, + WRONG_BASE_TEMPLATE_CONFIG, WRONG_BASE_TEMPLATE_CONFIG_ASSERTIONS +) +from cap.modules.mail.utils import path_value_equals + +@mark.parametrize("config,expected",[ + (EMPTY_CONFIG, EMPTY_CONFIG_ASSERTIONS), + (DEFAULT_CONFIG, DEFAULT_CONFIG_ASSERTIONS), + (DEFAULT_CONFIG_PLAIN, DEFAULT_CONFIG_PLAIN_ASSERTIONS), + (DEFAULT_CONFIG_WITH_ERRORS, DEFAULT_CONFIG_WITH_ERRORS_ASSERTIONS), + (SIMPLE_CONFIG, SIMPLE_CONFIG_ASSERTIONS), + (SIMPLE_GLOBAL_CTX_CONFIG, SIMPLE_GLOBAL_CTX_CONFIG_ASSERTIONS), + (SIMPLE_GLOBAL_CTX_2_CONFIG, SIMPLE_GLOBAL_CTX_2_CONFIG_ASSERTIONS), + (WRONG_TEMPLATE_FILE_CONFIG, WRONG_TEMPLATE_FILE_CONFIG_ASSERTIONS), + (NESTED_CONDITION_WITH_ERRORS_CONFIG, NESTED_CONDITION_WITH_ERRORS_CONFIG_ASSERTIONS), + (CTX_EXAMPLES_CONFIG, CTX_EXAMPLES_CONFIG_ASSERTIONS), + (WRONG_CTX_CONFIG, WRONG_CTX_CONFIG_ASSERTIONS), + (WRONG_TEMPLATE_CONFIG, WRONG_TEMPLATE_CONFIG_ASSERTIONS), + (CONDITION_RECIPIENTS_CONFIG, CONDITION_RECIPIENTS_CONFIG_ASSERTIONS), + (WRONG_CONDITION_RECIPIENTS_CONFIG, WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS), + (METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG, METHOD_AND_WRONG_CONDITION_RECIPIENTS_CONFIG_ASSERTIONS), + (MUTLIPLE_PUBLISH_CONFIG, MUTLIPLE_PUBLISH_CONFIG_ASSERTIONS), + (NESTED_CONDITIONS_CONFIG, NESTED_CONDITIONS_CONFIG_ASSERTIONS), + (HAS_PERMISSION_CONFIG, HAS_PERMISSION_CONFIG_ASSERTIONS), + (CMS_STATS_QUESTIONNAIRE, CMS_STATS_QUESTIONNAIRE_ASSERTIONS), + (WRONG_BASE_TEMPLATE_CONFIG, WRONG_BASE_TEMPLATE_CONFIG_ASSERTIONS) +]) +@patch('cap.modules.mail.conditions.get_cern_extra_data_egroups', lambda x: ['cms-access']) +def test_post_action_mail(app, users, create_deposit, create_schema, client, auth_headers_for_user, config, expected): + if expected.get("validationError"): + with raises(ValidationError): + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) + return + else: + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) + + creator = users['cms_user'] + user = users['superuser'] + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit( + creator, + 'cms-stats-questionnaire', + { + '$schema': 'https://analysispreservation.cern.ch/schemas/' + 'deposits/records/cms-stats-questionnaire-v0.0.1.json', + 'general_title': 'test analysis', + 'analysis_context': { + 'cadi_id': 'ABC-11-111', + 'wg': 'ABC' + }, + 'parton_distribution_functions': { + 'test': "exists" + }, + 'ml_app_use': ['not empty'], + 'multivariate_discriminants': { + 'use_of_centralized_cms_apps': { + 'options': "Yes" + } + } + }, + experiment='CMS' + ) + + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + if expected.get("response"): + assert resp.status_code == expected.get("response") + pid = resp.json['recid'] + + ctx = {} + for ctx_key, ctx_item in expected.get("ctx", {}).items(): + ctx_type = ctx_item.get("type") + if ctx_type == "response": + ctx[ctx_key] = _path_value_equals(ctx_item['path'], resp.json) + elif ctx_type == "deposit": + ctx[ctx_key] = path_value_equals(ctx_item['path'], deposit) + + if not expected.get("outbox"): + assert len(outbox) == 0 + + if expected.get("outbox_length"): + assert len(outbox) == expected.get("outbox_length") + + for outbox_key, outbox_assert in expected.get("outbox", {}).items(): + _outbox = outbox[outbox_key] + + for _assertion in outbox_assert.get("subject", []): + if _assertion[0] is "in": + assert _assertion[1].format(**ctx) in _outbox.subject + elif _assertion[0] is "equals": + assert _assertion[1].format(**ctx) == _outbox.subject + + for _assertion in outbox_assert.get("body", []): + assert _outbox.body + if _assertion[0] is "in": + assert _assertion[1].format(**ctx) in _outbox.body + elif _assertion[0] is "equals": + assert _assertion[1].format(**ctx) == _outbox.body + + for _assertion in outbox_assert.get("html", []): + assert _outbox.html + + if _assertion[0] is "in": + assert _assertion[1].format(**ctx) in _outbox.html + elif _assertion[0] is "equals": + assert _assertion[1].format(**ctx) == _outbox.html + elif _assertion[0] is "pid": + if _assertion[1] is "in": + assert _assertion[2].format(pid=pid) in _outbox.html + elif _assertion[1] is "equals": + assert _assertion[2].format(pid=pid) == _outbox.html + + for rec_type in ["recipients", "bcc", "cc"]: + for _assertion in outbox_assert.get("recipients", {}).get(rec_type, []): + _list = [] + if rec_type is "recipients": + _list = _outbox.recipients + elif rec_type is "bcc": + _list = _outbox.bcc + elif rec_type is "cc": + _list = _outbox.cc + + if _assertion[0] is "in": + assert _assertion[1] in _list + elif _assertion[0] is "notin": + assert _assertion[1] not in _list + +def _path_value_equals(path, record): + """Given a string path, retrieve the JSON item.""" + paths = path.split(".") + data = record + try: + for i in range(0, len(paths)): + data = data[paths[i]] + return data + except: + return None diff --git a/tests/integration/test_review_deposit.py b/tests/integration/test_review_deposit.py index 7a266ed41b..f422bfc180 100644 --- a/tests/integration/test_review_deposit.py +++ b/tests/integration/test_review_deposit.py @@ -24,7 +24,10 @@ # or submit itself to any jurisdiction. """Integration tests for deposit review.""" +import pytest + import json +from cap.modules.deposit.api import CAPDeposit default_headers = [('Content-Type', 'application/json'), ('Accept', 'application/form+json')] @@ -234,3 +237,82 @@ def test_deposit_review_create_reviewable( headers=[('Accept', 'application/form+json')] + auth_headers_for_user(other_user)) assert resp.status_code == 403 + + +def test_review_and_published_revision_ids( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', + version="0.0.1", config={"reviewable": True}) + + with app.app_context(): + deposit = create_deposit( + user, + 'cms-analysis', + { + '$schema': 'https://analysispreservation.cern.ch/schemas/' + 'deposits/records/cms-stats-questionnaire-v0.0.1.json', + 'general_title': 'test analysis', + 'analysis_context': { + 'cadi_id': 'ABC-11-111' + } + }, + experiment='CMS' + ) + depid = deposit["_deposit"]["id"] + + # 1. review, revision should throw error (not published yet) + client.post( + f'/deposits/{depid}/actions/review', + data=json.dumps({"type": "request_changes", "body": "test"}), + headers=auth_headers_for_user(user) + default_headers + ) + + with pytest.raises(KeyError): + rec = CAPDeposit.get_record(deposit.id) + rec.fetch_published() + + # 2. publish, revision should be 0 + client.post( + f"/deposits/{depid}/actions/publish", + headers=auth_headers_for_user(user) + ) + + rec = CAPDeposit.get_record(deposit.id) + _, record = rec.fetch_published() + assert record.revision_id == 0 + + # 3. review after publish, revision should be 0 + client.post( + f'/deposits/{depid}/actions/review', + data=json.dumps({"type": "request_changes", "body": "test"}), + headers=auth_headers_for_user(user) + default_headers + ) + + _, record = rec.fetch_published() + assert record.revision_id == 0 + + # 4. edit, in order to publish again, on re-publish revision should be 1 + client.post( + f"/deposits/{depid}/actions/edit", + headers=auth_headers_for_user(user) + ) + + client.post( + f"/deposits/{depid}/actions/publish", + headers=auth_headers_for_user(user) + ) + + rec = CAPDeposit.get_record(deposit.id) + _, record = rec.fetch_published() + assert record.revision_id == 1 + + # 5. review after re-publish, revision should be 1 + client.post( + f'/deposits/{depid}/actions/review', + data=json.dumps({"type": "request_changes", "body": "test"}), + headers=auth_headers_for_user(user) + default_headers + ) + + _, record = rec.fetch_published() + assert record.revision_id == 1 diff --git a/tests/integration/test_schemas_views.py b/tests/integration/test_schemas_views.py index 70176ae86b..bdec08969f 100644 --- a/tests/integration/test_schemas_views.py +++ b/tests/integration/test_schemas_views.py @@ -793,7 +793,7 @@ def test_post_when_validation_errors_returns_400_2( assert resp.status_code == 400 - assert resp.json['message'] == 'ERROR: Invalid \'config\' object.' + assert resp.json['message'] == 'Schema configuration validation error' def test_post_when_validation_errors_returns_400_3( client, db, users, auth_headers_for_user, json_headers): @@ -822,7 +822,7 @@ def test_post_when_validation_errors_returns_400_3( assert resp.status_code == 400 - assert resp.json['message'] == 'ERROR: Invalid \'config\' object.' + assert resp.json['message'] == 'Schema configuration validation error' assert len(resp.json['errors']) == 8 assert {'field': ['repositories', 'baaam'], 'message': "'' is not of type 'object'"} in resp.json['errors'] @@ -1022,7 +1022,7 @@ def test_post_with_invalid_config_validation_gitlab(client, db, users, auth_head ) assert resp.status_code == 400 - assert resp.json['message'] == 'ERROR: Invalid \'config\' object.' + assert resp.json['message'] == 'Schema configuration validation error' ##################################### @@ -1230,9 +1230,98 @@ def test_put_when_not_an_schema_owner_returns_403( assert resp.status_code == 403 -##################################### -# api/jsonschemas/{id}/{version} [DELETE] -##################################### +def test_put_schema_with_valid_config(client, db, auth_headers_for_user, users, json_headers): + owner = users['superuser'] + schema = dict( + name='new-schema', + version='1.0.0', + deposit_schema={'title': 'deposit_schema'}, + config={'reviewable': True}, + is_indexed=True, + use_deposit_as_record=True, + ) + + resp = client.post( + '/jsonschemas/', + data=json.dumps(schema), + headers=json_headers + auth_headers_for_user(owner), + ) + + assert resp.status_code == 200 + + +def test_put_schema_with_invalid_config(client, db, auth_headers_for_user, users, json_headers): + owner = users['superuser'] + schema = dict( + name='new-schema', + version='1.0.0', + deposit_schema={'title': 'deposit_schema'}, + config={ + 'reviewable': 123, + 'notifications': { 'wrong_key': {}}, + 'wrong_property': { 'title': "test"} + }, # INVALID, SHOULD FAIL + is_indexed=True, + use_deposit_as_record=True, + ) + + resp = client.post( + '/jsonschemas/', + data=json.dumps(schema), + headers=json_headers + auth_headers_for_user(owner), + ) + + assert resp.status_code == 400 + errors = resp.json['errors'] + assert len(errors) == 3 + errors = sorted(errors, key=lambda x: x['field']) + assert errors[0]['field'] == [] + assert errors[1]['field'] == ['notifications'] + assert errors[2] == { + 'field': ['reviewable'], + 'message': "123 is not of type 'boolean'" + } + + + +def test_put_schema_with_invalid_notificaiton_config(client, db, auth_headers_for_user, users, json_headers): + owner = users['superuser'] + schema = dict( + name='new-schema', + version='1.0.0', + deposit_schema={'title': 'deposit_schema'}, + config={ + 'reviewable': 123, + 'notifications': { + 'actions': { + 'publish': {}, + 'publishhh': {} + } + }, + }, # INVALID, SHOULD FAIL + is_indexed=True, + use_deposit_as_record=True, + ) + + resp = client.post( + '/jsonschemas/', + data=json.dumps(schema), + headers=json_headers + auth_headers_for_user(owner), + ) + + assert resp.status_code == 400 + errors = resp.json['errors'] + assert len(errors) == 3 + assert errors[0]['field'] == ['notifications', 'actions', ] + assert errors[1]['field'] == ['notifications', 'actions', 'publish'] + assert errors[2] == { + 'field': ['reviewable'], + 'message': "123 is not of type 'boolean'" + } + +# ##################################### +# # api/jsonschemas/{id}/{version} [DELETE] +# ##################################### def test_delete_schema_when_user_not_logged_in_returns_401( diff --git a/tests/unit/mail/test_generate_attrs.py b/tests/unit/mail/test_generate_attrs.py new file mode 100644 index 0000000000..3ffb799be4 --- /dev/null +++ b/tests/unit/mail/test_generate_attrs.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2021 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +from mock import patch +from cap.modules.mail.attributes import generate_recipients, generate_body, \ + generate_subject + + +def _generate_schema_config(notification_config): + return { + "notifications": { + "actions": { + "publish": notification_config + } + } + } + +def test_generate_recipients(app, users, location, create_schema, create_deposit): + user = users['cms_user'] + # mock_user.email = user.email + config = { + "recipients": { + "bcc": [{ + "op": "and", + "checks": [ + { + "path": "ml_app_use", + "condition": "exists" + } + ], + "mails": { + "default": ["ml-conveners-test@cern0.ch", "ml-conveners-jira-test@cern0.ch"] + } + }], + "recipients": [ + 'test-mail@cern0.ch', + {"method": "get_owner"}, + {"method": "get_submitter"}, + { + "mails": { + "default": ['default@cern0.ch'] + } + } + ] + } + } + + _config = _generate_schema_config([config]) + create_schema('test', experiment='CMS', config=_config) + deposit = create_deposit(user, 'test', + { + '$ana_type': 'test', + 'general_title': 'Test', + 'ml_app_use': True + }, + experiment='CMS', + publish=True) + + recipients, cc, bcc = generate_recipients(deposit, config, + default_ctx={'submitter_id': user.id}) + assert set(recipients) == {'cms_user@cern.ch', 'default@cern0.ch', 'test-mail@cern0.ch'} + assert set(bcc) == {"ml-conveners-test@cern0.ch", "ml-conveners-jira-test@cern0.ch"} + assert cc == [] + + +# @patch('cap.modules.mail.custom.body.current_user') +def test_generate_body(app, users, location, create_schema, create_deposit): + user = users['cms_user'] + # mock_user.email = user.email + + config = { + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + }, { + "method": "published_url" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + "recipients": ["test-mail@cern0.ch"] + } + } + _config = _generate_schema_config([config]) + create_schema('test', experiment='CMS', config=_config) + deposit = create_deposit(user, 'test', + { + '$ana_type': 'test', + 'general_title': 'Test', + 'analysis_context': { + 'cadi_id': 'ABC-11-111' + }, + }, + experiment='CMS', + publish=True) + + body, _ = generate_body(deposit, config, + default_ctx={'submitter_id': user.id, 'action': 'publish'}) + assert 'CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=ABC-11-111' in body + assert 'Title: Test' in body + assert 'Submitted by cms_user@cern.ch.' in body + +def test_generate_subject(app, users, location, create_schema, create_deposit): + user = users['cms_user'] + # mock_user.email = user.email + + config = { + "subject": { + "template_file": "mail/subject/questionnaire_subject_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "recipients": { + "recipients": ["test-mail@cern0.ch"] + } + } + _config = _generate_schema_config([config]) + create_schema('test', experiment='CMS', config=_config) + deposit = create_deposit(user, 'test', + { + '$ana_type': 'test', + 'analysis_context': { + 'cadi_id': 'ABC-11-111' + }, + }, + experiment='CMS', + publish=True) + + pid = deposit['_deposit']['pid']['value'] + subject = generate_subject(deposit, config, + default_ctx={'submitter_id': user.id, 'action': 'publish'}) + + assert subject == f'Questionnaire for ABC-11-111 {pid} - New Published Analysis | CERN Analysis Preservation' + +def test_generate_recipients_with_nested_conditions(app, users, location, create_schema, create_deposit): + user = users['cms_user'] + # mock_user.email = user.email + + config = { + "recipients": { + 'recipients': [ + {"method": "get_owner"}, + {"method": "get_submitter"}, + { + "mails": { + "default": ['default@cern0.ch'] + } + } + ], + 'bcc': [{ + 'op': 'and', + "checks": [ + { + "path": "ml_app_use", + "condition": "exists", + }, + { + # 1st nested: should return true, 1 of them is false and we have or + "op": "or", + "checks": [{ + "path": "some_other_field", + "condition": "equals", + "value": 'yes', + }, { + "path": "some_field", + "condition": "exists" + }, { + # 2nd nested + "op": "and", + "checks": [{ + "path": "some_third_field", + "condition": "equals", + "value": 'yes', + }] + }] + } + ], + "mails": { + "default": ["ml-conveners-test@cern0.ch", "ml-conveners-jira-test@cern0.ch"] + } + }] + } + } + _config = _generate_schema_config([config]) + create_schema('test', experiment='CMS', config=_config) + deposit = create_deposit(user, 'test', + { + '$ana_type': 'test', + 'general_title': 'Test', + 'ml_app_use': True, + 'some_other_field': 'yes', + 'some_third_field': 'yes' + }, + experiment='CMS', + publish=True) + + recipients, cc, bcc = generate_recipients(deposit, config, + default_ctx={'submitter_id': user.id, 'action': 'publish'}) + assert set(recipients) == {'cms_user@cern.ch', 'default@cern0.ch'} + assert set(bcc) == {"ml-conveners-test@cern0.ch", "ml-conveners-jira-test@cern0.ch"} + assert cc == [] diff --git a/tests/unit/mail/test_mail.py b/tests/unit/mail/test_mail.py index 99c996f4f6..342ec5e20f 100644 --- a/tests/unit/mail/test_mail.py +++ b/tests/unit/mail/test_mail.py @@ -25,23 +25,118 @@ import json from pytest import raises -from mock import patch from invenio_deposit.signals import post_action -from cap.modules.mail.utils import create_and_send -def test_create_and_send_no_recipients_fails(app): - with raises(AssertionError): - create_and_send(None, None, 'Test subject', []) - - -@patch('cap.modules.mail.utils.current_user') -def test_send_mail_published(mock_user, app, users, create_deposit, create_schema, client, auth_headers_for_user): - mock_user.email = 'test@cern.ch' +def test_send_mail_published(app, users, create_deposit, create_schema, client, auth_headers_for_user): + config = { + "notifications": { + "actions": { + "publish": [{ + "subject": { + "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} {{ published_id }} - ' + '{{ "New Version of Published Analysis" if revision > 0 else "New Published Analysis" }} ' + '| CERN Analysis Preservation', + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published_plain.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + }, { + "method": "published_url" + }, { + "method": "submitter_email" + }], + "plain": True + }, + "recipients": { + 'recipients': [ + 'test-recipient@cern0.ch', + { + "checks": [{ + "path": "analysis_context.cadi_id", + "condition": "exists" + }], + "mails": { + "formatted": [{ + "template": "{% if cadi_id %}hn-cms-{{ cadi_id }}@cern0.ch{% endif %}", + "ctx": [{ + "name": "cadi_id", + "type": "path", + "path": "analysis_context.cadi_id" + }] + }] + } + } + ] + } + }, { + "subject": { + "template_file": "mail/subject/questionnaire_subject_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "method": "revision" + }, { + "method": "published_id" + }] + }, + "body": { + "template_file": "mail/body/experiments/cms/questionnaire_message_published.html", + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }, { + "name": "title", + "path": "general_title" + }, { + "method": "published_url" + }, { + "method": "submitter_email" + }] + }, + "recipients": { + 'recipients': [ + {"method": "get_owner"}, + {"method": "get_submitter"} + ], + 'bcc': [ + {"method": "get_cms_stat_recipients"}, + { + 'op': 'and', + "checks": [ + { + "path": "ml_app_use", + "condition": "exists" + } + ], + 'mails': { + 'default': ["ml-conveners-test@cern0.ch", "ml-conveners-jira-test@cern0.ch"] + } + } + ] + } + }] + } + } + } user = users['cms_user'] - create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1") + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) with app.app_context(): with app.extensions['mail'].record_messages() as outbox: @@ -69,8 +164,7 @@ def test_send_mail_published(mock_user, app, users, create_deposit, create_schem # hypernews mail needs to be sent as plain text hypernews_mail = outbox[0] - jira_ml_mail = outbox[1] - standard_mail = outbox[2] + standard_mail = outbox[1] # subject is the same in both assert hypernews_mail.subject == \ @@ -80,37 +174,34 @@ def test_send_mail_published(mock_user, app, users, create_deposit, create_schem # hypernews # message assert 'Title: test analysis' in hypernews_mail.body - assert 'Submitted by test@cern.ch' in hypernews_mail.body - assert f'Questionnaire URL : http://analysispreservation.cern.ch/published/{resp.json["recid"]}' \ + assert 'Submitted by cms_user@cern.ch' in hypernews_mail.body + assert f'Questionnaire URL: https://analysispreservation.cern.ch/published/{resp.json["recid"]}' \ in hypernews_mail.body assert 'https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=ABC-11-111' in hypernews_mail.body # recipients assert 'ml-conveners-test@cern0.ch' not in hypernews_mail.recipients assert 'hn-cms-ABC-11-111@cern0.ch' in hypernews_mail.recipients + assert 'test-recipient@cern0.ch' in hypernews_mail.recipients # standard # message assert 'Title: test analysis' in standard_mail.html - assert 'Submitted by test@cern.ch' in standard_mail.html - assert f'Questionnaire URL : http://analysispreservation.cern.ch/published/{resp.json["recid"]}' \ + assert 'Submitted by cms_user@cern.ch' in standard_mail.html + assert f'Questionnaire URL: https://analysispreservation.cern.ch/published/{resp.json["recid"]}' \ in standard_mail.html assert 'https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=ABC-11-111' in standard_mail.html # recipients - assert 'test@cern.ch' in standard_mail.recipients - assert 'ml-conveners-test@cern0.ch' in standard_mail.recipients - assert 'ml-conveners-jira-test@cern0.ch' not in standard_mail.recipients - assert 'hn-cms-ABC-11-111@cern0.ch' not in standard_mail.recipients + assert set(standard_mail.bcc) == {'ml-conveners-jira-test@cern0.ch', 'ml-conveners-test@cern0.ch'} + assert set(standard_mail.recipients) == {'cms_user@cern.ch'} -@patch('cap.modules.mail.utils.current_user') def test_send_mail_published_with_signal_failure( - mock_user, app, users, create_deposit, create_schema, client, auth_headers_for_user, json_headers): - mock_user.email = 'test@cern.ch' - user = users['cms_user'] + app, users, create_deposit, create_schema, client, auth_headers_for_user, json_headers): def fake_receiver(sender, action=None, pid=None, deposit=None): raise Exception + user = users['cms_user'] create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1") with app.app_context(): @@ -182,3 +273,86 @@ def fake_receiver(sender, action=None, pid=None, deposit=None): assert resp.status_code == 202 assert resp.json['general_title'] == 'NEW TITLE' + + +def test_review_mail_doesnt_send_on_review_resolve( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + config = { + "reviewable": True, + "notifications": { + "actions": { + "review": [{ + "subject": { + "template": 'Questionnaire for {{ cadi_id if cadi_id else "" }} reviewed.', + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "body": { + "template": 'Message for review with cadi id: {{ cadi_id if cadi_id else "" }}.', + "ctx": [{ + "name": "cadi_id", + "path": "analysis_context.cadi_id" + }] + }, + "recipients": {'recipients': ['test-recipient@cern0.ch']} + }] + } + } + } + + user = users['cms_user'] + form_headers = [('Content-Type', 'application/json'), + ('Accept', 'application/form+json')] + + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit( + user, + 'cms-analysis', + { + '$schema': 'https://analysispreservation.cern.ch/schemas/' + 'deposits/records/cms-stats-questionnaire-v0.0.1.json', + 'general_title': 'test analysis', + 'analysis_context': { + 'cadi_id': 'ABC-11-111' + } + }, + experiment='CMS' + ) + depid = deposit["_deposit"]["id"] + + # 1. publish deposit and make sure nothing gets sent on publish + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + assert len(outbox) == 0 + + # 2. review, mail should be sent + resp = client.post(f'/deposits/{depid}/actions/review', + data=json.dumps({ + "type": "request_changes", + "body": "Please change X to Z" + }), + headers=auth_headers_for_user(user) + form_headers) + + assert resp.status_code == 201 + assert len(outbox) == 1 + + # 3. delete review, mail should NOT be sent + review_item = resp.json["review"][0] + review_item_id = review_item["id"] + + resp = client.post(f'/deposits/{depid}/actions/review', + data=json.dumps({ + "id": review_item_id, + "action": "resolve" + }), + headers=auth_headers_for_user(user) + form_headers) + + assert resp.status_code == 201 + assert len(outbox) == 1 diff --git a/tests/unit/mail/test_mail_configs.py b/tests/unit/mail/test_mail_configs.py new file mode 100644 index 0000000000..c7f783c41e --- /dev/null +++ b/tests/unit/mail/test_mail_configs.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# +# This file is part of CERN Analysis Preservation Framework. +# Copyright (C) 2020 CERN. +# +# CERN Analysis Preservation Framework is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# CERN Analysis Preservation Framework is distributed in the hope that it will +# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with CERN Analysis Preservation Framework; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. +"""Tests for mail configs.""" + +from pytest import mark +from data.mail_configs import NESTED_CONDITION_WITH_ERRORS_CONFIG, CONDITION_THAT_DOESNT_EXIST_CONFIG, \ + EMPTY_CONFIG, NO_RECIPIENTS_CONFIG, SUBJECT_METHOD_DOESNT_EXIST_CONFIG, SUBJECT_MISSING_CONFIG, \ + SIMPLE_CONFIG, BODY_MISSING_CONFIG, MULTIPLE_RECIPIENTS_CONFIG, CTX_METHOD_MISSING_CONFIG + + +TEST_DATA = { + '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/' + 'records/cms-stats-questionnaire-v0.0.1.json', + 'general_title': 'test analysis', + 'analysis_context': { + 'cadi_id': 'ABC-11-111' + } +} + + +def test_send_mail_simple_config( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", + config=SIMPLE_CONFIG) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 1 + assert outbox[0].subject == 'Questionnaire for ABC-11-111 published.' + assert 'Message with cadi id: ABC-11-111.' in outbox[0].html + + + +def test_send_mail_no_body_success( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", + config=BODY_MISSING_CONFIG) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 1 + assert outbox[0].html + assert not outbox[0].body + + +def test_send_mail_multiple( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", + config=MULTIPLE_RECIPIENTS_CONFIG) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 2 + assert outbox[0].recipients == ['test-recipient@cern0.ch'] + assert outbox[1].bcc == ['test-recipient-bcc@cern0.ch'] + + +def test_send_mail_when_ctx_method_doesnt_exist( + app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", + config=CTX_METHOD_MISSING_CONFIG) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 1 + assert 'Message with cadi id: ABC-11-111 and val ' in outbox[0].html + + +@mark.parametrize('config', [ + EMPTY_CONFIG, + NESTED_CONDITION_WITH_ERRORS_CONFIG, + CONDITION_THAT_DOESNT_EXIST_CONFIG +]) +def test_configs_where_mail_is_not_sent( + config, app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 0 + + +@mark.parametrize('config', [ + SUBJECT_METHOD_DOESNT_EXIST_CONFIG, + SUBJECT_MISSING_CONFIG +]) +def test_send_mail_when_subject_method_doesnt_exist_or_subject_missing_returns_default( + config, app, users, create_deposit, create_schema, client, auth_headers_for_user): + user = users['cms_user'] + create_schema('cms-stats-questionnaire', experiment='CMS', version="0.0.1", config=config) + + with app.app_context(): + with app.extensions['mail'].record_messages() as outbox: + deposit = create_deposit(user, 'cms-analysis', TEST_DATA, experiment='CMS') + resp = client.post( + f"/deposits/{deposit['_deposit']['id']}/actions/publish", + headers=auth_headers_for_user(user) + ) + + assert resp.status_code == 202 + assert len(outbox) == 1 + assert outbox[0].subject == 'New published document | CERN Analysis Preservation' diff --git a/tests/unit/mail/test_recipients.py b/tests/unit/mail/test_recipients.py deleted file mode 100644 index ecc2874446..0000000000 --- a/tests/unit/mail/test_recipients.py +++ /dev/null @@ -1,184 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of CERN Analysis Preservation Framework. -# Copyright (C) 2020 CERN. -# -# CERN Analysis Preservation Framework is free software; you can redistribute -# it and/or modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# CERN Analysis Preservation Framework is distributed in the hope that it will -# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with CERN Analysis Preservation Framework; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307, USA. -# -# In applying this license, CERN does not -# waive the privileges and immunities granted to it by virtue of its status -# as an Intergovernmental Organization or submit itself to any jurisdiction. -"""Tests for mail.""" -from mock import patch - -from cap.modules.mail.utils import get_cms_stat_recipients, get_review_recipients - - -@patch('cap.modules.mail.utils.current_user') -@patch('cap.modules.mail.utils.path_value_equals') -def test_subject_and_message_on_publish(mock_path_value, mock_user, app, default_config, users, create_deposit): - mock_path_value.return_value = 'key' - mock_user.email = 'test@cern.ch' - user = users['cms_user'] - host_url = 'https://analysispreservation.cern.ch/' - schema = 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json' - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'test': 'test' - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['PDF_FORUM_MAIL'] not in recipients - assert default_config['CONVENERS_ML_MAIL'] not in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'parton_distribution_functions': 'test' - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['PDF_FORUM_MAIL'] in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'multivariate_discriminants': { - 'use_of_centralized_cms_apps': { - 'options': ['Yes'] - } - } - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CONVENERS_ML_MAIL'] in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'multivariate_discriminants': { - 'mva_use': 'test' - } - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CONVENERS_ML_MAIL'] in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'ml_app_use': ['not empty'] - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CONVENERS_ML_MAIL'] in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'ml_survey': { - 'options': 'Yes' - } - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CONVENERS_ML_MAIL'] in recipients - - # test well-formed cadi id for recipients - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'analysis_context': { - 'cadi_id': 'ABC-11-111' - } - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CMS_HYPERNEWS_EMAIL_FORMAT'].format('ABC-11-111') in recipients - - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': schema, - 'analysis_context': { - 'cadi_id': 'AB-11-111' - } - }, - experiment='CMS', - publish=True - ) - _, _, recipients = get_cms_stat_recipients(record, host_url, {}) - assert default_config['CMS_HYPERNEWS_EMAIL_FORMAT'].format('AB0-11-111') not in recipients - - # current user mail should be in every time - assert 'test@cern.ch' in recipients - - -@patch('cap.modules.mail.utils.current_user') -def test_recipients_on_review(mock_user, app, default_config, users, create_deposit): - mock_user.email = 'test@cern.ch' - user = users['cms_user'] - host_url = 'https://analysispreservation.cern.ch/' - - deposit = create_deposit( - user, - 'cms-analysis', - { - '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json', - 'analysis_context': { - 'cadi_id': 'ABC-11-111' - } - }, - experiment='CMS', - publish=True - ) - - _, _, recipients = get_review_recipients(deposit, host_url, {}) - - # owner / reviewer - assert 'test@cern.ch' in recipients - assert 'cms_user@cern.ch' in recipients diff --git a/tests/unit/mail/test_subject_and_message.py b/tests/unit/mail/test_subject_and_message.py deleted file mode 100644 index c965d93f44..0000000000 --- a/tests/unit/mail/test_subject_and_message.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of CERN Analysis Preservation Framework. -# Copyright (C) 2020 CERN. -# -# CERN Analysis Preservation Framework is free software; you can redistribute -# it and/or modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# CERN Analysis Preservation Framework is distributed in the hope that it will -# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with CERN Analysis Preservation Framework; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307, USA. -# -# In applying this license, CERN does not -# waive the privileges and immunities granted to it by virtue of its status -# as an Intergovernmental Organization or submit itself to any jurisdiction. -"""Tests for mail.""" -from mock import patch - -from cap.modules.mail.utils import get_cms_stat_recipients, get_review_recipients - - -@patch('cap.modules.mail.utils.current_user') -@patch('cap.modules.mail.utils.path_value_equals') -def test_subject_and_message_on_publish(mock_path_value, mock_user, app, default_config, users, create_deposit): - mock_path_value.return_value = 'key' - mock_user.email = 'test@cern.ch' - user = users['cms_user'] - host_url = 'https://analysispreservation.cern.ch/' - config = { - "type": "method", - "method": "get_cms_stat_recipients", - "email_subject": "CMS Statistics Committee - " - } - - # with cadi id - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json', - 'analysis_context': { - 'cadi_id': 'ABC-11-111' - }, - 'general_title': 'test analysis' - }, - experiment='CMS', - publish=True - ) - pid = record['_deposit']['pid']['value'] - subject, message, _ = get_cms_stat_recipients(record, host_url, config) - - assert subject == f"Questionnaire for ABC-11-111 {pid} - " - assert 'Title: test analysis' in message - assert 'CADI URL: https://cms.cern.ch/' \ - 'iCMS/analysisadmin/cadi?ancode=ABC-11-111' in message - assert 'Submitted by test@cern.ch' in message - - # without cadi id - record = create_deposit( - user, - 'cms-analysis', - { - '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json', - 'general_title': 'test analysis' - }, - experiment='CMS', - publish=True - ) - pid = record['_deposit']['pid']['value'] - subject, message, _ = get_cms_stat_recipients(record, host_url, config) - - assert subject == f'Questionnaire for {pid} - ' - assert 'CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=None' in message - - -@patch('cap.modules.mail.utils.current_user') -def test_subject_and_message_on_review(mock_user, app, default_config, users, create_deposit): - mock_user.email = 'test@cern.ch' - user = users['cms_user'] - host_url = 'https://analysispreservation.cern.ch/' - config = { - "type": "method", - "method": "get_review_recipients", - "email_subject": "CMS Statistics Questionnaire - " - } - - # with cadi id - deposit = create_deposit( - user, - 'cms-analysis', - { - '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json', - 'analysis_context': { - 'cadi_id': 'ABC-11-111' - }, - 'general_title': 'test analysis' - }, - experiment='CMS', - publish=True - ) - - subject, message, _ = get_review_recipients(deposit, host_url, config) - - assert subject == "Questionnaire for ABC-11-111 - " - assert 'Title: test analysis' in message - assert 'CADI URL: https://cms.cern.ch/' \ - 'iCMS/analysisadmin/cadi?ancode=ABC-11-111' in message - assert 'Submitted by cms_user@cern.ch, and reviewed by test@cern.ch.' in message - - # without cadi id - deposit = create_deposit( - user, - 'cms-analysis', - { - '$schema': 'https://analysispreservation.cern.ch/schemas/deposits/records/cms-analysis-v1.0.0.json', - 'general_title': 'test analysis' - }, - experiment='CMS', - publish=True - ) - - subject, message, _ = get_review_recipients(deposit, host_url, config) - assert subject == "CMS Statistics Questionnaire - " - assert 'Title: test analysis' in message - assert 'CADI URL: https://cms.cern.ch/iCMS/analysisadmin/cadi?ancode=None' in message diff --git a/tests/unit/mail/test_users.py b/tests/unit/mail/test_users.py index 8d602b35c8..d15ed5f0c0 100644 --- a/tests/unit/mail/test_users.py +++ b/tests/unit/mail/test_users.py @@ -22,9 +22,10 @@ # waive the privileges and immunities granted to it by virtue of its status # as an Intergovernmental Organization or submit itself to any jurisdiction. """Tests for mail.""" - +from mock import patch from cap.modules.mail.users import get_all_users, get_users_by_record, \ get_users_by_experiment +from cap.modules.mail.custom.recipients import get_owner, get_submitter def test_get_all_user_mails(users): @@ -67,3 +68,19 @@ def test_get_user_by_experiment(remote_accounts): alice_users = get_users_by_experiment('alice') assert len(alice_users) == 1 + + +def test_get_current_user(app, db, users): + user1 = users['alice_user'] + assert get_submitter(None, + default_ctx={'submitter_id': user1.id}) == ['alice_user@cern.ch'] + + +def test_get_record_owner(users, location, create_schema, create_deposit): + user = users['cms_user'] + create_schema('test', experiment='CMS') + deposit = create_deposit( + user, 'test', + experiment='CMS', + ) + assert get_owner(deposit) == ['cms_user@cern.ch']