Feature/ted 1085

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -208,11 +208,13 @@ def _write_file_resources(self, file_resources: List[FileResource], path: pathli
                 f.write(file_resource.file_content)
 
     @classmethod
-    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+            List[FileResource]:
         """
         This method reads a folder (with nested-tree structure) of resources and returns a flat list of file-type
         resources from all beyond levels.
         Used for folders that contains files with unique names, but grouped into sub-folders.
+        :param with_content:
         :param extension:
         :param path:
         :param file_resources:
@@ -230,7 +232,8 @@ def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, exten
                     continue
                 file_path = pathlib.Path(os.path.join(root, f))
                 file_resource = FileResource(file_name=file_path.name,
-                                             file_content=file_path.read_text(encoding="utf-8"),
+                                             file_content=file_path.read_text(
+                                                 encoding="utf-8") if with_content else "",
                                              original_name=file_path.name,
                                              parents=file_parents)
                 file_resources.append(file_resource)
@@ -350,6 +353,10 @@ def _read_mapping_suite_package(self, mapping_suite_identifier: str) -> Optional
             return MappingSuite(**package_metadata)
         return None
 
+    @classmethod
+    def mapping_suite_notice_path_by_group_depth(cls, path: pathlib.Path, group_depth: int = 0) -> pathlib.Path:
+        return pathlib.Path(*path.parts[:(-group_depth if group_depth else None)]) if path else None
+
     def add(self, mapping_suite: MappingSuite):
         """
             This method allows you to add MappingSuite objects to the repository.

ted_sws/data_manager/services/mapping_suite_resource_manager.py

@@ -31,18 +31,29 @@ def mapping_suite_skipped_notice(notice_id: str, notice_ids: List[str]) -> bool:
     return notice_ids and notice_id not in notice_ids
 
 
-def mapping_suite_notices_grouped_by_path(mapping_suite: MappingSuite, notice_ids: List[str] = None) -> \
-        Dict[Path, List[ReportNotice]]:
+def mapping_suite_notice_path_by_group_depth(path: Path, group_depth: int = 0) -> Path:
+    return MappingSuiteRepositoryInFileSystem.mapping_suite_notice_path_by_group_depth(path, group_depth=group_depth)
+
+
+def mapping_suite_notices_grouped_by_path(mapping_suite: MappingSuite = None, with_content=True,
+                                          file_resources: List[FileResource] = None, group_depth: int = 0,
+                                          notice_ids: List[str] = None) -> Dict[Path, List[ReportNotice]]:
     grouped_notices: Dict[Path, List[ReportNotice]] = {}
-    for data in mapping_suite.transformation_test_data.test_data:
+    if file_resources is None:
+        file_resources = mapping_suite.transformation_test_data.test_data
+    for data in file_resources:
         notice_id = Path(data.file_name).stem
         if mapping_suite_skipped_notice(notice_id, notice_ids):
             continue
         notice = Notice(ted_id=notice_id)
-        notice.set_xml_manifestation(XMLManifestation(object_data=data.file_content))
+        if with_content:
+            notice.set_xml_manifestation(XMLManifestation(object_data=data.file_content))
         report_notice = ReportNotice(notice=notice,
                                      metadata=ReportNoticeMetadata(path=file_resource_path(data)))
-        group = file_resource_output_path(data)
+        group = mapping_suite_notice_path_by_group_depth(
+            path=file_resource_output_path(data),
+            group_depth=group_depth
+        )
         grouped_notices.setdefault(group, []).append(report_notice)
 
     return grouped_notices
@@ -58,9 +69,11 @@ def mapping_suite_files_grouped_by_path(file_resources: List[FileResource], grou
     return grouped_files
 
 
-def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+        List[FileResource]:
     return MappingSuiteRepositoryInFileSystem.read_flat_file_resources(
         path=path,
         file_resources=file_resources,
-        extension=extension
+        extension=extension,
+        with_content=with_content
     )

ted_sws/notice_transformer/adapters/notice_transformer.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from ted_sws.core.model.notice import Notice
+from ted_sws.core.model.validation_report import ReportNotice
+from ted_sws.core.model.validation_report_data import ReportNoticeData
+from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
+
+
+class NoticeTransformer:
+    @classmethod
+    def transform_report_notice(cls, report_notice: ReportNotice, group_depth: int = 0) -> ReportNoticeData:
+        report_path = str(MappingSuiteRepositoryInFileSystem.mapping_suite_notice_path_by_group_depth(
+            report_notice.metadata.path,
+            group_depth=group_depth)) if report_notice.metadata else None
+        return ReportNoticeData(
+            notice_id=report_notice.notice.ted_id,
+            path=report_path
+        )
+
+    @classmethod
+    def transform_report_notices(cls, report_notices: List[ReportNotice]) -> List[ReportNoticeData]:
+        report_datas = []
+        for report_notice in report_notices:
+            report_datas.append(cls.transform_report_notice(report_notice))
+        return report_datas
+
+    @classmethod
+    def transform_validation_report_notices(cls, report_notices: List[ReportNotice], group_depth: int = 0) \
+            -> List[ReportNoticeData]:
+        report_datas = []
+        for report_notice in report_notices:
+            report_datas.append(cls.transform_report_notice(report_notice, group_depth=group_depth))
+        return report_datas
+
+    @classmethod
+    def map_report_notices_to_notices(cls, report_notices: List[ReportNotice]) -> List[Notice]:
+        notices: List[Notice] = []
+        for report_notice in report_notices:
+            notices.append(report_notice.notice)
+        return notices

ted_sws/notice_transformer/services/notice_transformer.py

@@ -8,10 +8,11 @@
 from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
 from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC, MappingSuiteRepositoryABC
 from ted_sws.data_manager.services.mapping_suite_resource_manager import file_resource_output_path, \
-    mapping_suite_skipped_notice
+    mapping_suite_skipped_notice, mapping_suite_notice_path_by_group_depth
 from ted_sws.event_manager.adapters.event_logger import EventLogger, EventMessageLogSettings
 from ted_sws.event_manager.model.event_message import NoticeEventMessage
 from ted_sws.event_manager.services.logger_from_context import get_env_logger
+from ted_sws.notice_transformer.adapters.notice_transformer import NoticeTransformer
 from ted_sws.notice_transformer.adapters.rml_mapper import RMLMapperABC
 from ted_sws.notice_transformer.services import DEFAULT_TRANSFORMATION_FILE_EXTENSION
 from ted_sws.core.model.validation_report import ReportNotice
@@ -121,19 +122,5 @@ def transform_test_data(mapping_suite: MappingSuite, rml_mapper: RMLMapperABC, o
             logger.info(event_message, settings=EventMessageLogSettings(briefly=True))
 
 
-def transform_report_notice(report_notice: ReportNotice) -> ReportNoticeData:
-    return ReportNoticeData(notice_id=report_notice.notice.ted_id, path=str(report_notice.metadata.path))
-
-
 def transform_report_notices(report_notices: List[ReportNotice]) -> List[ReportNoticeData]:
-    report_datas = []
-    for report_notice in report_notices:
-        report_datas.append(transform_report_notice(report_notice))
-    return report_datas
-
-
-def transform_validation_report_notices(report_notices: List[Notice]) -> List[ReportNoticeData]:
-    report_datas = []
-    for report_notice in report_notices:
-        report_datas.append(ReportNoticeData(notice_id=report_notice.ted_id))
-    return report_datas
+    return NoticeTransformer.transform_report_notices(report_notices)

ted_sws/notice_validator/adapters/validation_summary_runner.py

@@ -8,7 +8,9 @@
     XPATHCoverageSummaryResult, SPARQLSummaryCountReport, SHACLSummarySeverityCountReport, SPARQLSummaryResult, \
     SHACLSummaryResult, ValidationSummaryReport
 from ted_sws.core.model.notice import Notice
-from ted_sws.notice_transformer.services.notice_transformer import transform_validation_report_notices
+from ted_sws.core.model.validation_report import ReportNotice
+from ted_sws.notice_transformer.adapters.notice_transformer import NoticeTransformer
+from ted_sws.notice_validator.resources.templates import TEMPLATE_METADATA_KEY
 
 TEMPLATES = Environment(loader=PackageLoader("ted_sws.notice_validator.resources", "templates"))
 VALIDATION_SUMMARY_REPORT_TEMPLATE = "validation_summary_report.jinja2"
@@ -206,9 +208,11 @@ def validation_summary_for_notice(cls, notice: Notice) -> ValidationSummaryRepor
         return cls.validation_summary([notice])
 
     @classmethod
-    def validation_summary_for_notices(cls, notices: List[Notice]) -> ValidationSummaryReport:
-        report: ValidationSummaryReport = cls.validation_summary(notices)
-        report.notices = sorted(transform_validation_report_notices(notices),
+    def validation_summary_for_notices(cls, notices: List[ReportNotice]) -> ValidationSummaryReport:
+        report: ValidationSummaryReport = cls.validation_summary(
+            sorted(NoticeTransformer.map_report_notices_to_notices(notices),
+                   key=lambda notice: notice.ted_id))
+        report.notices = sorted(NoticeTransformer.transform_validation_report_notices(notices, group_depth=1),
                                 key=lambda report_data: report_data.notice_id)
 
         return report
@@ -218,6 +222,7 @@ def json_report(cls, report) -> dict:
         return report.dict()
 
     @classmethod
-    def html_report(cls, report) -> str:
+    def html_report(cls, report, metadata: dict = None) -> str:
         data: dict = cls.json_report(report)
+        data[TEMPLATE_METADATA_KEY] = metadata
         return TEMPLATES.get_template(VALIDATION_SUMMARY_REPORT_TEMPLATE).render(data)

ted_sws/notice_validator/resources/templates/validation_summary_report.jinja2

@@ -71,26 +71,33 @@
 	<h1>Validation Summary HTML report </h1>
 	<hr>
     <h2>Report details: </h2>
+    {% set mapping_suite_identifier = xml_manifestation.xpath_coverage_summary.mapping_suite_identifier %}
     <ul>
         <li>Date created: {{ created }}</li>
-        <li><div><hr></div>Notice identifier(s):
-            <div data-role="collapsible" data-state="{% if notices|length > 1 %}collapsed{% endif %}"
+        <li><b>Mapping suite identifier:</b> {{ mapping_suite_identifier }}</li>
+        {%  if template_metadata.grouping %}
+            <li><b>Grouping:</b> {{ template_metadata.grouping }}</li>
+        {%  endif %}
+        <li><div><hr></div>
+            {%  set nr_notices = notices|length %}
+            {% if nr_notices > 1 %}
+            <b>Notice identifiers ({{ nr_notices }}):</b>
+            <div data-role="collapsible" data-state="collapsed"
                 class="collapsible-wrapper">
                 <h4><a href="#"></a></h4>
                 <div class="ui-collapsible-content">
-                {% if notices|length == 1 %}
-                    {{ notices[0].notice_id }}
-                {% else %}
                     <ul>
                     {% for report_data in notices %}
                         <li>
-                            {{ report_data.notice_id }}
+                            <a href="file://{{ template_metadata.package_output_path }}/{{ report_data.path }}/{{ report_data.notice_id }}" target="_blank">{{ report_data.notice_id }}</a>
                         </li>
                     {% endfor %}
                     </ul>
-                {% endif %}
                 </div>
             </div>
+            {% elif nr_notices == 1 %}
+                <b>Notice identifier: {{ notices[0].notice_id }}</b>
+            {% endif %}
         </li>
     </ul>
     <hr>

ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2

@@ -91,22 +91,22 @@
         <p>(Number of Conceptual XPATHs found in notices) / (Total number of unique Conceptual XPATHs)</p>
     </li>
     {%  set nr_notices = validation_result.notices|length %}
-    <li><div><hr></div>Notice identifier(s):
-        <div data-role="collapsible" data-state="{% if nr_notices > 1 %}collapsed{% endif %}"
-            class="collapsible-wrapper">
+    <li><div><hr></div>
+        {% if nr_notices > 1 %}
+            <b>Notice identifiers ({{ nr_notices }}):</b>
+        <div data-role="collapsible" data-state="collapsed" class="collapsible-wrapper">
             <h4><a href="#"></a></h4>
             <div class="ui-collapsible-content">
-            {% if nr_notices == 1 %}
-                {{ validation_result.notices[0].notice_id }}
-            {% else %}
                 <ul>
                 {% for report_data in validation_result.notices %}
                     <li><a href="file://{{ template_metadata.package_output_path }}/{{ report_data.path }}/{{ report_data.notice_id }}/test_suite_report/xpath_coverage_validation.html" target="_blank">{{ report_data.notice_id }}</a></li>
                 {% endfor %}
                 </ul>
-            {% endif %}
             </div>
         </div>
+        {% elif nr_notices == 1 %}
+            <b>Notice identifier: {{ validation_result.notices[0].notice_id }}</b>
+        {% endif %}
     </li>
 </ul>
 {% if validation_result.xpath_assertions|length > 0 %}

ted_sws/notice_validator/services/validation_summary_runner.py

@@ -2,16 +2,18 @@
 
 from ted_sws.core.model.manifestation import ValidationSummaryReport
 from ted_sws.core.model.notice import Notice
+from ted_sws.core.model.validation_report import ReportNotice
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.notice_validator.adapters.validation_summary_runner import ValidationSummaryRunner
 
 
-def generate_validation_summary_report_notices(notices: List[Notice],
-                                               with_html: bool = False) -> ValidationSummaryReport:
+def generate_validation_summary_report_notices(notices: List[ReportNotice],
+                                               with_html: bool = False,
+                                               template_metadata: dict = None) -> ValidationSummaryReport:
     validation_summary_runner = ValidationSummaryRunner()
     report = validation_summary_runner.validation_summary_for_notices(notices)
     if with_html:
-        report.object_data = ValidationSummaryRunner.html_report(report)
+        report.object_data = ValidationSummaryRunner.html_report(report, metadata=template_metadata)
     return report
 
 

ted_sws/supra_notice_manager/services/supra_notice_validator.py

@@ -4,8 +4,8 @@
 
 from pymongo import MongoClient
 
-from ted_sws.core.model.notice import Notice
 from ted_sws.core.model.supra_notice import SupraNoticeValidationReport, DailySupraNotice
+from ted_sws.core.model.validation_report import ReportNotice
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
 from ted_sws.data_manager.adapters.supra_notice_repository import DailySupraNoticeRepository
 from ted_sws.event_manager.services.log import log_technical_error
@@ -61,10 +61,12 @@ def summary_validation_for_daily_supra_notice(ted_publication_date: day_type, mo
         raise ValueError(SUPRA_NOTICE_NOT_FOUND_ERROR)
 
     notice_repository: NoticeRepository = NoticeRepository(mongodb_client=mongodb_client)
-    notices: List[Notice] = []
+    notices: List[ReportNotice] = []
 
     for notice_id in supra_notice.notice_ids:
-        notice: Notice = notice_repository.get(reference=notice_id)
+        notice: ReportNotice = ReportNotice(
+            notice=notice_repository.get(reference=notice_id)
+        )
         if notice:
             notices.append(notice)
 

tests/unit/notice_transformer/test_notice_transformer.py

@@ -58,3 +58,13 @@ def test_transform_test_data_function(fake_rml_mapper, fake_mapping_suite):
         test_data = fake_mapping_suite.transformation_test_data.test_data
         test_data_file_names = [pathlib.Path(data.file_name).stem for data in test_data]
         assert Counter(file_names) == Counter(test_data_file_names)
+
+
+def test_transform_test_data_function_with_notices(fake_rml_mapper, fake_mapping_suite):
+    with tempfile.TemporaryDirectory() as d:
+        output_path = pathlib.Path(d)
+        notice_ids = ['include-notice']
+        transform_test_data(mapping_suite=fake_mapping_suite, rml_mapper=fake_rml_mapper, output_path=output_path,
+                            notice_ids=notice_ids)
+        file_names = [path.stem for path in output_path.glob("**/*.ttl")]
+        assert len(file_names) == 0

tests/unit/notice_validator/conftest.py

@@ -433,6 +433,50 @@ def fake_validation_notice():
                 "query_result": 'False',
                 "error": "SOME_ERROR",
                 "identifier": 'sparql_query_46'
+            },
+            {
+                "query": {
+                    "title": "",
+                    "description": "",
+                    "query": ""
+                },
+                "result": 'unverifiable',
+                "query_result": 'True',
+                "error": None,
+                "identifier": 'sparql_query_46'
+            },
+            {
+                "query": {
+                    "title": "",
+                    "description": "",
+                    "query": ""
+                },
+                "result": 'invalid',
+                "query_result": 'True',
+                "error": None,
+                "identifier": 'sparql_query_46'
+            },
+            {
+                "query": {
+                    "title": "",
+                    "description": "",
+                    "query": ""
+                },
+                "result": 'warning',
+                "query_result": 'True',
+                "error": None,
+                "identifier": 'sparql_query_46'
+            },
+            {
+                "query": {
+                    "title": "",
+                    "description": "",
+                    "query": ""
+                },
+                "result": 'warning',
+                "query_result": 'False',
+                "error": None,
+                "identifier": 'sparql_query_46'
             }
         ]
     })]

tests/unit/notice_validator/test_validation_summary_runner.py

@@ -1,5 +1,6 @@
 import pytest
 
+from ted_sws.core.model.validation_report import ReportNotice
 from ted_sws.notice_validator.services.validation_summary_runner import validation_summary_report_notice_by_id, \
     generate_validation_summary_report_notices
 from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
@@ -24,7 +25,10 @@ def test_validation_summary_runner(fake_validation_notice, mongodb_client):
 
 
 def test_generate_validation_summary_report_notices(fake_validation_notice):
-    report = generate_validation_summary_report_notices([fake_validation_notice], with_html=True)
+    report = generate_validation_summary_report_notices([
+        ReportNotice(notice=fake_validation_notice),
+        ReportNotice(notice=fake_validation_notice)
+    ], with_html=True)
     assert report
     assert report.notices
     assert report.notices[0].notice_id == fake_validation_notice.ted_id