Merge pull request #471 from OP-TED/feature/TED-1312

ted_sws/core/model/manifestation.py

@@ -35,6 +35,7 @@ class SPARQLQueryRefinedResultType(Enum):
     INVALID = "invalid"
     ERROR = "error"
     WARNING = "warning"
+    UNKNOWN = "unknown"
 
 
 class Manifestation(PropertyBaseModel):
@@ -224,6 +225,7 @@ class SPARQLQueryResult(PropertyBaseModel):
     fields_covered: Optional[bool] = True
     missing_fields: Optional[List[str]] = []
     error: Optional[str]
+    message: Optional[str]
     identifier: Optional[str]
 
     class Config:

ted_sws/core/model/validation_report.py

@@ -30,6 +30,7 @@ class SPARQLValidationSummaryCountReport(PropertyBaseModel):
     warning: Optional[QueryValidationSummaryCountReportStatus] = QueryValidationSummaryCountReportStatus()
     invalid: Optional[QueryValidationSummaryCountReportStatus] = QueryValidationSummaryCountReportStatus()
     error: Optional[QueryValidationSummaryCountReportStatus] = QueryValidationSummaryCountReportStatus()
+    unknown: Optional[QueryValidationSummaryCountReportStatus] = QueryValidationSummaryCountReportStatus()
 
 
 class SPARQLSummaryQuery(PropertyBaseModel):

ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py

@@ -26,13 +26,15 @@
 SPARQL_PREFIX_LINE = 'PREFIX {prefix}: <{value}>'
 SPARQL_LOGGER_NAME = "SPARQL"
 
+SPARQL_XPATH_SEPARATOR = " ;; "
+
 
 def get_sparql_prefixes(sparql_q: str) -> list:
     finds: list = re.findall(SPARQL_PREFIX_PATTERN, sparql_q)
     return sorted(set(finds))
 
 
-def concat_field_xpath(base_xpath: str, field_xpath: str, separator: str = ", ") -> str:
+def concat_field_xpath(base_xpath: str, field_xpath: str, separator: str = SPARQL_XPATH_SEPARATOR) -> str:
     base_xpath = base_xpath if not pd.isna(base_xpath) else ''
     field_xpath = field_xpath if not pd.isna(field_xpath) else ''
     return separator.join(
@@ -137,7 +139,7 @@ def sparql_validation_generator(data: pd.DataFrame, base_xpath: str, controlled_
               f"{e_form_bt_name}” in eForms. The corresponding XML element is " \
               f"{concat_field_xpath(base_xpath, field_xpath)}. " \
               f"The expected ontology instances are epo: {class_path} .\n" \
-              f"#xpath: {concat_field_xpath(base_xpath, field_xpath, separator=',')}" \
+              f"#xpath: {concat_field_xpath(base_xpath, field_xpath, separator=SPARQL_XPATH_SEPARATOR)}" \
               "\n" + "\n" + "\n".join(prefixes) + "\n\n" \
                                                   f"ASK WHERE {{ " \
                                                   f"{subject_type_display}" \

ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2

@@ -124,7 +124,7 @@
             </tr>
             </thead>
             <tbody>
-            {% for _value in ['valid', 'unverifiable', 'warning', 'invalid', 'error']  %}
+            {% for _value in ['valid', 'unverifiable', 'warning', 'invalid', 'error', 'unknown']  %}
             <tr>
                 {% set _results = validation_results | selectattr("result", "equalto", _value) | list | count %}
                 <td class="strong {{ _value }}">{{ _value }}</td>
@@ -187,8 +187,16 @@
                         </ol>
                         </li>
                     {% endif %}
+                    {% if result.message and (result.message|length > 0) %}
+                        <li>
+                            <hr>
+                            <h4 class="info strong">MESSAGE:</h4>
+                            {{ result.message }}
+                        </li>
+                    {% endif %}
                     {% if result.error and (result.error|length > 0) %}
                         <li>
+                            <hr>
                             <h4 class="danger strong">ERROR:</h4>
                             {{ result.error }}
                         </li>

ted_sws/notice_validator/resources/templates/sparql_summary_report.jinja2

@@ -1,4 +1,4 @@
-{% set result_values = ['valid', 'unverifiable', 'warning', 'invalid', 'error'] %}
+{% set result_values = ['valid', 'unverifiable', 'warning', 'invalid', 'error', 'unknown'] %}
 
 <!DOCTYPE html>
 <html lang="en">

ted_sws/notice_validator/services/shacl_test_suite_runner.py

@@ -185,7 +185,7 @@ def validate_notice_with_shacl_suite(notice: Notice, mapping_suite_package: Mapp
     :return:
     """
 
-    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
+    def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
             -> List[SHACLTestSuiteValidationReport]:
         reports = []
         shacl_test_suites = mapping_suite_package.shacl_test_suites
@@ -199,12 +199,10 @@ def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, w
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     if execute_full_validation:
-        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
-                                       with_html=with_html):
+        for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
             notice.set_rdf_validation(rdf_validation=report)
 
-    for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
-                                   with_html=with_html):
+    for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
         notice.set_distilled_rdf_validation(rdf_validation=report)
 
     return notice

ted_sws/notice_validator/services/sparql_test_suite_runner.py

@@ -12,6 +12,7 @@
     ReportNotice
 from ted_sws.core.model.validation_report_data import ReportPackageNoticeData
 from ted_sws.data_manager.adapters.repository_abc import NoticeRepositoryABC, MappingSuiteRepositoryABC
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_sparql_queries import SPARQL_XPATH_SEPARATOR
 from ted_sws.notice_transformer.adapters.notice_transformer import NoticeTransformer
 from ted_sws.notice_validator.adapters.sparql_runner import SPARQLRunner
 from ted_sws.notice_validator.resources.templates import TEMPLATE_METADATA_KEY
@@ -58,11 +59,25 @@ def _sparql_query_from_file_resource(cls, file_resource: FileResource) -> SPARQL
             if QUERY_METADATA_TITLE in metadata else DEFAULT_QUERY_TITLE
         description = metadata[QUERY_METADATA_DESCRIPTION] \
             if QUERY_METADATA_DESCRIPTION in metadata else DEFAULT_QUERY_DESCRIPTION
-        xpath = metadata[QUERY_METADATA_XPATH].split(",") if QUERY_METADATA_XPATH in metadata and metadata[
-            QUERY_METADATA_XPATH] else DEFAULT_QUERY_XPATH
+        xpath = metadata[QUERY_METADATA_XPATH].split(
+            SPARQL_XPATH_SEPARATOR
+        ) if QUERY_METADATA_XPATH in metadata and metadata[QUERY_METADATA_XPATH] else DEFAULT_QUERY_XPATH
         query = cls._sanitize_query(file_resource.file_content)
         return SPARQLQuery(title=title, description=description, xpath=xpath, query=query)
 
+    @classmethod
+    def _refined_result(cls, ask_answer, sparql_query_result,
+                        result: SPARQLQueryRefinedResultType) -> SPARQLQueryRefinedResultType:
+        if ask_answer and sparql_query_result.fields_covered:
+            result = SPARQLQueryRefinedResultType.VALID.value
+        elif not ask_answer and not sparql_query_result.fields_covered:
+            result = SPARQLQueryRefinedResultType.UNVERIFIABLE.value
+        elif ask_answer and not sparql_query_result.fields_covered:
+            result = SPARQLQueryRefinedResultType.WARNING.value
+        elif not ask_answer and sparql_query_result.fields_covered:
+            result = SPARQLQueryRefinedResultType.INVALID.value
+        return result
+
     def _process_sparql_ask_result(self, query_result, sparql_query: SPARQLQuery,
                                    sparql_query_result: SPARQLQueryResult):
         ask_answer = query_result.askAnswer
@@ -87,14 +102,7 @@ def _process_sparql_ask_result(self, query_result, sparql_query: SPARQLQuery,
                 sparql_query_result.missing_fields = list(sparql_query_xpath - xpaths_in_notice)
 
             # Refined result
-            if ask_answer and sparql_query_result.fields_covered:
-                result = SPARQLQueryRefinedResultType.VALID.value
-            elif not ask_answer and not sparql_query_result.fields_covered:
-                result = SPARQLQueryRefinedResultType.UNVERIFIABLE.value
-            elif ask_answer and not sparql_query_result.fields_covered:
-                result = SPARQLQueryRefinedResultType.WARNING.value
-            elif not ask_answer and sparql_query_result.fields_covered:
-                result = SPARQLQueryRefinedResultType.INVALID.value
+            result = self._refined_result(ask_answer, sparql_query_result, result)
 
         sparql_query_result.result = result
 
@@ -153,11 +161,75 @@ def generate_report(self) -> SPARQLTestSuiteValidationReport:
         return self.sparql_test_suite_execution
 
 
-def generate_sparql_validation_summary_report(report_notices: List[ReportNotice], mapping_suite_package: MappingSuite,
-                                              execute_full_validation: bool = True,
-                                              with_html: bool = False,
-                                              report: SPARQLValidationSummaryReport = None,
-                                              metadata: dict = None) -> SPARQLValidationSummaryReport:
+def process_sparql_validation_summary_report_data_with_notice(
+        notice: Notice,
+        mapping_suite_package: MappingSuite,
+        report_notice_path: Path,
+        report: SPARQLValidationSummaryReport
+):
+    for sparql_validation in notice.rdf_manifestation.sparql_validations:
+        test_suite_id = sparql_validation.test_suite_identifier
+        report.test_suite_ids.append(test_suite_id)
+        mapping_suite_versioned_id = sparql_validation.mapping_suite_identifier
+        report.mapping_suite_ids.append(mapping_suite_versioned_id)
+        validation: SPARQLQueryResult
+        for validation in sparql_validation.validation_results:
+            validation_query_result: SPARQLValidationSummaryQueryResult
+            found_validation_query_result = list(filter(
+                lambda record:
+                (record.query.query == validation.query.query)
+                and (record.query.title == validation.query.title)
+                and (record.test_suite_identifier == test_suite_id),
+                report.validation_results
+            ))
+            if found_validation_query_result:
+                validation_query_result = found_validation_query_result[0]
+            else:
+                validation_query_result = SPARQLValidationSummaryQueryResult(
+                    test_suite_identifier=test_suite_id,
+                    **validation.dict()
+                )
+            notice_data: ReportPackageNoticeData = ReportPackageNoticeData(
+                notice_id=notice.ted_id,
+                path=str(report_notice_path),
+                mapping_suite_versioned_id=mapping_suite_versioned_id,
+                mapping_suite_identifier=mapping_suite_package.identifier
+            )
+            if validation.result == SPARQLQueryRefinedResultType.VALID.value:
+                validation_query_result.aggregate.valid.count += 1
+                validation_query_result.aggregate.valid.notices.append(notice_data)
+            elif validation.result == SPARQLQueryRefinedResultType.UNVERIFIABLE.value:
+                validation_query_result.aggregate.unverifiable.count += 1
+                validation_query_result.aggregate.unverifiable.notices.append(notice_data)
+            elif validation.result == SPARQLQueryRefinedResultType.INVALID.value:
+                validation_query_result.aggregate.invalid.count += 1
+                validation_query_result.aggregate.invalid.notices.append(notice_data)
+            elif validation.result == SPARQLQueryRefinedResultType.WARNING.value:
+                validation_query_result.aggregate.warning.count += 1
+                validation_query_result.aggregate.warning.notices.append(notice_data)
+            elif validation.result == SPARQLQueryRefinedResultType.ERROR.value:
+                validation_query_result.aggregate.error.count += 1
+                validation_query_result.aggregate.error.notices.append(notice_data)
+            elif validation.result == SPARQLQueryRefinedResultType.UNKNOWN.value:
+                validation_query_result.aggregate.unknown.count += 1
+                validation_query_result.aggregate.unknown.notices.append(notice_data)
+            if not found_validation_query_result:
+                report.validation_results.append(validation_query_result)
+
+
+def add_sparql_validation_summary_html_report(
+        report: SPARQLValidationSummaryReport,
+        metadata: dict = None
+):
+    template_data: dict = report.dict()
+    template_data[TEMPLATE_METADATA_KEY] = metadata
+    html_report = TEMPLATES.get_template(SPARQL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data)
+    report.object_data = html_report
+
+
+def init_sparql_validation_summary_report(
+        report: SPARQLValidationSummaryReport,
+        report_notices: List[ReportNotice]) -> SPARQLValidationSummaryReport:
     if report is None:
         report: SPARQLValidationSummaryReport = SPARQLValidationSummaryReport(
             object_data="SPARQLValidationSummaryReport",
@@ -168,6 +240,32 @@ def generate_sparql_validation_summary_report(report_notices: List[ReportNotice]
     report.notices = sorted(NoticeTransformer.transform_validation_report_notices(report_notices, group_depth=1) + (
             report.notices or []), key=lambda report_data: report_data.notice_id)
 
+    return report
+
+
+def finalize_sparql_validation_summary_report(report: SPARQLValidationSummaryReport, metadata: dict = None,
+                                              with_html: bool = False):
+    report.test_suite_ids = list(set(report.test_suite_ids))
+    report.mapping_suite_ids = list(set(report.mapping_suite_ids))
+
+    if with_html:
+        add_sparql_validation_summary_html_report(
+            report=report,
+            metadata=metadata
+        )
+
+
+def generate_sparql_validation_summary_report(report_notices: List[ReportNotice],
+                                              mapping_suite_package: MappingSuite,
+                                              execute_full_validation: bool = True,
+                                              with_html: bool = False,
+                                              report: SPARQLValidationSummaryReport = None,
+                                              metadata: dict = None) -> SPARQLValidationSummaryReport:
+    report = init_sparql_validation_summary_report(
+        report=report,
+        report_notices=report_notices
+    )
+
     for report_notice in report_notices:
         notice = report_notice.notice
         validate_notice_with_sparql_suite(
@@ -176,65 +274,19 @@ def generate_sparql_validation_summary_report(report_notices: List[ReportNotice]
             execute_full_validation=execute_full_validation,
             with_html=False
         )
-        for sparql_validation in notice.rdf_manifestation.sparql_validations:
-            test_suite_id = sparql_validation.test_suite_identifier
-            report.test_suite_ids.append(test_suite_id)
-            mapping_suite_versioned_id = sparql_validation.mapping_suite_identifier
-            report.mapping_suite_ids.append(mapping_suite_versioned_id)
-
-            validation: SPARQLQueryResult
-            for validation in sparql_validation.validation_results:
-                validation_query_result: SPARQLValidationSummaryQueryResult
-                found_validation_query_result = list(filter(
-                    lambda record:
-                    (record.query.query == validation.query.query)
-                    and (record.query.title == validation.query.title)
-                    and (record.test_suite_identifier == test_suite_id),
-                    report.validation_results
-                ))
-
-                if found_validation_query_result:
-                    validation_query_result = found_validation_query_result[0]
-                else:
-                    validation_query_result = SPARQLValidationSummaryQueryResult(
-                        test_suite_identifier=test_suite_id,
-                        **validation.dict()
-                    )
-
-                notice_data: ReportPackageNoticeData = ReportPackageNoticeData(
-                    notice_id=notice.ted_id,
-                    path=str(report_notice.metadata.path),
-                    mapping_suite_versioned_id=mapping_suite_versioned_id,
-                    mapping_suite_identifier=mapping_suite_package.identifier
-                )
 
-                if validation.result == SPARQLQueryRefinedResultType.VALID.value:
-                    validation_query_result.aggregate.valid.count += 1
-                    validation_query_result.aggregate.valid.notices.append(notice_data)
-                elif validation.result == SPARQLQueryRefinedResultType.UNVERIFIABLE.value:
-                    validation_query_result.aggregate.unverifiable.count += 1
-                    validation_query_result.aggregate.unverifiable.notices.append(notice_data)
-                elif validation.result == SPARQLQueryRefinedResultType.INVALID.value:
-                    validation_query_result.aggregate.invalid.count += 1
-                    validation_query_result.aggregate.invalid.notices.append(notice_data)
-                elif validation.result == SPARQLQueryRefinedResultType.WARNING.value:
-                    validation_query_result.aggregate.warning.count += 1
-                    validation_query_result.aggregate.warning.notices.append(notice_data)
-                elif validation.result == SPARQLQueryRefinedResultType.ERROR.value:
-                    validation_query_result.aggregate.error.count += 1
-                    validation_query_result.aggregate.error.notices.append(notice_data)
-
-                if not found_validation_query_result:
-                    report.validation_results.append(validation_query_result)
-
-    report.test_suite_ids = list(set(report.test_suite_ids))
-    report.mapping_suite_ids = list(set(report.mapping_suite_ids))
+        process_sparql_validation_summary_report_data_with_notice(
+            notice=notice,
+            mapping_suite_package=mapping_suite_package,
+            report_notice_path=report_notice.metadata.path,
+            report=report
+        )
 
-    if with_html:
-        template_data: dict = report.dict()
-        template_data[TEMPLATE_METADATA_KEY] = metadata
-        html_report = TEMPLATES.get_template(SPARQL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data)
-        report.object_data = html_report
+    finalize_sparql_validation_summary_report(
+        report=report,
+        metadata=metadata,
+        with_html=with_html
+    )
 
     return report
 
@@ -250,7 +302,7 @@ def validate_notice_with_sparql_suite(notice: Notice, mapping_suite_package: Map
     :return:
     """
 
-    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
+    def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
             -> List[SPARQLTestSuiteValidationReport]:
         reports = []
         sparql_test_suites = mapping_suite_package.sparql_test_suites
@@ -266,12 +318,10 @@ def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation,
         return sorted(reports, key=lambda x: x.test_suite_identifier)
 
     if execute_full_validation:
-        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
-                                        with_html=with_html):
+        for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
             notice.set_rdf_validation(rdf_validation=report)
 
-    for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
-                                    with_html=with_html):
+    for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
         notice.set_distilled_rdf_validation(rdf_validation=report)
 
     return notice