Skip to content

Commit

Permalink
Tests coverage
Browse files Browse the repository at this point in the history
  • Loading branch information
@kaleanych
kaleanych committed Apr 7, 2023
1 parent 1adc078 commit 5994de7
Show file tree
Hide file tree
Showing 7 changed files with 264 additions and 100 deletions.
8 changes: 3 additions & 5 deletions ted_sws/notice_validator/services/shacl_test_suite_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ def validate_notice_with_shacl_suite(notice: Notice, mapping_suite_package: Mapp
:return:
"""

def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
-> List[SHACLTestSuiteValidationReport]:
reports = []
shacl_test_suites = mapping_suite_package.shacl_test_suites
Expand All @@ -199,12 +199,10 @@ def shacl_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, w
return sorted(reports, key=lambda x: x.test_suite_identifier)

if execute_full_validation:
for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
with_html=with_html):
for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
notice.set_rdf_validation(rdf_validation=report)

for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
with_html=with_html):
for report in shacl_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
notice.set_distilled_rdf_validation(rdf_validation=report)

return notice
Expand Down
201 changes: 123 additions & 78 deletions ted_sws/notice_validator/services/sparql_test_suite_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,19 @@ def _sparql_query_from_file_resource(cls, file_resource: FileResource) -> SPARQL
query = cls._sanitize_query(file_resource.file_content)
return SPARQLQuery(title=title, description=description, xpath=xpath, query=query)

@classmethod
def _refined_result(cls, ask_answer, sparql_query_result,
result: SPARQLQueryRefinedResultType) -> SPARQLQueryRefinedResultType:
if ask_answer and sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.VALID.value
elif not ask_answer and not sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.UNVERIFIABLE.value
elif ask_answer and not sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.WARNING.value
elif not ask_answer and sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.INVALID.value
return result

def _process_sparql_ask_result(self, query_result, sparql_query: SPARQLQuery,
sparql_query_result: SPARQLQueryResult):
ask_answer = query_result.askAnswer
Expand All @@ -89,14 +102,7 @@ def _process_sparql_ask_result(self, query_result, sparql_query: SPARQLQuery,
sparql_query_result.missing_fields = list(sparql_query_xpath - xpaths_in_notice)

# Refined result
if ask_answer and sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.VALID.value
elif not ask_answer and not sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.UNVERIFIABLE.value
elif ask_answer and not sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.WARNING.value
elif not ask_answer and sparql_query_result.fields_covered:
result = SPARQLQueryRefinedResultType.INVALID.value
result = self._refined_result(ask_answer, sparql_query_result, result)

sparql_query_result.result = result

Expand Down Expand Up @@ -155,11 +161,75 @@ def generate_report(self) -> SPARQLTestSuiteValidationReport:
return self.sparql_test_suite_execution


def generate_sparql_validation_summary_report(report_notices: List[ReportNotice], mapping_suite_package: MappingSuite,
execute_full_validation: bool = True,
with_html: bool = False,
report: SPARQLValidationSummaryReport = None,
metadata: dict = None) -> SPARQLValidationSummaryReport:
def process_sparql_validation_summary_report_data_with_notice(
notice: Notice,
mapping_suite_package: MappingSuite,
report_notice_path: Path,
report: SPARQLValidationSummaryReport
):
for sparql_validation in notice.rdf_manifestation.sparql_validations:
test_suite_id = sparql_validation.test_suite_identifier
report.test_suite_ids.append(test_suite_id)
mapping_suite_versioned_id = sparql_validation.mapping_suite_identifier
report.mapping_suite_ids.append(mapping_suite_versioned_id)
validation: SPARQLQueryResult
for validation in sparql_validation.validation_results:
validation_query_result: SPARQLValidationSummaryQueryResult
found_validation_query_result = list(filter(
lambda record:
(record.query.query == validation.query.query)
and (record.query.title == validation.query.title)
and (record.test_suite_identifier == test_suite_id),
report.validation_results
))
if found_validation_query_result:
validation_query_result = found_validation_query_result[0]
else:
validation_query_result = SPARQLValidationSummaryQueryResult(
test_suite_identifier=test_suite_id,
**validation.dict()
)
notice_data: ReportPackageNoticeData = ReportPackageNoticeData(
notice_id=notice.ted_id,
path=str(report_notice_path),
mapping_suite_versioned_id=mapping_suite_versioned_id,
mapping_suite_identifier=mapping_suite_package.identifier
)
if validation.result == SPARQLQueryRefinedResultType.VALID.value:
validation_query_result.aggregate.valid.count += 1
validation_query_result.aggregate.valid.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.UNVERIFIABLE.value:
validation_query_result.aggregate.unverifiable.count += 1
validation_query_result.aggregate.unverifiable.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.INVALID.value:
validation_query_result.aggregate.invalid.count += 1
validation_query_result.aggregate.invalid.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.WARNING.value:
validation_query_result.aggregate.warning.count += 1
validation_query_result.aggregate.warning.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.ERROR.value:
validation_query_result.aggregate.error.count += 1
validation_query_result.aggregate.error.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.UNKNOWN.value:
validation_query_result.aggregate.unknown.count += 1
validation_query_result.aggregate.unknown.notices.append(notice_data)
if not found_validation_query_result:
report.validation_results.append(validation_query_result)


def add_sparql_validation_summary_html_report(
report: SPARQLValidationSummaryReport,
metadata: dict = None
):
template_data: dict = report.dict()
template_data[TEMPLATE_METADATA_KEY] = metadata
html_report = TEMPLATES.get_template(SPARQL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data)
report.object_data = html_report


def init_sparql_validation_summary_report(
report: SPARQLValidationSummaryReport,
report_notices: List[ReportNotice]) -> SPARQLValidationSummaryReport:
if report is None:
report: SPARQLValidationSummaryReport = SPARQLValidationSummaryReport(
object_data="SPARQLValidationSummaryReport",
Expand All @@ -170,6 +240,32 @@ def generate_sparql_validation_summary_report(report_notices: List[ReportNotice]
report.notices = sorted(NoticeTransformer.transform_validation_report_notices(report_notices, group_depth=1) + (
report.notices or []), key=lambda report_data: report_data.notice_id)

return report


def finalize_sparql_validation_summary_report(report: SPARQLValidationSummaryReport, metadata: dict = None,
with_html: bool = False):
report.test_suite_ids = list(set(report.test_suite_ids))
report.mapping_suite_ids = list(set(report.mapping_suite_ids))

if with_html:
add_sparql_validation_summary_html_report(
report=report,
metadata=metadata
)


def generate_sparql_validation_summary_report(report_notices: List[ReportNotice],
mapping_suite_package: MappingSuite,
execute_full_validation: bool = True,
with_html: bool = False,
report: SPARQLValidationSummaryReport = None,
metadata: dict = None) -> SPARQLValidationSummaryReport:
report = init_sparql_validation_summary_report(
report=report,
report_notices=report_notices
)

for report_notice in report_notices:
notice = report_notice.notice
validate_notice_with_sparql_suite(
Expand All @@ -178,68 +274,19 @@ def generate_sparql_validation_summary_report(report_notices: List[ReportNotice]
execute_full_validation=execute_full_validation,
with_html=False
)
for sparql_validation in notice.rdf_manifestation.sparql_validations:
test_suite_id = sparql_validation.test_suite_identifier
report.test_suite_ids.append(test_suite_id)
mapping_suite_versioned_id = sparql_validation.mapping_suite_identifier
report.mapping_suite_ids.append(mapping_suite_versioned_id)

validation: SPARQLQueryResult
for validation in sparql_validation.validation_results:
validation_query_result: SPARQLValidationSummaryQueryResult
found_validation_query_result = list(filter(
lambda record:
(record.query.query == validation.query.query)
and (record.query.title == validation.query.title)
and (record.test_suite_identifier == test_suite_id),
report.validation_results
))

if found_validation_query_result:
validation_query_result = found_validation_query_result[0]
else:
validation_query_result = SPARQLValidationSummaryQueryResult(
test_suite_identifier=test_suite_id,
**validation.dict()
)

notice_data: ReportPackageNoticeData = ReportPackageNoticeData(
notice_id=notice.ted_id,
path=str(report_notice.metadata.path),
mapping_suite_versioned_id=mapping_suite_versioned_id,
mapping_suite_identifier=mapping_suite_package.identifier
)

if validation.result == SPARQLQueryRefinedResultType.VALID.value:
validation_query_result.aggregate.valid.count += 1
validation_query_result.aggregate.valid.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.UNVERIFIABLE.value:
validation_query_result.aggregate.unverifiable.count += 1
validation_query_result.aggregate.unverifiable.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.INVALID.value:
validation_query_result.aggregate.invalid.count += 1
validation_query_result.aggregate.invalid.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.WARNING.value:
validation_query_result.aggregate.warning.count += 1
validation_query_result.aggregate.warning.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.ERROR.value:
validation_query_result.aggregate.error.count += 1
validation_query_result.aggregate.error.notices.append(notice_data)
elif validation.result == SPARQLQueryRefinedResultType.UNKNOWN.value:
validation_query_result.aggregate.unknown.count += 1
validation_query_result.aggregate.unknown.notices.append(notice_data)

if not found_validation_query_result:
report.validation_results.append(validation_query_result)

report.test_suite_ids = list(set(report.test_suite_ids))
report.mapping_suite_ids = list(set(report.mapping_suite_ids))
process_sparql_validation_summary_report_data_with_notice(
notice=notice,
mapping_suite_package=mapping_suite_package,
report_notice_path=report_notice.metadata.path,
report=report
)

if with_html:
template_data: dict = report.dict()
template_data[TEMPLATE_METADATA_KEY] = metadata
html_report = TEMPLATES.get_template(SPARQL_SUMMARY_HTML_REPORT_TEMPLATE).render(template_data)
report.object_data = html_report
finalize_sparql_validation_summary_report(
report=report,
metadata=metadata,
with_html=with_html
)

return report

Expand All @@ -255,7 +302,7 @@ def validate_notice_with_sparql_suite(notice: Notice, mapping_suite_package: Map
:return:
"""

def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation, with_html: bool = False) \
def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation) \
-> List[SPARQLTestSuiteValidationReport]:
reports = []
sparql_test_suites = mapping_suite_package.sparql_test_suites
Expand All @@ -271,12 +318,10 @@ def sparql_validation(notice_item: Notice, rdf_manifestation: RDFManifestation,
return sorted(reports, key=lambda x: x.test_suite_identifier)

if execute_full_validation:
for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation,
with_html=with_html):
for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.rdf_manifestation):
notice.set_rdf_validation(rdf_validation=report)

for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation,
with_html=with_html):
for report in sparql_validation(notice_item=notice, rdf_manifestation=notice.distilled_rdf_manifestation):
notice.set_distilled_rdf_validation(rdf_validation=report)

return notice
Expand Down
16 changes: 8 additions & 8 deletions tests/unit/core/model/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) ->
notice.set_preprocessed_xml_manifestation(xml_manifestation)
notice._status = NoticeStatus.DISTILLED
notice.set_rdf_manifestation(RDFManifestation(object_data="RDF manifestation content",
shacl_validations=[shacl_validation],
sparql_validations=[sparql_validation],
xpath_coverage_validation=xpath_coverage_validation
))
shacl_validations=[shacl_validation],
sparql_validations=[sparql_validation],
xpath_coverage_validation=xpath_coverage_validation
))
notice.set_distilled_rdf_manifestation(RDFManifestation(object_data="RDF manifestation content",
shacl_validations=[shacl_validation],
sparql_validations=[sparql_validation],
xpath_coverage_validation=xpath_coverage_validation
))
shacl_validations=[shacl_validation],
sparql_validations=[sparql_validation],
xpath_coverage_validation=xpath_coverage_validation
))
notice._status = NoticeStatus.ELIGIBLE_FOR_PACKAGING
notice.set_mets_manifestation(METSManifestation(object_data="METS manifestation content"))
notice._status = NoticeStatus.PUBLICLY_AVAILABLE
Expand Down
Loading

0 comments on commit 5994de7

Please sign in to comment.