Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

XPATHs coverage to support Mapping Remarks #474

Merged
merged 1 commit into from
Apr 13, 2023
Merged

XPATHs coverage to support Mapping Remarks #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ted_sws/core/model/manifestation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ class XPATHCoverageValidationResultBase(PropertyBaseModel):
xpath_covered: Optional[List[str]] = []
xpath_not_covered: Optional[List[str]] = []
xpath_extra: Optional[List[str]] = []
remarked_xpaths: Optional[List[str]] = []
coverage: Optional[float]
conceptual_coverage: Optional[float]

Expand Down
10 changes: 7 additions & 3 deletions ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def _read_conceptual_mapping_rules(cls, df: pd.DataFrame) -> List[ConceptualMapp
return rules

@classmethod
def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMappingRemark]:
def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame, base_xpath: str) -> List[ConceptualMappingRemark]:
"""

:param df:
Expand All @@ -161,7 +161,11 @@ def _read_conceptual_mapping_remarks(cls, df: pd.DataFrame) -> List[ConceptualMa
remark = ConceptualMappingRemark()
remark.standard_form_field_id = cls._read_pd_value(row[RULES_SF_FIELD_ID])
remark.standard_form_field_name = cls._read_pd_value(row[RULES_SF_FIELD_NAME])
remark.field_xpath = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
remarked_xpaths = cls._read_list_from_pd_multiline_value(row[RULES_FIELD_XPATH])
if remarked_xpaths:
remark.field_xpath = []
for remarked_xpath in remarked_xpaths:
remark.field_xpath.append(cls.xpath_with_base(remarked_xpath, base_xpath))
remarks.append(remark)
return remarks

Expand Down Expand Up @@ -273,7 +277,7 @@ def mapping_suite_read_conceptual_mapping(cls, conceptual_mappings_file_path: Pa
conceptual_mapping.metadata = metadata
conceptual_mapping.rules = cls._read_conceptual_mapping_rules(dfs[CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME])
conceptual_mapping.mapping_remarks = cls._read_conceptual_mapping_remarks(
dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME])
dfs[CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME], base_xpath=metadata.base_xpath)
conceptual_mapping.resources = cls._read_conceptual_mapping_resources(
dfs[CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME])
conceptual_mapping.rml_modules = cls._read_conceptual_mapping_rml_modules(
Expand Down
20 changes: 16 additions & 4 deletions ted_sws/notice_validator/adapters/xpath_coverage_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class CoverageRunner:
mapping_suite: MappingSuite
mapping_suite_id: str
conceptual_xpaths: Set[str] = set()
conceptual_remarked_xpaths: Set[str] = set()
conceptual_xpath_data: Dict[str, ConceptualMappingXPATH] = {}
base_xpath: str

Expand All @@ -41,6 +42,13 @@ def notice_xpaths(self, notice: Notice) -> List[str]:
return notice.xml_metadata.unique_xpaths

def init_xpath_data(self, conceptual_mapping: ConceptualMapping):
for cm_xpath in conceptual_mapping.mapping_remarks:
for xpath in cm_xpath.field_xpath:
self.conceptual_remarked_xpaths.add(xpath)
self.conceptual_xpath_data[xpath] = ConceptualMappingXPATH(
xpath=xpath,
form_field=f"{cm_xpath.standard_form_field_id} - {cm_xpath.standard_form_field_name}"
)
for cm_xpath in conceptual_mapping.xpaths:
self.conceptual_xpaths.add(cm_xpath.xpath)
self.conceptual_xpath_data[cm_xpath.xpath] = cm_xpath
Expand All @@ -63,10 +71,13 @@ def find_notice_by_xpath(cls, notice_xpaths: XPathDict, xpath: str) -> Dict[str,
notice_hit: Dict[str, int] = {k: v.count(xpath) for k, v in sorted(notice_xpaths.items()) if xpath in v}
return notice_hit

def get_all_conceptual_xpaths(self) -> Set[str]:
return self.conceptual_remarked_xpaths | self.conceptual_xpaths

def xpath_assertions(self, notice_xpaths: XPathDict,
xpaths_list: List[str]) -> List[XPATHCoverageValidationAssertion]:
xpath_assertions = []
for xpath in self.conceptual_xpaths:
for xpath in self.get_all_conceptual_xpaths():
xpath_assertion = XPATHCoverageValidationAssertion()
xpath_data = self.conceptual_xpath_data[xpath]
form_field = xpath_data.form_field
Expand All @@ -85,8 +96,10 @@ def validate_xpath_coverage_report(self, report: XPATHCoverageValidationReport,
validation_result: XPATHCoverageValidationResult = XPATHCoverageValidationResult()
validation_result.xpath_assertions = self.xpath_assertions(notice_xpaths, xpaths_list)
validation_result.xpath_covered = sorted(list(self.conceptual_xpaths & unique_notice_xpaths))
validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - self.conceptual_xpaths))
validation_result.xpath_extra = sorted(list(self.conceptual_xpaths - unique_notice_xpaths))
all_conceptual_xpaths = self.get_all_conceptual_xpaths()
validation_result.xpath_not_covered = sorted(list(unique_notice_xpaths - all_conceptual_xpaths))
validation_result.xpath_extra = sorted(list(all_conceptual_xpaths - unique_notice_xpaths))
validation_result.remarked_xpaths = sorted(list(self.conceptual_remarked_xpaths))
unique_notice_xpaths_len = len(unique_notice_xpaths)
xpath_covered_len = len(validation_result.xpath_covered)
conceptual_xpaths_len = len(self.conceptual_xpaths)
Expand Down Expand Up @@ -139,4 +152,3 @@ def html_report(cls, report: XPATHCoverageValidationReport, metadata: dict = Non
data[TEMPLATE_METADATA_KEY] = metadata
html_report = TEMPLATES.get_template(XPATH_COVERAGE_REPORT_TEMPLATE).render(data)
return html_report

46 changes: 40 additions & 6 deletions ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,13 @@
[data-role=collapsible] .ui-collapsible-content {
padding: 10px;
}

hr {
margin: 12px 0;
height: 5PX;
background: #ccc;
border: 0;
}
</style>
</head>
<body>
Expand Down Expand Up @@ -113,7 +120,7 @@
{% if validation_result.xpath_assertions|length > 0 %}
<hr>
<h2>XPATH Assertions</h2>
<table class="display" data-order='[[0, "asc"]]'>
<table class="display results" data-order='[[0, "asc"]]'>
<thead>
<tr>
<th>Form Field</th>
Expand Down Expand Up @@ -145,8 +152,8 @@
{% endif %}
{% if validation_result.xpath_covered|length > 0 %}
<hr>
<h2>XPATHs covered by Conceptual Mapping</h2>
<table class="display" data-order='[[0, "asc"]]'>
<h2>XPATHs covered in the "Rules" of Conceptual Mapping</h2>
<table class="display summary" data-order='[[0, "asc"]]'>
<thead>
<tr>
<th>XPATH</th>
Expand All @@ -161,10 +168,28 @@
</tbody>
</table>
{% endif %}
{% if validation_result.remarked_xpaths|length > 0 %}
<hr>
<h2>XPATHs covered in the "Mapping Remarks" of Conceptual Mapping</h2>
<table class="display summary" data-order='[[0, "asc"]]'>
<thead>
<tr>
<th>XPATH</th>
</tr>
</thead>
<tbody>
{% for xpath in validation_result.remarked_xpaths %}
<tr>
<td class="break-word">{{ xpath }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if validation_result.xpath_not_covered|length > 0 %}
<hr>
<h2>XPATHs not covered by Conceptual Mapping</h2>
<table class="display" data-order='[[0, "asc"]]'>
<table class="display summary" data-order='[[0, "asc"]]'>
<thead>
<tr>
<th>XPATH</th>
Expand All @@ -182,7 +207,7 @@
{% if validation_result.xpath_extra|length > 0 %}
<hr>
<h2>Extra XPATHs in Conceptual Mapping</h2>
<table class="display" data-order='[[0, "asc"]]'>
<table class="display summary" data-order='[[0, "asc"]]'>
<thead>
<tr>
<th>XPATH</th>
Expand Down Expand Up @@ -214,7 +239,7 @@
$c.attr("data-state", $c.attr("data-state") == "collapsed" ? "expanded" : "collapsed");
return false;
});
$("table.display").DataTable({
$("table.display.results").DataTable({
dom: 'B<"clear">lfiprtip',
buttons: [],
"lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
Expand All @@ -227,6 +252,15 @@
]
});

$("table.display.summary").DataTable({
dom: 'B<"clear">lfiprtip',
buttons: [],
"lengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
"pageLength": 15,
responsive: {
details: true
}
});
});

</script>