Merge pull request #465 from OP-TED/feature/TED-1147

ted_sws/data_manager/adapters/notice_repository.py

@@ -1,18 +1,16 @@
-import copy
+import json
 import json
 import logging
 import pathlib
 from datetime import datetime
-from typing import Iterator, Union, Optional, Tuple, Any
+from typing import Iterator, Union, Optional, Any
 
-import gridfs
-from bson import ObjectId
 from pymongo import MongoClient, ASCENDING
 
 from ted_sws import config
 from ted_sws.core.model.lazy_object import LazyObjectFieldsLoaderABC
-from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, Manifestation
-from ted_sws.core.model.metadata import NormalisedMetadata, TEDMetadata, Metadata, XMLMetadata
+from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation
+from ted_sws.core.model.metadata import NormalisedMetadata, TEDMetadata, XMLMetadata
 from ted_sws.core.model.notice import Notice, NoticeStatus
 from ted_sws.data_manager.adapters import inject_date_string_fields, remove_date_string_fields
 from ted_sws.data_manager.adapters.manifestation_repository import XMLManifestationRepository, \

ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py

@@ -13,7 +13,6 @@
     RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION, CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME, \
     CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME, CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME, CL_MAPPING_REFERENCE, \
     CL_SUPERTYPE, CL_FIELD_VALUE, CL_XML_PATH_FRAGMENT
-from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import FILE_NAME_KEY
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 
 # This set of constants refers to fields in the Conceptual Mapping file
@@ -31,6 +30,9 @@
 RULES_CLASS_PATH = 'Class path (M)'
 RULES_PROPERTY_PATH = 'Property path (M)'
 
+FILE_NAME_KEY = "File name"
+REF_INTEGRATION_TESTS_KEY = "Reference to Integration Tests (O)"
+
 
 class ConceptualMappingReader:
     """

ted_sws/mapping_suite_processor/services/conceptual_mapping_files_injection.py

@@ -2,10 +2,10 @@
 import shutil
 
 import pandas as pd
-from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, \
-    CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME
 
-FILE_NAME_KEY = "File name"
+from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, \
+    CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME, CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME
+from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import FILE_NAME_KEY, REF_INTEGRATION_TESTS_KEY
 
 
 def mapping_suite_processor_inject_resources(conceptual_mappings_file_path: pathlib.Path,
@@ -83,3 +83,28 @@ def mapping_suite_processor_inject_sparql_queries(sparql_queries_folder_path: pa
     :return:
     """
     shutil.copytree(sparql_queries_folder_path, output_sparql_queries_folder_path, dirs_exist_ok=True)
+
+
+def mapping_suite_processor_inject_integration_sparql_queries(
+        conceptual_mappings_file_path: pathlib.Path,
+        sparql_queries_folder_path: pathlib.Path,
+        output_sparql_queries_folder_path: pathlib.Path
+):
+    """
+        This function reads the SPARQL files from conceptual_mappings_file_path Rules sheet, and then, based on this list,
+        the resources in sparql_queries_folder_path will be copied to output_sparql_queries_folder_path.
+        :param conceptual_mappings_file_path:
+        :param sparql_queries_folder_path:
+        :param output_sparql_queries_folder_path:
+        :return:
+        """
+    sparql_files_df = pd.read_excel(conceptual_mappings_file_path, sheet_name=CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME,
+                                    skiprows=1)
+    sparql_df_values = sparql_files_df[REF_INTEGRATION_TESTS_KEY].dropna().str.split(',').values.tolist()
+    sparql_files_names = list(set([item.strip() for sublist in sparql_df_values for item in sublist]))
+    for sparql_file_name in sparql_files_names:
+        src_sparql_file_path = sparql_queries_folder_path / sparql_file_name
+        dest_sparql_file_path = output_sparql_queries_folder_path / sparql_file_name
+        if src_sparql_file_path.exists():
+            shutil.copy(src_sparql_file_path, dest_sparql_file_path)
+

tests/unit/mapping_suite_processor/services/test_conceptual_mapping_files_injection.py

@@ -8,7 +8,7 @@
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import \
     mapping_suite_processor_inject_resources, mapping_suite_processor_inject_shacl_shapes, \
     mapping_suite_processor_inject_shacl_shape, mapping_suite_processor_inject_sparql_queries, \
-    mapping_suite_processor_inject_rml_modules
+    mapping_suite_processor_inject_rml_modules, mapping_suite_processor_inject_integration_sparql_queries
 
 CONCEPTUAL_MAPPINGS_FILE_TEMPLATE = '{mappings_path}/{mapping_suite_id}/' + MS_TRANSFORM_FOLDER_NAME + '/' \
                                     + MS_CONCEPTUAL_MAPPING_FILE_NAME
@@ -26,7 +26,7 @@ def test_mapping_suite_processor_inject_resources(fake_mapping_suite_id, file_sy
             mapping_suite_id=fake_mapping_suite_id
         ))
 
-        output_folder_path = Path(temp_folder) / "_mappings_files"
+        output_folder_path = temp_mapping_suite_path / "_mappings_files"
         output_folder_path.mkdir(exist_ok=True)
         mapping_suite_processor_inject_resources(
             conceptual_mappings_file_path=conceptual_mappings_file_path,
@@ -47,7 +47,7 @@ def test_mapping_suite_processor_inject_rml_modules(fake_mapping_suite_id, file_
             mapping_suite_id=fake_mapping_suite_id
         ))
 
-        output_folder_path = Path(temp_folder) / "_rml_modules"
+        output_folder_path = temp_mapping_suite_path / "_rml_modules"
         output_folder_path.mkdir(exist_ok=True)
 
         mapping_suite_processor_inject_rml_modules(
@@ -64,7 +64,7 @@ def test_mapping_suite_processor_inject_shacl_shapes(fake_mapping_suite_id, file
         temp_mapping_suite_path = Path(temp_folder)
         shutil.copytree(file_system_repository_path, temp_mapping_suite_path, dirs_exist_ok=True)
 
-        output_folder_path = Path(temp_folder) / "_shacl_shapes"
+        output_folder_path = temp_mapping_suite_path / "_shacl_shapes"
         output_folder_path.mkdir(exist_ok=True)
 
         mapping_suite_processor_inject_shacl_shapes(
@@ -75,14 +75,14 @@ def test_mapping_suite_processor_inject_shacl_shapes(fake_mapping_suite_id, file
 
 
 def test_mapping_suite_processor_inject_shacl_shape(fake_mapping_suite_id, file_system_repository_path,
-                                                     resources_shacl_files_path):
+                                                    resources_shacl_files_path):
     with tempfile.TemporaryDirectory() as temp_folder:
         temp_mapping_suite_path = Path(temp_folder)
         shutil.copytree(file_system_repository_path, temp_mapping_suite_path, dirs_exist_ok=True)
 
         random_file = random.choice([x for x in resources_shacl_files_path.iterdir()])
 
-        output_folder_path = Path(temp_folder) / "_shacl_shapes"
+        output_folder_path = temp_mapping_suite_path / "_shacl_shapes"
         output_folder_path.mkdir(exist_ok=True)
 
         mapping_suite_processor_inject_shacl_shape(
@@ -98,10 +98,30 @@ def test_mapping_suite_processor_inject_sparql_queries(fake_mapping_suite_id, fi
         temp_mapping_suite_path = Path(temp_folder)
         shutil.copytree(file_system_repository_path, temp_mapping_suite_path, dirs_exist_ok=True)
 
-        output_folder_path = Path(temp_folder) / "_sparql_queries"
+        output_folder_path = temp_mapping_suite_path / "_sparql_queries"
 
         mapping_suite_processor_inject_sparql_queries(
             sparql_queries_folder_path=resources_sparql_files_path,
             output_sparql_queries_folder_path=output_folder_path)
 
         assert any(output_folder_path.iterdir())
+
+
+def test_mapping_suite_processor_inject_integration_sparql_queries(fake_mapping_suite_id, file_system_repository_path,
+                                                                   resources_sparql_files_path):
+    with tempfile.TemporaryDirectory() as temp_folder:
+        temp_mapping_suite_path = Path(temp_folder)
+        shutil.copytree(file_system_repository_path, temp_mapping_suite_path, dirs_exist_ok=True)
+
+        output_folder_path = temp_mapping_suite_path / "_integration_sparql_queries"
+        output_folder_path.mkdir(exist_ok=True)
+
+        mapping_suite_processor_inject_integration_sparql_queries(
+            conceptual_mappings_file_path=Path(CONCEPTUAL_MAPPINGS_FILE_TEMPLATE.format(
+                mappings_path=temp_mapping_suite_path,
+                mapping_suite_id=fake_mapping_suite_id
+            )),
+            sparql_queries_folder_path=resources_sparql_files_path,
+            output_sparql_queries_folder_path=output_folder_path)
+
+        assert any(output_folder_path.iterdir())