Merge pull request #15 from meaningfy-ws/feature/TED-84

Feature/ted 84

requirements.dev.txt

@@ -5,4 +5,5 @@ pytest-bdd~=5.0.0
 pytest-cov~=3.0.0
 pytest-subtests~=0.6.0
 tox~=3.24.5
-tox-pytest-summary~=0.1.2
+tox-pytest-summary~=0.1.2
+mongomock==4.0.0

requirements.txt

@@ -3,4 +3,5 @@ pydantic~=1.9.0
 requests~=2.27.1
 deepdiff~=5.7.0
 git+https://github.com/meaningfy-ws/mfy-data-core
-python-dotenv~=0.19.2
+python-dotenv~=0.19.2
+pymongo~=4.0.1

ted_sws/data_manager/adapters/notice_repository.py

@@ -0,0 +1,55 @@
+from typing import Iterator
+from pymongo import MongoClient
+from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
+from ted_sws.domain.model.notice import Notice
+
+
+class NoticeRepository(NoticeRepositoryABC):
+    """
+       This repository is intended for storing Notice objects.
+    """
+
+    _collection_name = "notice_collection"
+    _database_name = "notice_db"
+
+    def __init__(self, mongodb_client: MongoClient, database_name: str = None):
+        mongodb_client = mongodb_client
+        notice_db = mongodb_client[database_name if database_name else self._database_name]
+        self.collection = notice_db[self._collection_name]
+
+    def add(self, notice: Notice):
+        """
+            This method allows you to add notice objects to the repository.
+        :param notice:
+        :return:
+        """
+        notice_dict = notice.dict()
+        notice_dict["_id"] = notice_dict["ted_id"]
+        self.collection.insert_one(notice_dict)
+
+    def update(self, notice: Notice):
+        """
+            This method allows you to update notice objects to the repository
+        :param notice:
+        :return:
+        """
+        notice_dict = notice.dict()
+        notice_dict["_id"] = notice_dict["ted_id"]
+        self.collection.update_one({'_id':notice_dict["_id"]}, {"$set": notice_dict})
+
+    def get(self, reference) -> Notice:
+        """
+            This method allows a notice to be obtained based on an identification reference.
+        :param reference:
+        :return: Notice
+        """
+        result_dict = self.collection.find_one({"ted_id": reference})
+        return Notice(**result_dict) if result_dict else None
+
+    def list(self) -> Iterator[Notice]:
+        """
+            This method allows all records to be retrieved from the repository.
+        :return: list of notices
+        """
+        for result_dict in self.collection.find():
+            yield Notice(**result_dict)

ted_sws/domain/adapters/repository_abc.py

@@ -1,5 +1,5 @@
 import abc
-from typing import List
+from typing import Iterator
 
 from ted_sws.domain.model.notice import Notice
 
@@ -9,10 +9,12 @@ class RepositoryABC(abc.ABC):
        This class implements a common interface for all repositories.
     """
 
+
 class NoticeRepositoryABC(RepositoryABC):
     """
        This repository is intended for storing Notice objects.
     """
+
     @abc.abstractmethod
     def add(self, notice: Notice):
         """
@@ -22,16 +24,24 @@ def add(self, notice: Notice):
         """
 
     @abc.abstractmethod
-    def get(self, reference)->Notice:
+    def update(self, notice: Notice):
+        """
+            This method allows you to update notice objects to the repository
+        :param notice:
+        :return:
+        """
+
+    @abc.abstractmethod
+    def get(self, reference) -> Notice:
         """
             This method allows a notice to be obtained based on an identification reference.
         :param reference:
-        :return:
+        :return: Notice
         """
 
     @abc.abstractmethod
-    def list(self)->List[str]:
+    def list(self) -> Iterator[Notice]:
         """
             This method allows all records to be retrieved from the repository.
-        :return:
-        """
+        :return: list of notices
+        """

ted_sws/domain/model/notice.py

@@ -14,7 +14,7 @@
 
 import abc
 from datetime import datetime
-from enum import Enum
+from enum import IntEnum
 from typing import Optional
 from functools import total_ordering
 
@@ -31,7 +31,7 @@ class UnsupportedStatusTransition(Exception):
 
 
 @total_ordering
-class NoticeStatus(Enum):
+class NoticeStatus(IntEnum):
     """
         The status of the notice in the pipeline
     """

ted_sws/notice_fetcher/services/notice_fetcher.py

@@ -1,6 +1,5 @@
 import abc
 from datetime import date
-from typing import List
 
 from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
 from ted_sws.domain.model.manifestation import XMLManifestation

tests/conftest.py

@@ -5,7 +5,6 @@
 from tests.fakes.fake_ted_api import FakeRequestAPI
 
 
-
 @pytest.fixture
 def notice_repository():
     return FakeNoticeRepository()

tests/e2e/data_manager/conftest.py

@@ -0,0 +1,11 @@
+import pytest
+from pymongo import MongoClient
+
+from ted_sws import config
+
+
+@pytest.fixture
+def mongodb_client():
+    uri = config.MONGO_DB_AUTH_URL
+    mongodb_client = MongoClient(uri)
+    return mongodb_client

tests/e2e/data_manager/test_mongodb_client.py

@@ -0,0 +1,17 @@
+from pymongo import MongoClient
+
+from ted_sws import config
+
+
+def test_mongodb_client():
+    uri = config.MONGO_DB_AUTH_URL
+    mongodb_client = MongoClient(uri)
+    mongodb_client.drop_database('test')
+    test_db  = mongodb_client['test']
+    fruits_collection = test_db['fruits']
+    fruits_collection.insert_one({"banana": 10, "orange": 50})
+    fruits_collection.insert_one({"banana": 15, "orange": 50})
+    result_fruits = fruits_collection.find_one({"banana": 10})
+    assert isinstance(result_fruits,dict)
+    assert result_fruits["orange"] == 50
+    assert result_fruits["banana"] == 10

tests/e2e/data_manager/test_notice_repository.py

@@ -0,0 +1,33 @@
+import pytest
+
+from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from ted_sws.domain.model.manifestation import XMLManifestation
+from ted_sws.domain.model.metadata import TEDMetadata
+from ted_sws.domain.model.notice import Notice
+
+NOTICE_TED_ID = "123456"
+TEST_DB_NAME = 'test_db'
+
+
+def test_notice_repository_create(mongodb_client):
+    mongodb_client.drop_database(TEST_DB_NAME)
+    notice_repository = NoticeRepository(mongodb_client=mongodb_client, database_name=TEST_DB_NAME)
+    notice = Notice(ted_id=NOTICE_TED_ID, original_metadata=TEDMetadata(**{"AA": "Metadata"}),
+                    xml_manifestation=XMLManifestation(object_data="HELLO"))
+    notice_repository.add(notice)
+    result_notice = notice_repository.get(reference=NOTICE_TED_ID)
+    assert result_notice
+    assert result_notice.ted_id == NOTICE_TED_ID
+    assert result_notice.original_metadata.AA == "Metadata"
+    result_notices = list(notice_repository.list())
+    assert result_notices
+    assert len(result_notices) == 1
+    with pytest.raises(Exception):
+        notice_repository.add(notice)
+    notice.original_metadata = TEDMetadata(**{"AA": "Updated metadata"})
+    notice_repository.update(notice)
+    result_notice = notice_repository.get(reference=NOTICE_TED_ID)
+    assert result_notice
+    assert result_notice.ted_id == NOTICE_TED_ID
+    assert result_notice.original_metadata.AA == "Updated metadata"
+    mongodb_client.drop_database(TEST_DB_NAME)

tests/e2e/notice_fetcher/test_notice_fetcher.py

@@ -22,7 +22,7 @@ def test_notice_fetcher_by_search_query(notice_repository, ted_document_search):
 
     NoticeFetcher(notice_repository=notice_repository, ted_api_adapter=ted_document_search).fetch_notices_by_query(
         query=query)
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     assert isinstance(notices, list)
     assert len(notices) == 1
     assert isinstance(notices[0], Notice)
@@ -32,7 +32,7 @@ def test_notice_fetcher_by_date_range(notice_repository, ted_document_search):
     NoticeFetcher(notice_repository=notice_repository, ted_api_adapter=ted_document_search).fetch_notices_by_date_range(
         start_date=datetime.date(2022, 2, 3), end_date=datetime.date(2022, 2, 3))
     xml_text = "<NOTICE_DATA>"
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     assert isinstance(notices, list)
     assert len(notices) == 95
     assert isinstance(notices[0], Notice)
@@ -44,7 +44,7 @@ def test_notice_fetcher_by_date_wild_card(notice_repository, ted_document_search
                   ted_api_adapter=ted_document_search).fetch_notices_by_date_wild_card(
         wildcard_date="20220203*")
     xml_text = "<NOTICE_DATA>"
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     assert isinstance(notices, list)
     assert len(notices) == 95
     assert isinstance(notices[0], Notice)

tests/fakes/fake_repository.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Iterator
 
 from ted_sws.domain.adapters.repository_abc import NoticeRepositoryABC
 from ted_sws.domain.model.notice import Notice
@@ -23,6 +23,14 @@ def add(self, notice: Notice):
         """
         self.repository[notice.ted_id] = notice
 
+    def update(self, notice: Notice):
+        """
+            This method allows you to update notice objects to the repository.
+        :param notice:
+        :return:
+        """
+        self.repository[notice.ted_id] = notice
+
     def get(self, reference) -> Notice:
         """
             This method allows a notice to be obtained based on an identification reference.
@@ -34,9 +42,9 @@ def get(self, reference) -> Notice:
 
         return None
 
-    def list(self) -> List[str]:
+    def list(self) -> Iterator[Notice]:
         """
             This method allows all records to be retrieved from the repository.
         :return:
         """
-        return list(self.repository.keys())
+        return list(self.repository.values())

tests/unit/data_manager/conftest.py

@@ -0,0 +1,9 @@
+import mongomock
+import pymongo
+import pytest
+
+
+@pytest.fixture
+@mongomock.patch(servers=(('server.example.com', 27017),))
+def mongodb_client():
+    return pymongo.MongoClient('server.example.com')

tests/unit/data_manager/test_notice_repository.py

@@ -0,0 +1,33 @@
+import pytest
+
+from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
+from ted_sws.domain.model.manifestation import XMLManifestation
+from ted_sws.domain.model.metadata import TEDMetadata
+from ted_sws.domain.model.notice import Notice
+
+NOTICE_TED_ID = "123456"
+TEST_DB_NAME = 'test_db'
+
+
+def test_notice_repository_create(mongodb_client):
+    mongodb_client.drop_database(TEST_DB_NAME)
+    notice_repository = NoticeRepository(mongodb_client=mongodb_client, database_name=TEST_DB_NAME)
+    notice = Notice(ted_id=NOTICE_TED_ID, original_metadata=TEDMetadata(**{"AA": "Metadata"}),
+                    xml_manifestation=XMLManifestation(object_data="HELLO"))
+    notice_repository.add(notice)
+    result_notice = notice_repository.get(reference=NOTICE_TED_ID)
+    assert result_notice
+    assert result_notice.ted_id == NOTICE_TED_ID
+    assert result_notice.original_metadata.AA == "Metadata"
+    result_notices = list(notice_repository.list())
+    assert result_notices
+    assert len(result_notices) == 1
+    with pytest.raises(Exception):
+        notice_repository.add(notice)
+    notice.original_metadata = TEDMetadata(**{"AA": "Updated metadata"})
+    notice_repository.update(notice)
+    result_notice = notice_repository.get(reference=NOTICE_TED_ID)
+    assert result_notice
+    assert result_notice.ted_id == NOTICE_TED_ID
+    assert result_notice.original_metadata.AA == "Updated metadata"
+    mongodb_client.drop_database(TEST_DB_NAME)

tests/unit/domain/test_fake_notice_repository.py

@@ -9,7 +9,7 @@ def test_fake_notice_repository(ted_document_search):
     NoticeFetcher(notice_repository=fake_notice_repository,
                             ted_api_adapter=ted_document_search).fetch_notices_by_date_range(
         start_date=datetime.date(2022, 2, 3), end_date=datetime.date(2022, 2, 3))
-    notices = [fake_notice_repository.get(reference=reference) for reference in fake_notice_repository.list()]
+    notices = list(fake_notice_repository.list())
     for notice in notices:
         fake_notice_repository.add(notice)
     for notice in notices:
@@ -18,7 +18,3 @@ def test_fake_notice_repository(ted_document_search):
         assert extracted_notice == notice
     extracted_notice = fake_notice_repository.get(reference="INVALID_REFERENCE")
     assert extracted_notice is None
-
-    extracted_notice_references = fake_notice_repository.list()
-    for notice in notices:
-        assert notice.ted_id in extracted_notice_references

tests/unit/notice_fetcher/test_notice_fetcher.py

@@ -20,7 +20,7 @@ def test_notice_fetcher_by_search_query(notice_repository, ted_document_search):
     query = {"q": "ND=[67623-2022]"}
     NoticeFetcher(notice_repository=notice_repository, ted_api_adapter=ted_document_search).fetch_notices_by_query(
         query=query)
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     assert isinstance(notices, list)
     assert len(notices) == 1
     assert isinstance(notices[0], Notice)
@@ -29,7 +29,7 @@ def test_notice_fetcher_by_search_query(notice_repository, ted_document_search):
 def test_notice_fetcher_by_date_range(notice_repository, ted_document_search):
     NoticeFetcher(notice_repository=notice_repository, ted_api_adapter=ted_document_search).fetch_notices_by_date_range(
         start_date=datetime.date(2022, 2, 3), end_date=datetime.date(2022, 2, 3))
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     xml_text = "<NOTICE_DATA>"
 
     assert isinstance(notices, list)
@@ -41,7 +41,7 @@ def test_notice_fetcher_by_date_range(notice_repository, ted_document_search):
 def test_notice_fetcher_by_date_wild_card(notice_repository, ted_document_search):
     NoticeFetcher(notice_repository=notice_repository,ted_api_adapter=ted_document_search).fetch_notices_by_date_wild_card(
         wildcard_date="20220203*")
-    notices = [notice_repository.get(reference=reference) for reference in notice_repository.list()]
+    notices = list(notice_repository.list())
     xml_text = "<NOTICE_DATA>"
 
     assert isinstance(notices, list)