Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update supra_notice_repository.py #426

Merged
merged 2 commits into from
Jan 19, 2023
Merged

Update supra_notice_repository.py #426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
358bbfc
Update supra_notice_repository.py
CaptainOfHacks Jan 18, 2023
b323495
Update sparql_endpoint.py
CaptainOfHacks Jan 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions ted_sws/data_manager/adapters/sparql_endpoint.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,14 @@ class SPARQLTripleStoreEndpoint(TripleStoreEndpointABC):
def __init__(self, endpoint_url: str, user: str = None, password: str = None, use_post_method: bool = False):
user = user if user else config.AGRAPH_SUPER_USER
password = password if password else config.AGRAPH_SUPER_PASSWORD
self.endpoint = SPARQLClientPool.create_or_reuse_connection(endpoint_url, user, password,
use_post_method=use_post_method)
sparql_wrapper = SPARQLWrapper(endpoint_url)
sparql_wrapper.setCredentials(
user=user,
passwd=password
)
if use_post_method:
sparql_wrapper.setMethod(method=POST)
self.endpoint = sparql_wrapper

def _set_sparql_query(self, sparql_query: str):
"""
Expand Down
18 changes: 11 additions & 7 deletions ted_sws/data_manager/adapters/supra_notice_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
from ted_sws.data_manager.adapters import inject_date_string_fields, remove_date_string_fields
from ted_sws.data_manager.adapters.repository_abc import DailySupraNoticeRepositoryABC

DAILY_SUPRA_NOTICE_ID = "notice_fetched_date"
DAILY_SUPRA_NOTICE_FETCHED_DATE = "notice_fetched_date"
DAILY_SUPRA_NOTICE_CREATED_AT = "created_at"
DAILY_SUPRA_NOTICE_ID = "_id"


class DailySupraNoticeRepository(DailySupraNoticeRepositoryABC):
Expand All @@ -25,7 +26,7 @@ def __init__(self, mongodb_client: MongoClient, database_name: str = None):
daily_supra_notice_db = mongodb_client[self._database_name]
self.collection = daily_supra_notice_db[self._collection_name]
self.collection.create_index(
[(DAILY_SUPRA_NOTICE_ID, ASCENDING)]) # TODO: index creation may bring race condition error.
[(DAILY_SUPRA_NOTICE_FETCHED_DATE, ASCENDING)]) # TODO: index creation may bring race condition error.

def _create_dict_from_daily_supra_notice(self, daily_supra_notice: DailySupraNotice) -> dict:
"""
Expand All @@ -34,9 +35,10 @@ def _create_dict_from_daily_supra_notice(self, daily_supra_notice: DailySupraNot
:return:
"""
daily_supra_notice_dict = daily_supra_notice.dict()
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = datetime.combine(
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID], time())
inject_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_ID)
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_FETCHED_DATE] = datetime.combine(
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_FETCHED_DATE], time())
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = daily_supra_notice_dict[DAILY_SUPRA_NOTICE_FETCHED_DATE]
inject_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_FETCHED_DATE)
inject_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_CREATED_AT)
return daily_supra_notice_dict

Expand All @@ -47,8 +49,10 @@ def _create_daily_supra_notice_from_dict(self, daily_supra_notice_dict: dict) ->
:return:
"""
if daily_supra_notice_dict is not None:
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID] = daily_supra_notice_dict[DAILY_SUPRA_NOTICE_ID].date()
remove_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_ID)
daily_supra_notice_dict.pop(DAILY_SUPRA_NOTICE_ID, None)
daily_supra_notice_dict[DAILY_SUPRA_NOTICE_FETCHED_DATE] = daily_supra_notice_dict[
DAILY_SUPRA_NOTICE_FETCHED_DATE].date()
remove_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_FETCHED_DATE)
remove_date_string_fields(data=daily_supra_notice_dict, date_field_name=DAILY_SUPRA_NOTICE_CREATED_AT)
daily_supra_notice = DailySupraNotice.parse_obj(daily_supra_notice_dict)
return daily_supra_notice
Expand Down