This repository has been archived by the owner on May 2, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
main_weather.py
129 lines (101 loc) · 3.64 KB
/
main_weather.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import datetime
import logging
import requests
from pytz import timezone
from database import Database
def _save_to_db(db: Database, dataset: dict) -> None:
"""
Save a captured dataset to the database.
:param db: database to be used
:param dataset: dataset to be saved
:return: None
"""
try:
logger.debug("Saving to database '%s'", dataset)
# get keys for this dataset
keys = {
"timestamp": dataset["timestamp"],
"lat": dataset["lat"],
"lon": dataset["lon"]
}
# save (upsert) the dataset to the database
result = database.upsert(collection=db.mongo_data_weather, query=keys, update=dataset)
# update stats
global num_inserted
num_inserted += 1 - result.matched_count
global num_updated
num_updated += result.modified_count
global num_unchanged
num_unchanged += not result.modified_count and result.matched_count
except KeyError:
logger.exception("Given dataset missing key(s):")
def _load_api_data(lat: float, lon: float, current_time: datetime.datetime) -> None:
"""
Load data from API and save it to the database.
:param lat: latitude of position
:param lon: longitude of position
:param current_time: current datetime
:return: None
"""
# request api
url = "https://api.brightsky.dev/weather"
params = {
# date to look at ['YYYY-MM-DD']
"date": current_time.strftime("%Y-%m-%d"),
# lon of city [XX.XX]
"lon": lon,
# lat of city [XX.XX]
"lat": lat,
# timezone of city
"tz": "Europe/Berlin"
}
headers = {}
r = requests.get(url=url, params=params, headers=headers, timeout=20)
# request successful?
if r.status_code != 200:
logger.critical("Request return unexpected exit code '%s'", r.status_code)
assert False # exit with a big bang
# region process answer
answer = r.json()
# get only weather data
weather = answer['weather']
logger.info("Result lat=%s lon=%s has %s elements", lat, lon, len(weather))
# save processed dataset
for dataset in weather:
dataset["lat"] = lat
dataset["lon"] = lon
_save_to_db(db=database, dataset=dataset)
# endregion
###################################
# Main entry point of main_weather.py
# call with cronjob:
# 0 * * * * /usr/local/bin/python3.10 /home/bigdata/DBWeather/main_weather.py
# 30 * * * * /usr/local/bin/python3.10 /home/bigdata/DBWeather/main_weather.py
###################################
if __name__ == '__main__':
# get now
now = datetime.datetime.now(tz=timezone("Europe/Berlin"))
logDate = now.strftime("%d-%m-%y")
# setup logging
logging.basicConfig(
filename='execution_' + logDate + '.log',
filemode="a",
format='%(asctime)s %(levelname)-7s %(name)s: %(message)s',
encoding='utf-8',
level=logging.INFO
)
logger = logging.getLogger("weather")
logger.info("Start main_weather execution ...")
# setup global counter
num_inserted = 0
num_updated = 0
num_unchanged = 0
# setup database connection and load data
database = Database()
logger.debug("Start processing Frankfurt (Main) weather data ...")
_load_api_data(lat=50.05, lon=8.6, current_time=now)
logger.debug("Start processing Mannheim weather data ...")
_load_api_data(lat=49.5, lon=8.48, current_time=now)
database.close()
logger.info("finished: %s inserted, %s updated, %s unchanged", num_inserted, num_updated, num_unchanged)
logger.info("###########################################")