Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

an attempt at fixing the 15m vs 1h resolutions (for now) #196

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+672 −46
Open

an attempt at fixing the 15m vs 1h resolutions (for now) #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf10b72
an attempt at fixing the 15m vs 1h resolutions (for now)
wonko Oct 7, 2024
fcf4a86
silly cache
wonko Oct 7, 2024
2b32daf
fix 15m overlapping data for DE case
wonko Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 69 additions & 46 deletions custom_components/entsoe/api_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,52 +70,7 @@ def query_day_ahead_prices(

if response.status_code == 200:
try:
root = self._remove_namespace(ET.fromstring(response.content))
_LOGGER.debug(f"content: {root}")
series = {}

# Extract TimeSeries data
for timeseries in root.findall(".//TimeSeries"):
for period in timeseries.findall(".//Period"):
resolution = period.find(".//resolution").text

if resolution != "PT60M":
continue

response_start = period.find(".//timeInterval/start").text
start_time = (
datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
.replace(tzinfo=pytz.UTC)
.astimezone()
)

response_end = period.find(".//timeInterval/end").text
end_time = (
datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
.replace(tzinfo=pytz.UTC)
.astimezone()
)

_LOGGER.debug(f"Period found is from {start_time} till {end_time}")

for point in period.findall(".//Point"):
position = point.find(".//position").text
price = point.find(".//price.amount").text
hour = int(position) - 1
series[start_time + timedelta(hours=hour)] = float(price)

# Now fill in any missing hours
current_time = start_time
last_price = series[current_time]

while current_time < end_time: # upto excluding! the endtime
if current_time in series:
last_price = series[current_time] # Update to the current price
else:
_LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}")
series[current_time] = last_price # Fill with the last known price
current_time += timedelta(hours=1)

series = self.parse_price_document(response.content)
return dict(sorted(series.items()))

except Exception as exc:
Expand All @@ -125,6 +80,74 @@ def query_day_ahead_prices(
print(f"Failed to retrieve data: {response.status_code}")
return None

def parse_price_document(
self, document: str
) -> str:

# note: this function prefers hourly data over 15m interval data.
# if we don't get a hourly datapoint, but we have a 15m datapoint which matches, we use it

root = self._remove_namespace(ET.fromstring(document))
_LOGGER.debug(f"content: {root}")
series = {}
series_datasource = {}

# Extract TimeSeries data
for timeseries in root.findall(".//TimeSeries"):
for period in timeseries.findall(".//Period"):
resolution = period.find(".//resolution").text

if resolution == "PT60M" or resolution == "PT1H":
resolution = "PT60M"
delta_unit = timedelta(hours = 1)
elif resolution == "PT15M":
# we bluntly assume we have only one datapoint per hour
# todo - allow to have an aggregation function
delta_unit = timedelta(minutes=15)
else:
raise(f"Unknown resolution passed: '%s'", resolution)

response_start = period.find(".//timeInterval/start").text
start_time = (
datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
.replace(tzinfo=pytz.UTC)
.astimezone()
)

response_end = period.find(".//timeInterval/end").text
end_time = (
datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
.replace(tzinfo=pytz.UTC)
.astimezone()
)

_LOGGER.debug(f"Period found is from {start_time} till {end_time}")

for point in period.findall(".//Point"):
position = int(point.find(".//position").text)
if resolution == "PT15M" and (position - 1) % 4: # not interested in non-full-hour-datapoints
continue
price = point.find(".//price.amount").text
key = start_time + (position - 1) * delta_unit

# only write if we don't have a 60m data (so, either it's empty, or it has 15m data, which we overwrite)
if not key in series_datasource or series_datasource[key] != "PT60M":
series[key] = float(price)
series_datasource[key] = resolution

# Now fill in any missing hours
current_time = start_time
last_price = series[current_time]

while current_time < end_time: # upto excluding! the endtime
if current_time in series:
last_price = series[current_time] # Update to the current price
else:
_LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}")
series[current_time] = last_price # Fill with the last known price
current_time += timedelta(hours=1)

return series

class Area(enum.Enum):
"""
Expand Down
Empty file added custom_components/entsoe/test/__init__.py
View file
Open in desktop
Empty file.
129 changes: 129 additions & 0 deletions custom_components/entsoe/test/datasets/BE_60M.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
<?xml version="1.0" encoding="utf-8"?>
<Publication_MarketDocument xmlns="urn:iec62325.351:tc57wg16:451-3:publicationdocument:7:3">
<mRID>c3891474d3014008b9c6036658010b88</mRID>
<revisionNumber>1</revisionNumber>
<type>A44</type>
<sender_MarketParticipant.mRID codingScheme="A01">10X1001A1001A450</sender_MarketParticipant.mRID>
<sender_MarketParticipant.marketRole.type>A32</sender_MarketParticipant.marketRole.type>
<receiver_MarketParticipant.mRID codingScheme="A01">10X1001A1001A450</receiver_MarketParticipant.mRID>
<receiver_MarketParticipant.marketRole.type>A33</receiver_MarketParticipant.marketRole.type>
<createdDateTime>2024-10-07T14:34:54Z</createdDateTime>
<period.timeInterval>
<start>2024-10-07T22:00Z</start>
<end>2024-10-08T22:00Z</end>
</period.timeInterval>
<TimeSeries>
<mRID>1</mRID>
<auction.type>A01</auction.type>
<businessType>A62</businessType>
<in_Domain.mRID codingScheme="A01">10YBE----------2</in_Domain.mRID>
<out_Domain.mRID codingScheme="A01">10YBE----------2</out_Domain.mRID>
<contract_MarketAgreement.type>A01</contract_MarketAgreement.type>
<currency_Unit.name>EUR</currency_Unit.name>
<price_Measure_Unit.name>MWH</price_Measure_Unit.name>
<curveType>A03</curveType>
<Period>
<timeInterval>
<start>2024-10-07T22:00Z</start>
<end>2024-10-08T22:00Z</end>
</timeInterval>
<resolution>PT60M</resolution>
<Point>
<position>1</position>
<price.amount>64.98</price.amount>
</Point>
<Point>
<position>2</position>
<price.amount>57.86</price.amount>
</Point>
<Point>
<position>3</position>
<price.amount>53.73</price.amount>
</Point>
<Point>
<position>4</position>
<price.amount>47.52</price.amount>
</Point>
<Point>
<position>5</position>
<price.amount>47.05</price.amount>
</Point>
<Point>
<position>6</position>
<price.amount>56.89</price.amount>
</Point>
<Point>
<position>7</position>
<price.amount>77.77</price.amount>
</Point>
<Point>
<position>8</position>
<price.amount>88.24</price.amount>
</Point>
<Point>
<position>9</position>
<price.amount>100</price.amount>
</Point>
<Point>
<position>10</position>
<price.amount>84.92</price.amount>
</Point>
<Point>
<position>11</position>
<price.amount>74.6</price.amount>
</Point>
<Point>
<position>12</position>
<price.amount>68.82</price.amount>
</Point>
<Point>
<position>13</position>
<price.amount>60.56</price.amount>
</Point>
<Point>
<position>14</position>
<price.amount>63.86</price.amount>
</Point>
<Point>
<position>15</position>
<price.amount>68.1</price.amount>
</Point>
<Point>
<position>16</position>
<price.amount>68.37</price.amount>
</Point>
<Point>
<position>17</position>
<price.amount>76.35</price.amount>
</Point>
<Point>
<position>18</position>
<price.amount>54.04</price.amount>
</Point>
<Point>
<position>19</position>
<price.amount>98.97</price.amount>
</Point>
<Point>
<position>20</position>
<price.amount>115.47</price.amount>
</Point>
<Point>
<position>21</position>
<price.amount>86.85</price.amount>
</Point>
<Point>
<position>22</position>
<price.amount>69.59</price.amount>
</Point>
<Point>
<position>23</position>
<price.amount>57.42</price.amount>
</Point>
<Point>
<position>24</position>
<price.amount>50</price.amount>
</Point>
</Period>
</TimeSeries>
</Publication_MarketDocument>
Loading