From 93f64c9aa8b62db58c16264aec7255023fe3bda7 Mon Sep 17 00:00:00 2001 From: Lucas Kruitwagen Date: Thu, 29 Feb 2024 10:22:46 +0000 Subject: [PATCH] [SMS-228] construct runspec (#45) * tests: base oseomosys construction * feat: improvement to base abstraction * feat: add validation for time_defintion * tests: for time_definition * add test_otoole_roundtrip pytest * [SMS-239] cleanup noqa test data (#36) * fix: rm otoole sample data from tests * feat: import utils by name * fix: cleanup some DS store * fix: import utils * Refactor root_validator to model_validator (#37) * tests: otoole_roundtrip * feat: fitler pandas=3. dep warning * tests: roundtrip otoole timedefn * delint * otoole_roundtrip as pytest * fix: rename otoole-csv paths * fix: make long_name and description optional * tests: skip full otoole construction for now * tests: test region construction * feat: make base osemosys data built from args[0] * fix: rm composable assumptions and targets for now * fix: accidental rename * fix: accident rename test case * tests: commodity construction and compatability * feat: begin refactor to /compat * feat: add 'isnumeric' helper util * refactor: defaults to initial import, don't import pydatnic schemas * refactor: defaults and compat * feat: commodity schema * feat: make data construction more flexible * feat: build and test impact construction * tests: impact otoole roundtrip * fix: some cleanup on commodity testing * tests: touchup test-impact * tests: technology construction * feat: otoole compatability for technology * feat: add to defaults * feat: region and impact compat * feat: validation for technology * tests: runspec construction and roundtrip * feat: add defaults for discount rate depreciation method * feat: instantiate RunSpec on load model * feat: add a depreciation_method enum * feat: include reserve margin in commodity and technology defn * feat: otoole compatability for RunSpec and Technology * fix: return 'self' from model_validator(mode='after') * feat: finish RunSpec model * tests: roundtrip and yaml compatability * fix: merge issues --------- Co-authored-by: edwardxtg Co-authored-by: edwardxtg <71764756+edwardxtg@users.noreply.github.com> --- feo/osemosys/defaults.py | 3 + feo/osemosys/io/load_model.py | 3 + feo/osemosys/schemas/base.py | 17 + feo/osemosys/schemas/commodity.py | 10 +- feo/osemosys/schemas/compat/model.py | 385 ++++++++++++++++++ feo/osemosys/schemas/impact.py | 2 +- feo/osemosys/schemas/model.py | 324 +++------------ feo/osemosys/schemas/region.py | 18 +- feo/osemosys/schemas/technology.py | 18 +- .../schemas/validation/validation_utils.py | 4 +- .../test_compatability/test_otoole_runspec.py | 74 ++++ .../test_construction/test_load_from_yaml.py | 1 - tests/test_construction/test_runspec.py | 35 ++ 13 files changed, 601 insertions(+), 293 deletions(-) create mode 100644 feo/osemosys/schemas/compat/model.py create mode 100644 tests/test_compatability/test_otoole_runspec.py create mode 100644 tests/test_construction/test_runspec.py diff --git a/feo/osemosys/defaults.py b/feo/osemosys/defaults.py index 7b05ef3b..984d0c03 100644 --- a/feo/osemosys/defaults.py +++ b/feo/osemosys/defaults.py @@ -14,6 +14,9 @@ class Defaults(BaseSettings): technology_storage_minimum_charge: float = Field(0.0) technology_storage_initial_level: float = Field(0.0) technology_storage_residual_capacity: float = Field(0.0) + depreciation_method: str = "straight-line" + discount_rate: float = Field(0.1) + reserve_margin: float = Field(1.0) class DefaultsLinopy(BaseSettings): diff --git a/feo/osemosys/io/load_model.py b/feo/osemosys/io/load_model.py index 6313d1bc..4941a423 100644 --- a/feo/osemosys/io/load_model.py +++ b/feo/osemosys/io/load_model.py @@ -4,6 +4,7 @@ import yaml from feo.osemosys import utils +from feo.osemosys.schemas import RunSpec def load_model(*spec_files): @@ -108,3 +109,5 @@ def load_model(*spec_files): # eval strings cfg = utils.walk_dict(cfg, utils.maybe_eval_string) + + return RunSpec(**cfg) diff --git a/feo/osemosys/schemas/base.py b/feo/osemosys/schemas/base.py index a3d735cd..717a4ba0 100644 --- a/feo/osemosys/schemas/base.py +++ b/feo/osemosys/schemas/base.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import Annotated, Dict, Mapping, Union import numpy as np @@ -178,3 +179,19 @@ class OSeMOSYSData_Bool(OSeMOSYSData): Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, bool]]]], Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, bool]]]]], ] + + +class DepreciationMethod(str, Enum): + sinking_fund = "sinking-fund" + straight_line = "straight-line" + + +class OSeMOSYSData_DepreciationMethod(OSeMOSYSData): + data: Union[ + DepreciationMethod, + Dict[IdxVar, DepreciationMethod], + Dict[IdxVar, Dict[IdxVar, DepreciationMethod]], + Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, DepreciationMethod]]], + Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, DepreciationMethod]]]], + Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, Dict[IdxVar, DepreciationMethod]]]]], + ] diff --git a/feo/osemosys/schemas/commodity.py b/feo/osemosys/schemas/commodity.py index 97a98095..8aba7175 100644 --- a/feo/osemosys/schemas/commodity.py +++ b/feo/osemosys/schemas/commodity.py @@ -22,6 +22,10 @@ class Commodity(OSeMOSYSBase, OtooleCommodity): demand_profile: OSeMOSYSData_SumOne | None = Field(None) is_renewable: OSeMOSYSData_Bool | None = Field(None) + # include this technology in joint reserve margin and renewables targets + include_in_joint_reserve_margin: OSeMOSYSData_Bool | None = Field(None) + include_in_joint_renewable_target: OSeMOSYSData_Bool | None = Field(None) + @field_validator("demand_annual", mode="before") @classmethod def passthrough_float(cls, v: Any) -> OSeMOSYSData: @@ -47,9 +51,5 @@ def passthrough_bool(cls, v: Any) -> OSeMOSYSData_Bool: @classmethod def check_demand_exists_if_profile(cls, values): if values.get("demand_profile") is not None and values.get("demand_annual") is None: - commodity = values.get("id") - raise ValueError( - f"If demand_profile is defined for commodity '{commodity}', " - f"demand_annual must also be defined." - ) + raise ValueError("If demand_profile is defined, demand_annual must also be defined.") return values diff --git a/feo/osemosys/schemas/compat/model.py b/feo/osemosys/schemas/compat/model.py new file mode 100644 index 00000000..c1c02914 --- /dev/null +++ b/feo/osemosys/schemas/compat/model.py @@ -0,0 +1,385 @@ +import os +from pathlib import Path +from typing import ClassVar, Union + +import pandas as pd +import xarray as xr +from pydantic import BaseModel, Field + +from feo.osemosys.defaults import defaults +from feo.osemosys.schemas.base import OSeMOSYSData_Bool +from feo.osemosys.schemas.commodity import Commodity +from feo.osemosys.schemas.compat.base import DefaultsOtoole, OtooleCfg +from feo.osemosys.schemas.impact import Impact +from feo.osemosys.schemas.region import Region +from feo.osemosys.schemas.technology import Technology +from feo.osemosys.schemas.time_definition import TimeDefinition +from feo.osemosys.utils import group_to_json, merge, to_df_helper + + +class RunSpecOtoole(BaseModel): + otoole_cfg: OtooleCfg | None = Field(None) + + # Default values + defaults_otoole: DefaultsOtoole | None = Field(None) + + otoole_stems: ClassVar[dict[str : dict[str : Union[str, list[str]]]]] = { + "ReserveMargin": { + "attribute": "reserve_margin", + "columns": ["REGION", "YEAR", "VALUE"], + }, + "ReserveMarginTagFuel": { + "attribute": "reserve_margin", + "columns": ["REGION", "FUEL", "YEAR", "VALUE"], + }, + "ReserveMarginTagTechnology": { + "attribute": "reserve_margin", + "columns": ["REGION", "TECHNOLOGY", "YEAR", "VALUE"], + }, + "REMinProductionTarget": { + "attribute": "renewable_production_target", + "columns": ["REGION", "YEAR", "VALUE"], + }, + "RETagFuel": { + "attribute": "renewable_production_target", + "columns": ["REGION", "FUEL", "YEAR", "VALUE"], + }, + "RETagTechnology": { + "attribute": "renewable_production_target", + "columns": ["REGION", "TECHNOLOGY", "YEAR", "VALUE"], + }, + "DepreciationMethod": { + "attribute": "depreciation_method", + "columns": ["REGION", "VALUE"], + }, + "DiscountRate": { + "attribute": "discount_rate", + "columns": ["REGION", "VALUE"], + }, + "DiscountRateIdv": { + "attribute": "discount_rate", + "columns": ["REGION", "TECHNOLOGY", "VALUE"], + }, + } + + def to_xr_ds(self): + """ + Return the current RunSpec as an xarray dataset + + Args: + self: this RunSpec instance + + Returns: + xr.Dataset: An XArray dataset containing all data from the RunSpec + """ + + # Convert Runspec data to dfs + data_dfs = to_df_helper(self) + + # Set index to columns other than "VALUE" (only for parameter dataframes) + for df_name, df in data_dfs.items(): + if not df_name.isupper(): + data_dfs[df_name] = df.set_index(df.columns.difference(["VALUE"]).tolist()) + # Convert params to data arrays + data_arrays = {x: y.to_xarray()["VALUE"] for x, y in data_dfs.items() if not x.isupper()} + # Create dataset + ds = xr.Dataset(data_vars=data_arrays) + + # If runspec not generated using otoole config yaml, use linopy defaults + if self.defaults_otoole is None: + default_values = defaults.otoole_name_defaults + # If storage technologies present, use additional relevant default values + if self.storage_technologies: + default_values = {**default_values, **defaults.otoole_name_storage_defaults} + # Extract defaults data from OSeMOSYSData objects + for name, osemosys_data in default_values.items(): + default_values[name] = osemosys_data.data + # Otherwise take defaults from otoole config yaml file + else: + default_values = {} + for name, data in self.defaults_otoole.values.items(): + if data["type"] == "param": + default_values[name] = data["default"] + + # Replace any nan values in ds with default values (or None) for corresponding param, + # adding default values as attribute of each data array + for name in ds.data_vars.keys(): + # Replace nan values with default values if available + if name in default_values.keys(): + ds[name].attrs["default"] = default_values[name] + ds[name] = ds[name].fillna(default_values[name]) + # Replace all other nan values with None + # TODO: remove this code if nan values wanted in the ds + # else: + # ds[name].attrs["default"] = None + # ds[name] = ds[name].fillna(None) + + return ds + + @classmethod + def from_otoole_csv(cls, root_dir, id: str | None = None): + dfs = {} + otoole_cfg = OtooleCfg(empty_dfs=[]) + for key in list(cls.otoole_stems): + try: + dfs[key] = pd.read_csv(Path(root_dir) / f"{key}.csv") + if dfs[key].empty: + otoole_cfg.empty_dfs.append(key) + except FileNotFoundError: + otoole_cfg.empty_dfs.append(key) + + # load from other objects + impacts = Impact.from_otoole_csv(root_dir=root_dir) + regions = Region.from_otoole_csv(root_dir=root_dir) + technologies = Technology.from_otoole_csv(root_dir=root_dir) + commodities = Commodity.from_otoole_csv(root_dir=root_dir) + time_definition = TimeDefinition.from_otoole_csv(root_dir=root_dir) + + # read in depreciation_method and replace enum + if "DepreciationMethod" not in otoole_cfg.empty_dfs: + dfs["DepreciationMethod"]["VALUE"] = dfs["DepreciationMethod"]["VALUE"].map( + {1: "sinking-fund", 2: "straight-line"} + ) + depreciation_method = dfs["DepreciationMethod"].set_index("REGION")["VALUE"].to_dict() + else: + depreciation_method = None + + # discount rate from data + discount_rate = ( + dfs["DiscountRate"].set_index("REGION")["VALUE"].to_dict() + if "DiscountRate" not in otoole_cfg.empty_dfs + else None + ) + if "DiscountRateIdv" not in otoole_cfg.empty_dfs: + discount_rate_idv = group_to_json( + g=dfs["DiscountRateIdv"], + root_column=None, + data_columns=["REGION", "TECHNOLOGY"], + target_column="VALUE", + ) + else: + discount_rate_idv = None + + # merge with Idv if necessary or just take discount_rate_idv + if discount_rate is not None and discount_rate_idv is not None: + # merge together + discount_rate = {k: {"*": v} for k, v in discount_rate.items()} + discount_rate = merge(discount_rate, discount_rate_idv) + elif discount_rate is None and discount_rate_idv is not None: + discount_rate = discount_rate_idv + + # reserve margin and renewable production target + reserve_margin = ( + group_to_json( + g=dfs["ReserveMargin"], + root_column=None, + data_columns=["REGION", "YEAR"], + target_column="VALUE", + ) + if "ReserveMargin" not in otoole_cfg.empty_dfs + else None + ) + renewable_production_target = ( + group_to_json( + g=dfs["REMinProductionTarget"], + root_column=None, + data_columns=["REGION", "YEAR"], + target_column="VALUE", + ) + if "REMinProductionTarget" not in otoole_cfg.empty_dfs + else None + ) + + if "RETagFuel" not in otoole_cfg.empty_dfs: + dfs["RETagFuel"]["VALUE"] = dfs["RETagFuel"]["VALUE"].map({0: False, 1: True}) + re_tagfuel_data = group_to_json( + g=dfs["RETagFuel"], + root_column=None, + data_columns=["FUEL", "REGION", "YEAR"], + target_column="VALUE", + ) + for commodity in commodities: + if commodity.id in re_tagfuel_data.keys(): + commodity.include_in_joint_renewable_target = OSeMOSYSData_Bool( + re_tagfuel_data[commodity.id] + ) + + if "RETagTechnology" not in otoole_cfg.empty_dfs: + dfs["RETagTechnology"]["VALUE"] = dfs["RETagTechnology"]["VALUE"].map( + {0: False, 1: True} + ) + re_tagtechnology_data = group_to_json( + g=dfs["RETagTechnology"], + root_column=None, + data_columns=["TECHNOLOGY", "REGION", "YEAR"], + target_column="VALUE", + ) + for technology in technologies: + if technology.id in re_tagtechnology_data.keys(): + technology.include_in_joint_renewable_target = OSeMOSYSData_Bool( + re_tagtechnology_data[technology.id] + ) + + if "ReserveMarginTagFuel" not in otoole_cfg.empty_dfs: + dfs["ReserveMarginTagFuel"]["VALUE"] = dfs["ReserveMarginTagFuel"]["VALUE"].map( + {0: False, 1: True} + ) + reserve_margin_fuel_data = group_to_json( + g=dfs["ReserveMarginTagFuel"], + root_column=None, + data_columns=["FUEL", "REGION", "YEAR"], + target_column="VALUE", + ) + for commodity in commodities: + if commodity.id in reserve_margin_fuel_data.keys(): + commodity.include_in_joint_reserve_margin = OSeMOSYSData_Bool( + reserve_margin_fuel_data[commodity.id] + ) + + if "ReserveMarginTagTechnology" not in otoole_cfg.empty_dfs: + dfs["ReserveMarginTagTechnology"]["VALUE"] = dfs["ReserveMarginTagTechnology"][ + "VALUE" + ].map({0: False, 1: True}) + reserve_margin_technology_data = group_to_json( + g=dfs["ReserveMarginTagTechnology"], + root_column=None, + data_columns=["TECHNOLOGY", "REGION", "YEAR"], + target_column="VALUE", + ) + for technology in technologies: + if technology.id in reserve_margin_technology_data.keys(): + technology.include_in_joint_reserve_margin = OSeMOSYSData_Bool( + reserve_margin_technology_data[technology.id] + ) + + return cls( + id=id if id else Path(root_dir).name, + discount_rate=discount_rate, + depreciation_method=depreciation_method, + reserve_margin=reserve_margin, + renewable_production_target=renewable_production_target, + impacts=impacts, + regions=regions, + technologies=technologies, + commodities=commodities, + time_definition=time_definition, + otoole_cfg=otoole_cfg, + ) + + def to_otoole_csv(self, output_directory): + """ + Convert Runspec to otoole style output CSVs and config.yaml + + Parameters + ---------- + output_directory: str + Path to the output directory for CSV files to be placed + """ + + # do subsidiary objects + Technology.to_otoole_csv(technologies=self.technologies, output_directory=output_directory) + Impact.to_otoole_csv(impacts=self.impacts, output_directory=output_directory) + Commodity.to_otoole_csv(commodities=self.commodities, output_directory=output_directory) + Region.to_otoole_csv(regions=self.regions, output_directory=output_directory) + self.time_definition.to_otoole_csv(output_directory=output_directory) + + # collect dataframes + dfs = {} + + # depreciation_method + if self.depreciation_method: + df = pd.json_normalize(self.depreciation_method.data).T.rename(columns={0: "VALUE"}) + df[["REGION"]] = pd.DataFrame(df.index.str.split(".").to_list(), index=df.index) + df["VALUE"] = df["VALUE"].map({"sinking-fund": 1, "straight-line": 2}) + dfs["DepreciationMethod"] = df + + # discount rate + if self.discount_rate: + df = pd.json_normalize(self.discount_rate.data).T.rename(columns={0: "VALUE"}) + df[["REGION", "TECHNOLOGY"]] = pd.DataFrame( + df.index.str.split(".").to_list(), index=df.index + ) + # if there are different discount rates per technology, use Idv + if (df.groupby(["REGION"])["VALUE"].nunique() > 1).any(): + idv_regions = (df.groupby(["REGION"])["VALUE"].nunique() > 1).index + dfs["DiscountRateIdv"] = df.loc[df["REGIONS"].isin(idv_regions)] + dfs["DiscountRate"] = df.loc[~df["REGIONS"].isin(idv_regions)].drop( + columns=["TECHNOLOGY"] + ) + else: + dfs["DiscountRate"] = df.drop(columns=["TECHNOLOGY"]) + + # reserve margins + if self.reserve_margin: + df = pd.json_normalize(self.reserve_margin).T.rename(columns={0: "VALUE"}) + df[["REGION", "YEAR"]] = pd.DataFrame(df.index.str.split(".").to_list(), index=df.index) + dfs["ReserveMargin"] = df + + dfs_tag_technology = [] + for technology in self.technologies: + if technology.include_in_joint_reserve_margin is not None: + df = pd.json_normalize( + technology.include_in_joint_reserve_margin.data + ).T.rename(columns={0: "VALUE"}) + df["TECHNOLOGY"] = technology.id + df[["REGION", "YEAR"]] = pd.DataFrame( + df.index.str.split(".").to_list(), index=df.index + ) + df["VALUE"] = df["VALUE"].astype(int) + dfs_tag_technology.append(df) + + dfs_tag_fuel = [] + for commodity in self.commodities: + if commodity.include_in_joint_reserve_margin is not None: + df = pd.json_normalize(commodity.include_in_joint_reserve_margin.data).T.rename( + columns={0: "VALUE"} + ) + df["FUEL"] = commodity.id + df[["REGION", "YEAR"]] = pd.DataFrame( + df.index.str.split(".").to_list(), index=df.index + ) + df["VALUE"] = df["VALUE"].astype(int) + dfs_tag_fuel.append(df) + + dfs["ReserveMarginTagTechnology"] = pd.concat(dfs_tag_technology) + dfs["ReserveMarginTagFuel"] = pd.concat(dfs_tag_fuel) + + # min renewable production targets + if self.renewable_production_target: + df = pd.json_normalize(self.renewable_production_target).T.rename(columns={0: "VALUE"}) + df[["REGION", "YEAR"]] = pd.DataFrame(df.index.str.split(".").to_list(), index=df.index) + dfs["ReserveMargin"] = df + + dfs_tag_technology = [] + for technology in self.technologies: + if technology.include_in_joint_renewable_target is not None: + df = pd.json_normalize( + technology.include_in_joint_renewable_target.data + ).T.rename(columns={0: "VALUE"}) + df["TECHNOLOGY"] = technology.id + df[["REGION", "YEAR"]] = pd.DataFrame( + df.index.str.split(".").to_list(), index=df.index + ) + df["VALUE"] = df["VALUE"].astype(int) + dfs_tag_technology.append(df) + + dfs_tag_fuel = [] + for commodity in self.commodities: + if commodity.include_in_joint_renewable_target is not None: + df = pd.json_normalize( + commodity.include_in_joint_renewable_target.data + ).T.rename(columns={0: "VALUE"}) + df["FUEL"] = commodity.id + df[["REGION", "YEAR"]] = pd.DataFrame( + df.index.str.split(".").to_list(), index=df.index + ) + df["VALUE"] = df["VALUE"].astype(int) + dfs_tag_fuel.append(df) + + dfs["RETagTechnology"] = pd.concat(dfs_tag_technology) + dfs["RETagFuel"] = pd.concat(dfs_tag_fuel) + + # write dataframes + for stem, _params in self.otoole_stems.items(): + if stem not in self.otoole_cfg.empty_dfs: + dfs[stem].to_csv(os.path.join(output_directory, f"{stem}.csv"), index=False) diff --git a/feo/osemosys/schemas/impact.py b/feo/osemosys/schemas/impact.py index 875169bb..2bed01af 100644 --- a/feo/osemosys/schemas/impact.py +++ b/feo/osemosys/schemas/impact.py @@ -46,4 +46,4 @@ def validate_exogenous_lt_constraint(self): ) if self.constraint_total is not None and self.exogenous_total is not None: exogenous_total_within_constraint(self.id, self.constraint_total, self.exogenous_total) - return True + return self diff --git a/feo/osemosys/schemas/model.py b/feo/osemosys/schemas/model.py index d10ca3e0..658b2cf7 100644 --- a/feo/osemosys/schemas/model.py +++ b/feo/osemosys/schemas/model.py @@ -1,297 +1,81 @@ -import os import warnings -from typing import List, Optional +from typing import Any, List -import xarray as xr -import yaml -from pydantic import model_validator +from pydantic import Field, model_validator from feo.osemosys.defaults import defaults -from feo.osemosys.schemas.base import OSeMOSYSBase +from feo.osemosys.schemas.base import ( + OSeMOSYSBase, + OSeMOSYSData, + OSeMOSYSData_DepreciationMethod, + OSeMOSYSData_Int, +) from feo.osemosys.schemas.commodity import Commodity -from feo.osemosys.schemas.compat.base import DefaultsOtoole +from feo.osemosys.schemas.compat.model import RunSpecOtoole from feo.osemosys.schemas.impact import Impact from feo.osemosys.schemas.region import Region from feo.osemosys.schemas.technology import Technology, TechnologyStorage from feo.osemosys.schemas.time_definition import TimeDefinition -from feo.osemosys.schemas.validation.model_composition import ( - check_tech_consuming_commodity, - check_tech_producing_commodity, - check_tech_producing_impact, -) -from feo.osemosys.schemas.validation.model_presolve import check_able_to_meet_demands -from feo.osemosys.utils import to_df_helper +from feo.osemosys.utils import isnumeric # filter this pandas-3 dep warning for now warnings.filterwarnings("ignore", "\nPyarrow", DeprecationWarning) -import pandas as pd # noqa: E402 -class RunSpec(OSeMOSYSBase): - # time definition +class RunSpec(OSeMOSYSBase, RunSpecOtoole): + # COMPONENTS + # ---------- time_definition: TimeDefinition - - # nodes regions: List[Region] - - # commodities commodities: List[Commodity] - - # Impact constraints (e.g. CO2) impacts: List[Impact] - - # technologies technologies: List[Technology] - storage_technologies: List[TechnologyStorage] + storage_technologies: List[TechnologyStorage] | None = Field(None) # TODO # production_technologies: List[TechnologyProduction] # transmission_technologies: List[TechnologyTransmission] - # Default values - defaults_otoole: Optional[DefaultsOtoole] = None - - @model_validator(mode="after") - def validation(cls, values): - values = check_tech_producing_commodity(values) - values = check_tech_producing_impact(values) - values = check_tech_consuming_commodity(values) - values = check_able_to_meet_demands(values) - - return values - - def to_xr_ds(self): - """ - Return the current RunSpec as an xarray dataset - - Args: - self: this RunSpec instance - - Returns: - xr.Dataset: An XArray dataset containing all data from the RunSpec - """ - - # Convert Runspec data to dfs - data_dfs = to_df_helper(self) - - # Set index to columns other than "VALUE" (only for parameter dataframes) - for df_name, df in data_dfs.items(): - if not df_name.isupper(): - data_dfs[df_name] = df.set_index(df.columns.difference(["VALUE"]).tolist()) - # Convert params to data arrays - data_arrays = {x: y.to_xarray()["VALUE"] for x, y in data_dfs.items() if not x.isupper()} - # Create dataset - ds = xr.Dataset(data_vars=data_arrays) + # ASSUMPIONS + # ---------- + depreciation_method: OSeMOSYSData_DepreciationMethod | None = Field( + OSeMOSYSData_DepreciationMethod(defaults.depreciation_method) + ) + discount_rate: OSeMOSYSData | None = Field(OSeMOSYSData(defaults.discount_rate)) + reserve_margin: OSeMOSYSData | None = Field(OSeMOSYSData(defaults.reserve_margin)) - # If runspec not generated using otoole config yaml, use linopy defaults - if self.defaults_otoole is None: - default_values = defaults.otoole_name_defaults - # If storage technologies present, use additional relevant default values - if self.storage_technologies: - default_values = {**default_values, **defaults.otoole_name_storage_defaults} - # Extract defaults data from OSeMOSYSData objects - for name, osemosys_data in default_values.items(): - default_values[name] = osemosys_data.data - # Otherwise take defaults from otoole config yaml file - else: - default_values = {} - for name, data in self.defaults_otoole.values.items(): - if data["type"] == "param": - default_values[name] = data["default"] - - # Replace any nan values in ds with default values (or None) for corresponding param, - # adding default values as attribute of each data array - for name in ds.data_vars.keys(): - # Replace nan values with default values if available - if name in default_values.keys(): - ds[name].attrs["default"] = default_values[name] - ds[name] = ds[name].fillna(default_values[name]) - # Replace all other nan values with None - # TODO: remove this code if nan values wanted in the ds - # else: - # ds[name].attrs["default"] = None - # ds[name] = ds[name].fillna(None) - - return ds - - def to_otoole(self, output_directory): - """ - Convert Runspec to otoole style output CSVs and config.yaml - - Parameters - ---------- - output_directory: str - Path to the output directory for CSV files to be placed - """ - - # Clear comparison directory - for file in os.listdir(output_directory): - os.remove(os.path.join(output_directory, file)) - - # Convert Runspec data to dfs - output_dfs = to_df_helper(self) - - # Duplicate source REGION df for destination _REGION df - output_dfs["_REGION"] = output_dfs["REGION"] - - # Write output CSVs - for file in list(output_dfs): - output_dfs[file].to_csv(os.path.join(output_directory, file + ".csv"), index=False) - - # Write empty storage CSVs if no storage technologies present - if not self.storage_technologies: - storage_csv_dict = TechnologyStorage.otoole_stems - for file in list(storage_csv_dict): - ( - pd.DataFrame(columns=storage_csv_dict[file]["column_structure"]).to_csv( - os.path.join(output_directory, file + ".csv"), index=False - ) - ) - pd.DataFrame(columns=["VALUE"]).to_csv( - os.path.join(output_directory, "STORAGE.csv"), index=False - ) - - # write config yaml if used to generate Runspec - if self.defaults_otoole: - yaml_file_path = os.path.join(output_directory, "config.yaml") - with open(yaml_file_path, "w") as yaml_file: - yaml.dump(self.defaults_otoole.values, yaml_file, default_flow_style=False) + # TARGETS + # ------- + renewable_production_target: OSeMOSYSData | None = Field(None) + @model_validator(mode="before") @classmethod - def from_otoole(cls, root_dir): - return cls( - id="id", - impacts=Impact.from_otoole_csv(root_dir=root_dir), - regions=Region.from_otoole_csv(root_dir=root_dir), - technologies=Technology.from_otoole_csv(root_dir=root_dir), - storage_technologies=TechnologyStorage.from_otoole_csv(root_dir=root_dir), - # TODO - # production_technologies=TechnologyProduction.from_otoole_csv(root_dir=root_dir), - # transmission_technologies=TechnologyTransmission.from_otoole_csv(root_dir=root_dir), - commodities=Commodity.from_otoole_csv(root_dir=root_dir), - time_definition=TimeDefinition.from_otoole_csv(root_dir=root_dir), - defaults_otoole=DefaultsOtoole.from_otoole_yaml(root_dir=root_dir), - ) - - def to_osemosys_data_file(self, root_dir): - """ - Convert Runspec to osemosys ready text file (uses otoole) + def cast_values(cls, values: Any) -> Any: + for field, info in cls.model_fields.items(): + field_val = values.get(field) + + if all( + [ + field_val is not None, + isinstance(field_val, int), + "OSeMOSYSData_Int" in str(info.annotation), + ] + ): + values[field] = OSeMOSYSData_Int(field_val) + elif all( + [ + field_val is not None, + isinstance(field_val, str), + "OSeMOSYSData_DepreciationMethod" in str(info.annotation), + ] + ): + values[field] = OSeMOSYSData_DepreciationMethod(field_val) + elif all( + [ + field_val is not None, + isnumeric(field_val), + "OSeMOSYSData" in str(info.annotation), + ] + ): + values[field] = OSeMOSYSData(field_val) - Parameters - ---------- - root_dir: str - Path to the directory containing data CSVs and yaml config file - - #TODO: acceptable to use otoole here or should otoole only be for post processing? - """ - - # depreciation_method=( - # OSeMOSYSDataInt( - # data=group_to_json( - # g=dfs["DepreciationMethod"].loc[ - # dfs["DepreciationMethod"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # target_column="VALUE", - # ) - # ) - # if "DepreciationMethod" not in otoole_cfg.empty_dfs - # else None - # ), - # discount_rate=( - # OSeMOSYSData( - # data=group_to_json( - # g=dfs["DiscountRate"].loc[ - # dfs["DiscountRate"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # target_column="VALUE", - # ) - # ) - # if "DiscountRate" not in otoole_cfg.empty_dfs - # else None - # ), - # discount_rate_idv=( - # OSeMOSYSData( - # data=group_to_json( - # g=dfs["DiscountRateIdv"].loc[ - # dfs["DiscountRateIdv"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["TECHNOLOGY"], - # target_column="VALUE", - # ) - # ) - # if "DiscountRateIdv" not in otoole_cfg.empty_dfs - # else None - # ), - # discount_rate_storage=( - # OSeMOSYSData( - # data=group_to_json( - # g=dfs["DiscountRateStorage"].loc[ - # dfs["DiscountRateStorage"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["STORAGE"], - # target_column="VALUE", - # ) - # ) - # if "DiscountRateStorage" not in otoole_cfg.empty_dfs - # else None - # ), - # reserve_margin=( - # OSeMOSYSData( - # data=group_to_json( - # g=dfs["ReserveMargin"].loc[ - # dfs["ReserveMargin"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["YEAR"], - # target_column="VALUE", - # ) - # ) - # if "ReserveMargin" not in otoole_cfg.empty_dfs - # else None - # ), - # reserve_margin_tag_fuel=( - # OSeMOSYSDataInt( - # data=group_to_json( - # g=dfs["ReserveMarginTagFuel"].loc[ - # dfs["ReserveMarginTagFuel"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["FUEL", "YEAR"], - # target_column="VALUE", - # ) - # ) - # if "ReserveMarginTagFuel" not in otoole_cfg.empty_dfs - # else None - # ), - # reserve_margin_tag_technology=( - # OSeMOSYSDataInt( - # data=group_to_json( - # g=dfs["ReserveMarginTagTechnology"].loc[ - # dfs["ReserveMarginTagTechnology"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["TECHNOLOGY", "YEAR"], - # target_column="VALUE", - # ) - # ) - # if "ReserveMarginTagTechnology" not in otoole_cfg.empty_dfs - # else None - # ), - # renewable_production_target=( - # OSeMOSYSData( - # data=group_to_json( - # g=dfs["REMinProductionTarget"].loc[ - # dfs["REMinProductionTarget"]["REGION"] == region["VALUE"] - # ], - # root_column="REGION", - # data_columns=["YEAR"], - # target_column="VALUE", - # ) - # ) - # if "REMinProductionTarget" not in otoole_cfg.empty_dfs - # else None - # ), - # ) + return values diff --git a/feo/osemosys/schemas/region.py b/feo/osemosys/schemas/region.py index e7e6bc89..ccebc984 100644 --- a/feo/osemosys/schemas/region.py +++ b/feo/osemosys/schemas/region.py @@ -1,9 +1,13 @@ from typing import List -from pydantic import Field +from pydantic import Field, model_validator from feo.osemosys.schemas.base import OSeMOSYSBase, OSeMOSYSData from feo.osemosys.schemas.compat.region import OtooleRegion +from feo.osemosys.schemas.validation.region_validation import ( + discount_rate_as_decimals, + reserve_margin_fully_defined, +) ########## # REGION # @@ -22,10 +26,8 @@ class Region(OSeMOSYSBase, OtooleRegion): neighbours: List[str] | None = Field(default=None) trade_routes: OSeMOSYSData | None = Field(default=None) - # composable params - # reserve_margin: OSeMOSYSData | None = Field(default=None) - # renewable_production_target: OSeMOSYSData | None = Field(default=None) - # discount_rate: OSeMOSYSData | None = Field(default=None) - - # discount_rate_idv: OSeMOSYSData | None = Field(default=None) - # discount_rate_storage: OSeMOSYSData | None = Field(default=None) + @model_validator(mode="before") + def validation(cls, values): + values = reserve_margin_fully_defined(values) + values = discount_rate_as_decimals(values) + return values diff --git a/feo/osemosys/schemas/technology.py b/feo/osemosys/schemas/technology.py index 271ce2b9..a4c439f9 100644 --- a/feo/osemosys/schemas/technology.py +++ b/feo/osemosys/schemas/technology.py @@ -3,7 +3,12 @@ from pydantic import Field, conlist, model_validator from feo.osemosys.defaults import defaults -from feo.osemosys.schemas.base import OSeMOSYSBase, OSeMOSYSData, OSeMOSYSData_Int +from feo.osemosys.schemas.base import ( + OSeMOSYSBase, + OSeMOSYSData, + OSeMOSYSData_Bool, + OSeMOSYSData_Int, +) from feo.osemosys.schemas.compat.technology import OtooleTechnology from feo.osemosys.schemas.validation.technology_validation import technology_storage_validation from feo.osemosys.schemas.validation.validation_utils import check_min_vals_lower_max @@ -99,6 +104,10 @@ class Technology(OSeMOSYSBase, OtooleTechnology): activity_total_max: OSeMOSYSData | None = Field(None) activity_total_min: OSeMOSYSData | None = Field(None) + # include this technology in joint reserve margin and renewables targets + include_in_joint_reserve_margin: OSeMOSYSData_Bool | None = Field(None) + include_in_joint_renewable_target: OSeMOSYSData_Bool | None = Field(None) + @model_validator(mode="before") @classmethod def cast_values(cls, values: Any) -> Any: @@ -160,20 +169,17 @@ def validate_min_lt_max(self): ): raise ValueError("Minimum total activity is not less than maximum total activity.") - return True + return self class TechnologyStorage(OSeMOSYSBase): """ Class to contain all information pertaining to storage technologies - """ - - capex: OSeMOSYSData | None - operating_life: OSeMOSYSData_Int | None # Lower bound to the amount of energy stored, as a fraction of the maximum, (0-1) # Level of storage at the beginning of first modelled year, in units of activity # Maximum discharging rate for the storage, in units of activity per year # Maximum charging rate for the storage, in units of activity per year + """ # REQUIRED PARAMETERS # ------------------- diff --git a/feo/osemosys/schemas/validation/validation_utils.py b/feo/osemosys/schemas/validation/validation_utils.py index 84210618..c41ea3f7 100644 --- a/feo/osemosys/schemas/validation/validation_utils.py +++ b/feo/osemosys/schemas/validation/validation_utils.py @@ -63,5 +63,5 @@ def check_min_vals_lower_max(min_data, max_data, columns): min_df, max_df, on=columns, suffixes=("_min", "_max"), how="outer" ).dropna() - # Check that values in min_data are lower than those in max_data - return (merged_df["VALUE_min"] <= merged_df["VALUE_max"]).all() + # Check that values in min_data are lower than those in max_data + return (merged_df["VALUE_min"] <= merged_df["VALUE_max"]).all() diff --git a/tests/test_compatability/test_otoole_runspec.py b/tests/test_compatability/test_otoole_runspec.py new file mode 100644 index 00000000..d287f212 --- /dev/null +++ b/tests/test_compatability/test_otoole_runspec.py @@ -0,0 +1,74 @@ +import glob +from pathlib import Path + +import pandas as pd + +from feo.osemosys.schemas import RunSpec +from tests.fixtures.paths import OTOOLE_SAMPLE_PATHS + + +def test_files_equality(): + """ + Check CSVs are equivalent after creating a RunSpec object from CSVs and writing to CSVs + """ + + comparison_root = "./tests/otoole_compare/" + + for path in OTOOLE_SAMPLE_PATHS: + comparison_model = Path(path).name + output_directory = Path(comparison_root + comparison_model) + output_directory.mkdir(parents=True, exist_ok=True) + + spec = RunSpec.from_otoole_csv(root_dir=path) + + spec.to_otoole_csv(output_directory=output_directory) + + comparison_files = glob.glob(str(output_directory) + "*.csv") + comparison_files = {Path(f).stem: f for f in comparison_files} + + original_files = glob.glob(path + "*.csv") + original_files = {Path(f).stem: f for f in original_files} + + check_files_equality(original_files, comparison_files) + + +def check_files_equality(original_files, comparison_files): + """ + Check if the files from original and comparison directories are equal. + """ + for stem, original_file in original_files.items(): + try: + original_df_sorted = ( + pd.read_csv(original_file) + .sort_values(by=pd.read_csv(original_file).columns.tolist()) + .reset_index(drop=True) + ) + # Cast all parameter values to floats + if not stem.isupper(): + original_df_sorted["VALUE"] = original_df_sorted["VALUE"].astype(float) + + comparison_df_sorted = ( + pd.read_csv(comparison_files[stem]) + .sort_values(by=pd.read_csv(comparison_files[stem]).columns.tolist()) + .reset_index(drop=True) + ) + # Cast all parameter values to floats + if not stem.isupper(): + comparison_df_sorted["VALUE"] = comparison_df_sorted["VALUE"].astype(float) + + if original_df_sorted.empty and comparison_df_sorted.empty and stem != "TradeRoute": + assert list(original_df_sorted.columns) == list( + comparison_df_sorted.columns + ), f"unequal files: {stem}" + elif original_df_sorted.empty and not comparison_df_sorted.empty: + pass + elif stem != "TradeRoute": + assert original_df_sorted.equals(comparison_df_sorted), f"unequal files: {stem}" + + except AssertionError as e: + print(f"Assertion Error: {e}") + print("---------- original_df_sorted ----------") + print(original_df_sorted.head(10)) + print("---------- comparison_df_sorted ----------") + print(comparison_df_sorted.head(10)) + raise diff --git a/tests/test_construction/test_load_from_yaml.py b/tests/test_construction/test_load_from_yaml.py index 998b8f5b..a9054f1a 100644 --- a/tests/test_construction/test_load_from_yaml.py +++ b/tests/test_construction/test_load_from_yaml.py @@ -62,4 +62,3 @@ def test_expression_parse(): def test_sample_construction(): for path in YAML_SAMPLE_PATHS: load_model(path) - assert True diff --git a/tests/test_construction/test_runspec.py b/tests/test_construction/test_runspec.py new file mode 100644 index 00000000..f5dafcc7 --- /dev/null +++ b/tests/test_construction/test_runspec.py @@ -0,0 +1,35 @@ +import pytest + +from feo.osemosys.schemas.model import RunSpec + +PASSING_RUNSPEC_DEFINITIONS = dict( + most_basic=dict( + time_definition=dict(id="years-only", years=range(2020, 2051)), + regions=[dict(id="GB")], + commodities=[dict(id="WATER")], + impacts=[dict(id="CO2e")], + technologies=[ + dict( + id="most_basic", + operating_life=10, + capex=15, + opex_fixed=1.5, + operating_modes=[dict(id="mode_1")], + ) + ], + ) +) + +FAILING_RUNSPEC_DEFINITIONS = dict() + + +def test_tech_construction(): + for name, params in PASSING_RUNSPEC_DEFINITIONS.items(): + spec = RunSpec(id=name, **params) + assert isinstance(spec, RunSpec) + + +def test_tech_construction_failcases(): + for _name, params in FAILING_RUNSPEC_DEFINITIONS.items(): + with pytest.raises(ValueError) as e: # noqa: F841 + RunSpec(**params)