Merge pull request #23 from Geosyntec/features-dry-weather-and-divers…

…ions

Features dry weather and diversions

lyra/lyra/api/endpoints/plot.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import orjson
 from altair.utils.data import MaxRowsError
@@ -11,12 +11,13 @@
 from lyra.api.requests import LyraRoute
 from lyra.core.errors import HydstraIOError
 from lyra.models.plot_models import (
+    DiversionScenarioSchema,
     ListTimeseriesSchema,
     MultiVarSchema,
     RegressionSchema,
 )
 from lyra.models.response_models import ChartJSONResponse
-from lyra.src.viz import multi_variable, regression
+from lyra.src.viz import diversion_scenario, multi_variable, regression
 
 router = APIRouter(route_class=LyraRoute, default_response_class=ORJSONResponse)
 templates = Jinja2Templates(directory="lyra/site/templates")
@@ -235,15 +236,14 @@ def plot_multi_variable(
     req: MultiVarSchema = Depends(multi_var_schema_query),
     f: str = Query("json", regex="json$|html$"),
 ) -> Dict:
-    """Crate Multiple Timeseries Plots
+    """Create Multiple Timeseries Plots
     """
 
     chart_spec = None
     chart_status = None
     msg = []
 
     try:
-
         ts = multi_variable.make_timeseries(jsonable_encoder(req.timeseries))
         source = multi_variable.make_source(ts)
         warnings = ["\n".join(t.warnings) for t in ts]
@@ -271,7 +271,8 @@ def plot_multi_variable(
 
     if f == "html":
         return templates.TemplateResponse(  # type: ignore
-            "anyspec.html", {"request": request, "response": response,},
+            "anyspec.html",
+            {"request": request, "response": response, "title": "Timeseries"},
         )
 
     return response
@@ -365,7 +366,8 @@ def plot_regression(
 
     if f == "html":
         return templates.TemplateResponse(  # type: ignore
-            "anyspec.html", {"request": request, "response": response,},
+            "anyspec.html",
+            {"request": request, "response": response, "title": "Regression Analysis"},
         )
 
     return response
@@ -389,3 +391,106 @@ def plot_regression_data(
     else:
         _json = jsonable_encoder(regression.make_source_json(source))
         return ORJSONResponse(_json)
+
+
+### Diversion Scenario
+def diversion_scenario_schema_query(
+    request: Request,
+    site: Optional[str] = Query(None, example="ALISO_STP"),
+    start_date: Optional[str] = Query(None, example="2021-05-01"),
+    end_date: Optional[str] = Query(None, example="2021-08-01"),
+    diversion_rate_cfs: Optional[float] = Query(None, example=3.5),
+    storage_max_depth_ft: Optional[float] = 0.0,
+    storage_initial_depth_ft: Optional[float] = 0.0,
+    storage_area_sqft: Optional[float] = 0.0,
+    infiltration_rate_inhr: Optional[float] = 0.0,
+    diversion_months_active: Optional[List[int]] = Query(None),
+    diversion_days_active: Optional[List[int]] = Query(None),
+    diversion_hours_active: Optional[List[int]] = Query(None),
+    operated_weather_condition: Optional[str] = None,
+    nearest_rainfall_station: Optional[str] = None,
+    string: Optional[str] = Query(None, alias="json",),
+) -> DiversionScenarioSchema:
+    try:
+        if string is not None:
+            json_parsed = orjson.loads(string)
+            rsp = DiversionScenarioSchema(**json_parsed)
+        else:
+            rsp = DiversionScenarioSchema(**dict(request.query_params))  # type: ignore
+
+    except ValidationError as e:
+        raise HTTPException(status_code=422, detail=e.errors())
+
+    return rsp
+
+
+@router.get(
+    "/diversion_scenario", response_model=ChartJSONResponse,
+)
+def plot_diversion_scenario(
+    request: Request,
+    req: DiversionScenarioSchema = Depends(diversion_scenario_schema_query),
+    f: str = Query("json", regex="json$|html$"),
+) -> Dict:
+    """Create Diversion Scenario Plots
+    """
+
+    chart_spec = None
+    chart_status = None
+    msg = []
+
+    try:
+        # ts = multi_variable.make_timeseries(jsonable_encoder(req.timeseries))
+        source = diversion_scenario.make_source(**jsonable_encoder(req))
+        # warnings = ["\n".join(t.warnings) for t in ts]
+        # msg += warnings
+
+        chart = diversion_scenario.make_plot(source)
+        chart_spec = chart.to_dict()
+        chart_status = "SUCCESS"
+
+    except HydstraIOError as e:
+        chart_status = "FAILURE"
+        msg += [str(e)]
+
+    except MaxRowsError:
+        chart_status = "FAILURE"
+        msg += ["max data exceeded. Default max is 5000 data points"]
+
+    chart_pkg = {
+        "spec": chart_spec,
+        "chart_status": chart_status,
+        "messages": msg,
+    }
+
+    response = {"data": chart_pkg}
+
+    if f == "html":
+        return templates.TemplateResponse(  # type: ignore
+            "anyspec.html",
+            {"request": request, "response": response, "title": "Diversion Scenario"},
+        )
+
+    return response
+
+
+@router.get("/diversion_scenario/data")
+def plot_diversion_scenario_data(
+    req: DiversionScenarioSchema = Depends(diversion_scenario_schema_query),
+    f: str = Query("json"),
+) -> Union[ORJSONResponse, PlainTextResponse]:
+
+    try:
+        # ts = multi_variable.make_timeseries(jsonable_encoder(req.timeseries))
+        source = diversion_scenario.make_source(**jsonable_encoder(req))
+        # warnings = ["\n".join(t.warnings) for t in ts]
+
+    except HydstraIOError as e:
+        return ORJSONResponse({"error": str(e)})
+
+    if f == "csv":
+        csv = diversion_scenario.make_source_csv(source)
+        return PlainTextResponse(csv)
+    else:
+        _json = jsonable_encoder(diversion_scenario.make_source_json(source))
+        return ORJSONResponse(_json)

lyra/lyra/connections/database.py

@@ -4,7 +4,7 @@
 
 import pandas
 from sqlalchemy import create_engine
-from sqlalchemy.engine.url import URL
+from sqlalchemy.engine import URL
 from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed
 
 from lyra.core.config import settings
@@ -39,7 +39,7 @@ def sql_server_connection_string(
     timeout: int = 15,
 ) -> str:  # pragma: no cover
 
-    url = URL(
+    url = URL.create(
         drivername="mssql+pyodbc",
         username=user,
         password=password,

lyra/lyra/models/plot_models.py

@@ -9,7 +9,12 @@
 from lyra.core import utils
 from lyra.core.config import cfg
 from lyra.core.io import load_file
-from lyra.models.request_models import AggregationMethod, Interval, RegressionMethod
+from lyra.models.request_models import (
+    AggregationMethod,
+    Interval,
+    RegressionMethod,
+    Weather,
+)
 
 VALID_VARIABLES = list(cfg["variables"].keys())
 
@@ -107,6 +112,7 @@ class TimeseriesBaseSchema(BaseModel):
     start_date: Optional[str] = Field(None, example="2015-01-01")
     end_date: Optional[str] = Field(None, example="2020-01-01")
     interval: Optional[Interval] = None
+    weather_condition: Optional[Weather] = None
 
     @validator("start_date", pre=True, always=True)
     def set_start_date(cls, start_date):
@@ -120,6 +126,10 @@ def set_end_date(cls, end_date):
     def set_interval(cls, interval):
         return interval or "month"
 
+    @validator("weather_condition", pre=True, always=True)
+    def set_weather_condition(cls, weather_condition):
+        return weather_condition or "both"
+
     @root_validator
     def check_date_order(cls, values):
         assert values["start_date"] < values["end_date"], (
@@ -138,6 +148,7 @@ class TimeseriesSchema(TimeseriesBaseSchema):
     site: str = Field(..., example="ALISO_JERONIMO")
     trace_upstream: Optional[bool] = True
     aggregation_method: Optional[AggregationMethod] = Field(None, example="mean")
+    nearest_rainfall_station: Optional[str] = Field(None, example="ALISO_JERONIMO")
 
     @validator("site")
     def check_site(cls, v):
@@ -166,24 +177,68 @@ def set_trace_upstream(cls, trace_upstream):
             return True
         return trace_upstream
 
+    @validator("nearest_rainfall_station", pre=True, always=True)
+    def set_nearest_rainfall_station(cls, nearest_rainfall_station):
+        if nearest_rainfall_station is not None:
+            cls.check_site(nearest_rainfall_station)
+        return nearest_rainfall_station
+
+    @root_validator
+    def check_nearest_rainfall_station_if_provided(cls, values):
+        nearest_station = values.get("nearest_rainfall_station")
+
+        if nearest_station is not None:
+
+            site_props = [
+                f["properties"]
+                for f in json.loads(load_file(cfg["site_path"]))["features"]
+            ]
+
+            nearest_site_info: Dict = next(
+                (x for x in site_props if x["station"] == nearest_station), {}
+            )
+            assert nearest_site_info.get(
+                f"has_rainfall"
+            ), f"'rainfall' not found at site {nearest_station!r}."
+
+        return values
+
     @root_validator
     def check_site_variable(cls, values):
         site = values.get("site", r"¯\_(ツ)_/¯")
         variable = values.get("variable", r"¯\_(ツ)_/¯")
         source = cfg.get("variables", {}).get(variable, {}).get("source")
+        nearest_station = values.get("nearest_rainfall_station")
 
         # load_file will cache this so it doesn't happen for frequent requests
         site_props = [
             f["properties"] for f in json.loads(load_file(cfg["site_path"]))["features"]
         ]
-        var_info: Dict = next((x for x in site_props if x["station"] == site), {})
+        site_info: Dict = next((x for x in site_props if x["station"] == site), {})
 
         # we only need to validate that the variable is available for hydstra
         # variables. Dt_metric variables are always available.
         if source == "hydstra":
-            assert var_info.get(
+
+            if variable == "rainfall" and not site_info.get(f"has_{variable}"):
+                nearest_station = (
+                    values.get("nearest_rainfall_station")
+                    or site_info["nearest_rainfall_station"]
+                )
+
+                warnings = values.get("warnings", [])
+                warnings.append(
+                    f"Warning: variable {variable!r} not available for site {site!r}. "
+                    f"Overriding to nearest site {nearest_station!r}"
+                )
+                values["warnings"] = warnings
+                values["site"] = nearest_station
+
+                return values
+
+            assert site_info.get(
                 f"has_{variable}"
-            ), f"'{variable}' not found at site '{site}'."
+            ), f"{variable!r} not found at site {site!r}."
 
         return values
 
@@ -232,6 +287,21 @@ def check_dt_metric_dates(cls, values):
 
         return values
 
+    @root_validator
+    def check_dt_metric_weather_condition(cls, values):
+        variable = values.get("variable", r"¯\_(ツ)_/¯")
+        source = cfg.get("variables", {}).get(variable, {}).get("source")
+        weather_condition = values.get("weather_condition", "both")
+
+        if source == "dt_metrics" and weather_condition != "both":
+            warnings = values.get("warnings", [])
+            warnings.append(
+                f"Warning: `weather_condition` option is ignored for DT Metrics."
+            )
+            values["warnings"] = warnings
+
+        return values
+
 
 class ListTimeseriesSchema(BaseModel):
     timeseries: Union[List[TimeseriesSchema], str]
@@ -282,13 +352,56 @@ def check_len_timeseries(cls, timeseries):
     def check_interval(cls, values):
         timeseries = values.get("timeseries")
         sources = []
+        intervals = []
 
         for ts in timeseries:
             sources.append(cfg.get("variables", {}).get(ts.variable, {}).get("source"))
+            intervals.append(ts.interval)
+
+        assert all(
+            i == intervals[0] for i in intervals
+        ), f"invervals must be the same for both timeseries. '{intervals[0]}' != '{intervals[1]}'"
 
         if "dt_metrics" in sources:
             assert all(
                 ts.interval in ["month", "year"] for ts in timeseries
             ), f"inverval must be 'month' or 'year' for drool tool metrics."
 
         return values
+
+
+class DiversionScenarioSchema(BaseModel):
+    site: str
+    start_date: str
+    end_date: str
+    diversion_rate_cfs: float
+    storage_max_depth_ft: Optional[float] = 0.0
+    storage_initial_depth_ft: Optional[float] = 0.0
+    storage_area_sqft: Optional[float] = 0.0
+    infiltration_rate_inhr: Optional[float] = 0.0
+    diversion_months_active: Optional[List[int]] = None
+    diversion_days_active: Optional[List[int]] = None
+    diversion_hours_active: Optional[List[int]] = None
+    operated_weather_condition: Optional[Weather] = None
+    nearest_rainfall_station: Optional[str] = None
+
+    @validator("operated_weather_condition", pre=True, always=True)
+    def set_operated_weather_condition(cls, operated_weather_condition):
+        return operated_weather_condition or "dry"
+
+    @root_validator
+    def check_site(cls, values):
+        ts = TimeseriesSchema(
+            site=values.get("site"),
+            start_date=values.get("start_date"),
+            end_date=values.get("end_date"),
+            nearest_rainfall_station=values.get("nearest_rainfall_station"),
+            weather_condition=Weather.both,  # we are initializing discharge timeseries here, not the diversion behavior
+            variable="discharge",
+            aggregation_method=AggregationMethod.mean,
+            interval=Interval.hour,
+        )
+
+        values["ts"] = ts.dict()
+
+        return values

lyra/lyra/models/request_models.py

@@ -33,3 +33,9 @@ class SpatialResponseFormat(str, Enum):
 class ResponseFormat(str, Enum):
     json = "json"
     html = "html"
+
+
+class Weather(str, Enum):
+    both = "both"
+    wet = "wet"
+    dry = "dry"

lyra/lyra/site/templates/anyspec.html

@@ -33,7 +33,7 @@
 {% endblock %} {% block content %}
 <div class="dashboard">
   <div class="db_title">
-    <h1>Timeseries</h1>
+    <h1>{{title or "Timeseries"}}</h1>
   </div>
 
   <div id="vis-timeseries"></div>

lyra/lyra/src/diversion/__init__.py

@@ -0,0 +1 @@
+from lyra.src.diversion.scenario import simulate_diversion