Issue/probablistic (#202)

* add JSON column for ForecastValue ready for probablistic forecasts

* add into convert function

* add to forecast_value_latest

* add properties when updating foreast_value_latest

* add to tests

* start blending forecasts

* add method to just take properties from one model

* normalize properties values by the blended values

* add to tests

* pydantic==1.10.10

* add migrations

* add comments

* fix convert

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: peterdudfield

nowcasting_datamodel/fake.py

@@ -54,6 +54,7 @@ def make_fake_forecast_value(
         target_time=target_time,
         expected_power_generation_megawatts=power,
         adjust_mw=0.0,
+        properties={"10": power * 0.9, "90": power * 1.1},
     )
 
 

nowcasting_datamodel/migrations/forecast/versions/a6fb75892950_.py

@@ -0,0 +1,69 @@
+"""Add properties column to forecast_value
+
+Revision ID: a6fb75892950
+Revises: 08ba6879b865
+Create Date: 2023-07-03 11:00:31.216789
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a6fb75892950"
+down_revision = "08ba6879b865"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():  # noqa
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.add_column("forecast_value", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2022_09", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2022_10", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2022_11", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2022_12", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_01", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_02", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_03", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_04", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_05", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_06", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_07", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_08", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_09", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_10", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_11", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_2023_12", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column(
+        "forecast_value_last_seven_days", sa.Column("properties", sa.JSON(), nullable=True)
+    )
+    op.add_column("forecast_value_latest", sa.Column("properties", sa.JSON(), nullable=True))
+    op.add_column("forecast_value_old", sa.Column("properties", sa.JSON(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():  # noqa
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("forecast_value_old", "properties")
+    op.drop_column("forecast_value_latest", "properties")
+    op.drop_column("forecast_value_last_seven_days", "properties")
+    op.drop_column("forecast_value_2023_12", "properties")
+    op.drop_column("forecast_value_2023_11", "properties")
+    op.drop_column("forecast_value_2023_10", "properties")
+    op.drop_column("forecast_value_2023_09", "properties")
+    op.drop_column("forecast_value_2023_08", "properties")
+    op.drop_column("forecast_value_2023_07", "properties")
+    op.drop_column("forecast_value_2023_06", "properties")
+    op.drop_column("forecast_value_2023_05", "properties")
+    op.drop_column("forecast_value_2023_04", "properties")
+    op.drop_column("forecast_value_2023_03", "properties")
+    op.drop_column("forecast_value_2023_02", "properties")
+    op.drop_column("forecast_value_2023_01", "properties")
+    op.drop_column("forecast_value_2022_12", "properties")
+    op.drop_column("forecast_value_2022_11", "properties")
+    op.drop_column("forecast_value_2022_10", "properties")
+    op.drop_column("forecast_value_2022_09", "properties")
+    op.drop_column("forecast_value", "properties")
+
+    # ### end Alembic commands ###

nowcasting_datamodel/models/convert.py

@@ -67,6 +67,8 @@ def convert_df_to_national_forecast(
     :param forecast_values_df: Dataframe containing
         -- target_datetime_utc
         -- forecast_mw
+        -- (Optional) forecast_mw_plevel_10
+        -- (Optional) forecast_mw_plevel_90
     :param: session: database session
     :param: model_name: the name of the model
     :param: version: the version of the model
@@ -97,6 +99,14 @@ def convert_df_to_national_forecast(
             expected_power_generation_megawatts=forecast_value.forecast_mw,
         ).to_orm()
         forecast_value_sql.adjust_mw = 0.0
+
+        forecast_value_sql.properties = {}
+        if "forecast_mw_plevel_10" in forecast_values_df.columns:
+            forecast_value_sql.properties["10"] = forecast_value.forecast_mw_plevel_10
+
+        if "forecast_mw_plevel_90" in forecast_values_df.columns:
+            forecast_value_sql.properties["90"] = forecast_value.forecast_mw_plevel_90
+
         forecast_values.append(forecast_value_sql)
 
     # make forecast object

nowcasting_datamodel/models/forecast.py

@@ -12,6 +12,7 @@
 
 from pydantic import Field, validator
 from sqlalchemy import (
+    JSON,
     Boolean,
     Column,
     DateTime,
@@ -24,6 +25,7 @@
 )
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.declarative import DeclarativeMeta, declared_attr
+from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.ddl import DDL
 
@@ -112,6 +114,9 @@ class ForecastValueSQLMixin(CreatedMixin):
     target_time = Column(DateTime(timezone=True), nullable=False, primary_key=True)
     expected_power_generation_megawatts = Column(Float(precision=6))
     adjust_mw = Column(Float, default=0.0)
+    # this can be used to store any additional information about the forecast, like p_levels.
+    # Want to keep it as json so that we can store different properties for different forecasts
+    properties = Column(MutableDict.as_mutable(JSON), nullable=True)
 
     @declared_attr
     def forecast_id(self):
@@ -281,6 +286,9 @@ class ForecastValueLatestSQL(Base_Forecast, CreatedMixin):
     model_id = Column(Integer, index=True, primary_key=True, default=-1)
     is_primary = Column(Boolean, default=True)
     adjust_mw = Column(Float, default=0.0)
+    # this can be used to store any additional information about the forecast, like p_levels.
+    # Want to keep it as json so that we can store different properties for different forecasts
+    properties = Column(MutableDict.as_mutable(JSON), nullable=True)
 
     forecast_id = Column(Integer, ForeignKey("forecast.id"), index=True)
     forecast_latest = relationship("ForecastSQL", back_populates="forecast_values_latest")
@@ -310,6 +318,14 @@ class ForecastValue(EnhancedBaseModel):
         "The _ at the start means it is not expose in the API",
     )
 
+    # This its better to keep this out of the current pydantic models used by the API.
+    # A new pydantic mode can be made that includes the forecast plevels, perhaps in the API.
+    _properties: dict = Field(
+        None,
+        description="Dictionary to hold properties of the forecast, like p_levels. "
+        "The _ at the start means it is not expose in the API",
+    )
+
     _normalize_target_time = validator("target_time", allow_reuse=True)(datetime_must_have_timezone)
 
     def to_orm(self) -> ForecastValueSQL:

nowcasting_datamodel/models/gsp.py

@@ -53,8 +53,6 @@ class Location(EnhancedBaseModel):
         None, description="The installed capacity of the GSP in MW"
     )
 
-    rm_mode = True
-
     def to_orm(self) -> LocationSQL:
         """Change model to LocationSQL"""
 

nowcasting_datamodel/models/metric.py

@@ -41,8 +41,6 @@ class Metric(EnhancedBaseModel):
     name: str = Field(..., description="The name of the metric")
     description: str = Field(..., description="The description of the metric")
 
-    rm_mode = True
-
     def to_orm(self) -> MetricSQL:
         """Change model to LocationSQL"""
 
@@ -170,8 +168,6 @@ class MetricValue(EnhancedBaseModel):
     )
     location: Location = Field(..., description="The location object for this metric value")
 
-    rm_mode = True
-
     def to_orm(self) -> MetricValueSQL:
         """Change model to MetricValueSQL"""
 

nowcasting_datamodel/read/blend/blend.py

@@ -6,9 +6,11 @@
 
 """
 
+import json
 from datetime import datetime
 from typing import List, Optional
 
+import pandas as pd
 import structlog
 from sqlalchemy.orm.session import Session
 
@@ -34,6 +36,7 @@ def get_blend_forecast_values_latest(
     model_names: Optional[List[str]] = None,
     weights: Optional[List[float]] = None,
     forecast_horizon_minutes: Optional[int] = None,
+    properties_model: Optional[str] = None,
 ) -> List[ForecastValue]:
     """
     Get forecast values
@@ -44,6 +47,7 @@ def get_blend_forecast_values_latest(
         If None is given then all are returned.
     :param model_names: list of model names to use for blending
     :param weights: list of weights to use for blending, see structure in make_weights_df
+    :param properties_model: the model to use for the properties
 
     return: List of forecasts values blended from different models
     """
@@ -59,6 +63,11 @@ def get_blend_forecast_values_latest(
     else:
         weights_df = None
 
+    if properties_model is not None:
+        assert (
+            properties_model in model_names
+        ), f"properties_model must be in model_names {model_names}"
+
     # get forecast for the different models
     forecast_values_all_model = []
     for model_name in model_names:
@@ -98,7 +107,72 @@ def get_blend_forecast_values_latest(
     # blend together
     forecast_values_blended = blend_forecasts_together(forecast_values_all_model, weights_df)
 
+    # add properties
+    forecast_values_df = add_properties_to_forecast_values(
+        blended_df=forecast_values_blended,
+        properties_model=properties_model,
+        all_model_df=forecast_values_all_model,
+    )
+
     # convert back to list of forecast values
-    forecast_values = convert_df_to_list_forecast_values(forecast_values_blended)
+    forecast_values = convert_df_to_list_forecast_values(forecast_values_df)
 
     return forecast_values
+
+
+def add_properties_to_forecast_values(
+    blended_df: pd.DataFrame,
+    all_model_df: pd.DataFrame,
+    properties_model: Optional[str] = None,
+):
+    """
+    Add properties to blended forecast values, we just take it from one model.
+
+    We normalize all properties by the "expected_power_generation_megawatts" value,
+    and renormalize by the blended "expected_power_generation_megawatts" value.
+    This makes sure that plevels 10 and 90 surround the blended value.
+
+    :param blended_df: dataframe of blended forecast values
+    :param properties_model: which model to use for properties
+    :param all_model_df: dataframe of all forecast values for all models
+    :return:
+    """
+
+    logger.debug(
+        f"Adding properties to blended forecast values for properties_model {properties_model}"
+    )
+
+    if properties_model is None:
+        blended_df["properties"] = None
+        return blended_df
+
+    # get properties
+
+    properties_df = all_model_df[all_model_df["model_name"] == properties_model]
+
+    # adjust "properties" to be relative to the expected_power_generation_megawatts
+    # this is a bit tricky becasue the "properties" column is a list of dictionaries
+    # below we add "expected_power_generation_megawatts" value back to this.
+    # We do this so that plevels are relative to the blended values.
+    properties_only_df = pd.json_normalize(properties_df["properties"])
+    for c in properties_only_df.columns:
+        properties_only_df[c] -= properties_df["expected_power_generation_megawatts"]
+    properties_df["properties"] = properties_only_df.apply(
+        lambda x: json.loads(x.to_json()), axis=1
+    )
+
+    # reduce columns
+    properties_df = properties_df[["target_time", "properties"]]
+
+    # add properties to blended forecast values
+    blended_df = blended_df.merge(properties_df, on=["target_time"], how="left")
+
+    # add "expected_power_generation_megawatts" to the properties
+    properties_only_df = pd.json_normalize(blended_df["properties"])
+    for c in properties_only_df.columns:
+        properties_only_df[c] += blended_df["expected_power_generation_megawatts"]
+    blended_df["properties"] = properties_only_df.apply(lambda x: json.loads(x.to_json()), axis=1)
+
+    assert "properties" in blended_df.columns
+
+    return blended_df