Issue/no nan adjust mw (#195)

* TDD: add a test, to make sure adjust mw are zero

* force adjust_mw to be 0.0 if me not found

* fix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix

* fix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: peterdudfield

nowcasting_datamodel/read/read_metric.py

@@ -132,12 +132,13 @@ def read_latest_me_national(
     metric_values = query.all()
 
     # add timezone, show be the same datetime interval for all
-    datetime_interval = metric_values[0].datetime_interval
-    datetime_interval.start_datetime_utc = datetime_interval.start_datetime_utc.replace(
-        tzinfo=timezone.utc
-    )
-    datetime_interval.end_datetime_utc = datetime_interval.end_datetime_utc.replace(
-        tzinfo=timezone.utc
-    )
+    if len(metric_values) > 0:
+        datetime_interval = metric_values[0].datetime_interval
+        datetime_interval.start_datetime_utc = datetime_interval.start_datetime_utc.replace(
+            tzinfo=timezone.utc
+        )
+        datetime_interval.end_datetime_utc = datetime_interval.end_datetime_utc.replace(
+            tzinfo=timezone.utc
+        )
 
     return metric_values

nowcasting_datamodel/save/adjust.py

@@ -3,6 +3,7 @@
 from datetime import datetime, timedelta
 from typing import List, Union
 
+import numpy as np
 import pandas as pd
 
 from nowcasting_datamodel.models import Forecast, ForecastSQL, MetricValue, MetricValueSQL
@@ -56,8 +57,10 @@ def add_adjust_to_national_forecast(forecast: ForecastSQL, session):
 
     # 1. read metric values
     latest_me = read_latest_me_national(session=session, model_name=model_name)
-    assert len(latest_me) > 0
-    logger.debug(f"Found {len(latest_me)} latest ME values")
+    if len(latest_me) == 0:
+        logger.warning(f"Found no ME values found for {model_name=}")
+    else:
+        logger.debug(f"Found {len(latest_me)} latest ME values")
 
     # 2. filter value down to now onwards
     # get the number of hours to go ahead, we've added 1 to make sure we use the last one as well
@@ -78,9 +81,19 @@ def add_adjust_to_national_forecast(forecast: ForecastSQL, session):
 
             # add value to ForecastValueSQL
             forecast_value.adjust_mw = value
+            if np.isnan(value):
+                logger.debug(
+                    f"Found ME value for {target_time} in {latest_me_df}, "
+                    f"but it was NaN, therefore adding adjust_mw as 0"
+                )
+                forecast_value.adjust_mw = 0.0
 
         except Exception:
-            logger.debug(f"Could not find ME value for {target_time} in {latest_me_df}")
+            logger.debug(
+                f"Could not find ME value for {target_time} in {latest_me_df}, "
+                f"therefore adding adjust_mw as 0"
+            )
+            forecast_value.adjust_mw = 0.0
 
 
 def get_forecast_horizon_from_forecast(forecast: Union[ForecastSQL, Forecast]):
@@ -144,6 +157,9 @@ def reduce_metric_values_to_correct_forecast_horizon(
     )
 
     latest_me_df = pd.DataFrame([MetricValue.from_orm(m).dict() for m in latest_me])
+    if len(latest_me_df) == 0:
+        # no latest ME values, so just making an empty dataframe
+        latest_me_df = pd.DataFrame(columns=["forecast_horizon_minutes", "time_of_day", "value"])
 
     # Let now big a dataframe of datetimes from now onwards. Lets say the time is 04.30, then
     # time forecast_horizon value
@@ -157,7 +173,6 @@ def reduce_metric_values_to_correct_forecast_horizon(
             start=datetime_now, end=datetime_now + timedelta(hours=hours_ahead), freq="30T"
         ),
     )
-    print(results_df)
     results_df["datetime"] = results_df.index
     results_df["time_of_day"] = results_df.index.time
     results_df["forecast_horizon_minutes"] = range(0, 30 * len(results_df), 30)

tests/save/test_adjust.py

@@ -49,6 +49,20 @@ def test_add_adjust_to_forecasts(latest_me, db_session):
     assert forecasts[1].forecast_values[0].adjust_mw == 0.0
 
 
+@freeze_time("2023-01-09 16:25")
+def test_add_adjust_to_forecasts_no_me_values(db_session):
+    """Test to check, if there are no me values, make sure the adjust mw are 0 not NaN"""
+
+    datetime_now = datetime(2023, 1, 9, 16, 30, tzinfo=timezone.utc)
+    forecasts = make_fake_forecasts(
+        gsp_ids=list(range(0, 2)), session=db_session, t0_datetime_utc=datetime_now
+    )
+
+    add_adjust_to_forecasts(session=db_session, forecasts_sql=forecasts)
+
+    assert forecasts[1].forecast_values[0].adjust_mw == 0.0
+
+
 def test_get_forecast_horizon_from_forecast(db_session):
     forecasts = make_fake_forecasts(gsp_ids=range(0, 1), session=db_session)