fix for database

Author: peterdudfield

ocf_datapipes/load/gsp/database.py

@@ -59,14 +59,11 @@ def __iter__(self) -> xr.DataArray:
         gsp_id_to_shape = get_gsp_shape_from_eso(return_filename=False)
 
         # Ensure the centroids have the same GSP ID index as the GSP PV power:
-        logger.debug(gsp_pv_power_mw_df.columns)
-        logger.debug(gsp_pv_power_mw_df)
-        logger.debug(gsp_id_to_shape)
         gsp_id_to_shape = gsp_id_to_shape.loc[gsp_pv_power_mw_df.columns]
 
         data_array = put_gsp_data_into_an_xr_dataarray(
             gsp_pv_power_mw=gsp_pv_power_mw_df.astype(np.float32),
-            time_utc=gsp_pv_power_mw_df.index,
+            time_utc=gsp_pv_power_mw_df.index.values,
             gsp_id=gsp_pv_power_mw_df.columns,
             # TODO: Try using `gsp_id_to_shape.geometry.envelope.centroid`. See issue #76.
             x_osgb=gsp_id_to_shape.geometry.centroid.x.astype(np.float32),
@@ -76,6 +73,7 @@ def __iter__(self) -> xr.DataArray:
 
         del gsp_id_to_shape, gsp_pv_power_mw_df
         while True:
+            logger.debug(data_array)
             yield data_array
 
 
@@ -98,6 +96,8 @@ def get_gsp_power_from_database(
 
     logger.info("Loading GSP data from database")
     logger.debug(f"{history_duration=}")
+    logger.debug(f"{interpolate_minutes=}")
+    logger.debug(f"{load_extra_minutes=}")
 
     extra_duration = timedelta(minutes=load_extra_minutes)
     now = pd.to_datetime(datetime.now(tz=timezone.utc)).floor("30T")
@@ -153,10 +153,12 @@ def get_gsp_power_from_database(
         ["datetime_utc", "gsp_id", "solar_generation_mw", "installed_capacity_mw"]
     ]
     logger.debug(gsp_yields_df.columns)
+    logger.debug(gsp_yields_df.index)
     gsp_yields_df.drop_duplicates(
         ["datetime_utc", "gsp_id", "solar_generation_mw"], keep="last", inplace=True
     )
     logger.debug(gsp_yields_df.columns)
+    logger.debug(gsp_yields_df.index)
     gsp_power_df = gsp_yields_df.pivot(
         index="datetime_utc", columns="gsp_id", values="solar_generation_mw"
     )

ocf_datapipes/production/power_perceiver.py

@@ -84,7 +84,6 @@ def power_perceiver_production_datapipe(configuration_filename: Union[Path, str]
         interpolate_minutes=configuration.input_data.gsp.live_interpolate_minutes,
         load_extra_minutes=configuration.input_data.gsp.live_load_extra_minutes,
     )
-
     logger.debug("Normalize GSP data")
     gsp_datapipe = gsp_datapipe.normalize(
         normalize_fn=lambda x: x / x.capacity_megawatt_power

tests/config/test.yaml

@@ -5,6 +5,7 @@ git: null
 input_data:
   gsp:
     gsp_zarr_path: tests/data/gsp/test.zarr
+    history_minutes: 120
   nwp:
     nwp_channels:
       - t

tests/conftest.py

@@ -227,17 +227,19 @@ def pv_yields_and_systems(db_session):
 def gsp_yields(db_session):
     """Make fake GSP data"""
 
-    gsp_sql_1: LocationSQL = Location(gsp_id=1, label="GSP_1", installed_capacity_mw=1).to_orm()
-
     gsp_yield_sqls = []
-    for hour in range(0, 8):
-        for minute in range(0, 60, 30):
-            gsp_yield_1 = GSPYield(
-                datetime_utc=datetime(2022, 1, 1, hour, minute), solar_generation_kw=hour + minute
-            )
-            gsp_yield_1_sql = gsp_yield_1.to_orm()
-            gsp_yield_1_sql.location = gsp_sql_1
-            gsp_yield_sqls.append(gsp_yield_1_sql)
+    for gsp_id in range(1,3):
+        gsp_sql_1: LocationSQL = Location(gsp_id=gsp_id, label="GSP_1", installed_capacity_mw=1).to_orm()
+
+        gsp_yield_sqls = []
+        for hour in range(0, 8):
+            for minute in range(0, 60, 30):
+                gsp_yield_1 = GSPYield(
+                    datetime_utc=datetime(2022, 1, 1, hour, minute), solar_generation_kw=hour + minute
+                )
+                gsp_yield_1_sql = gsp_yield_1.to_orm()
+                gsp_yield_1_sql.location = gsp_sql_1
+                gsp_yield_sqls.append(gsp_yield_1_sql)
 
     # add to database
     db_session.add_all(gsp_yield_sqls)

tests/production/test_pp_production.py

@@ -21,7 +21,7 @@ def test_pp_production_datapipe(pv_yields_and_systems, gsp_yields):
     assert len(batch[BatchKey.nwp_target_time_utc][0]) == 3
     assert len(batch[BatchKey.nwp_init_time_utc][0]) == 3
     assert len(batch[BatchKey.pv_time_utc][0]) == 19
-    assert len(batch[BatchKey.gsp_time_utc][0]) == 4  # 1 history + now + 2 future = 4
+    assert len(batch[BatchKey.gsp_time_utc][0]) == 7  # 4 history + now + 2 future = 4
 
     assert batch[BatchKey.hrvsatellite_actual].shape == (
         4,
@@ -32,5 +32,5 @@ def test_pp_production_datapipe(pv_yields_and_systems, gsp_yields):
     )  # 2nd dim is 6 history + now
     assert batch[BatchKey.nwp].shape == (4, 3, 1, 2, 2)  # 2nd dim is 1 history + now + 1 future
     assert batch[BatchKey.pv].shape == (4, 7, 32)  # 2nd dim is 6 history + now
-    assert batch[BatchKey.gsp].shape == (4, 2, 1)  # 2nd dim is 1 history + now
+    assert batch[BatchKey.gsp].shape == (4, 5, 1)  # 2nd dim is 4 history + now
     assert batch[BatchKey.hrvsatellite_surface_height].shape == (4, 64, 64)