[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Author: pre-commit-ci[bot]

.github/workflows/workflows.yaml

@@ -14,5 +14,5 @@ jobs:
       pytest_cov_dir: "ocf_datapipes"
       # extra things to install
       sudo_apt_install: "libgeos++-dev libproj-dev proj-data proj-bin"
-#      brew_install: "proj geos librttopo"
-      os_list:   '["ubuntu-latest"]'
+      #      brew_install: "proj geos librttopo"
+      os_list: '["ubuntu-latest"]'

ocf_datapipes/load/configuration.py

@@ -1,6 +1,7 @@
+import logging
+
 import fsspec
 from pathy import Pathy
-import logging
 from pyaml_env import parse_config
 from torchdata.datapipes import functional_datapipe
 from torchdata.datapipes.iter import IterDataPipe
@@ -16,11 +17,11 @@ def __init__(self, configuration_filename: str):
         self.configuration_filename = configuration_filename
 
     def __iter__(self):
-        logger.debug(f'Going to open {self.configuration_filename}')
+        logger.debug(f"Going to open {self.configuration_filename}")
         with fsspec.open(self.configuration_filename, mode="r") as stream:
             configuration = parse_config(data=stream)
 
-        logger.debug(f'Converting to Configuration ({configuration})')
+        logger.debug(f"Converting to Configuration ({configuration})")
         configuration = Configuration(**configuration)
 
         while True:

ocf_datapipes/load/pv.py

@@ -214,10 +214,10 @@ def _load_pv_metadata(filename: str) -> pd.DataFrame:
             latitude, longitude, system_id, x_osgb, y_osgb
     """
     _log.info(f"Loading PV metadata from {filename}")
-    if 'passiv' in str(filename):
-        index_col = 'ss_id'
+    if "passiv" in str(filename):
+        index_col = "ss_id"
     else:
-        index_col = 'system_id'
+        index_col = "system_id"
     pv_metadata = pd.read_csv(filename, index_col=index_col)
 
     if "Unnamed: 0" in pv_metadata.columns:

ocf_datapipes/load/satellite.py

@@ -45,12 +45,24 @@ def open_sat_data(
     # Note that `rename` renames *both* the coordinates and dimensions, and keeps
     # the connection between the dims and coordinates, so we don't have to manually
     # use `data_array.set_index()`.
-    dataset = dataset.rename({"time": "time_utc",})
+    dataset = dataset.rename(
+        {
+            "time": "time_utc",
+        }
+    )
     if "y" in dataset.coords.keys():
-        dataset = dataset.rename({"y": "y_geostationary",})
+        dataset = dataset.rename(
+            {
+                "y": "y_geostationary",
+            }
+        )
 
     if "x" in dataset.coords.keys():
-        dataset = dataset.rename({"x": "x_geostationary",})
+        dataset = dataset.rename(
+            {
+                "x": "x_geostationary",
+            }
+        )
 
     # Flip coordinates to top-left first
     if dataset.y_geostationary[0] < dataset.y_geostationary[-1]:

ocf_datapipes/transform/xarray/add_t0idx_and_sample_period_duration.py

@@ -1,14 +1,21 @@
+from datetime import timedelta
 from typing import Union
 
 import xarray as xr
 from torchdata.datapipes import functional_datapipe
 from torchdata.datapipes.iter import IterDataPipe
-from datetime import timedelta
+
 
 @functional_datapipe("add_t0_idx_and_sample_period_duration")
 class AddT0IdxAndSamplePeriodDurationIterDataPipe(IterDataPipe):
     """Add t0_idx and sample_period_duration attributes to datasets for downstream tasks"""
-    def __init__(self, source_datapipe: IterDataPipe, sample_period_duration: timedelta, history_duration: timedelta):
+
+    def __init__(
+        self,
+        source_datapipe: IterDataPipe,
+        sample_period_duration: timedelta,
+        history_duration: timedelta,
+    ):
         """
         Adds two attributes, t0_idx, and sample_period_duration for downstream datapipes to use
 

ocf_datapipes/transform/xarray/convert_to_nwp_target_times.py

@@ -1,5 +1,5 @@
-from typing import Union
 from datetime import timedelta
+from typing import Union
 
 import pandas as pd
 import xarray as xr
@@ -10,6 +10,7 @@
 @functional_datapipe("convert_to_nwp_target_time")
 class ConvertToNWPTargetTimeIterDataPipe(IterDataPipe):
     """Converts NWP Xarray to use the target time"""
+
     def __init__(
         self,
         source_datapipe: IterDataPipe,

ocf_datapipes/transform/xarray/downsample.py

@@ -5,7 +5,15 @@
 @functional_datapipe("downsample")
 class DownsampleIterDataPipe(IterDataPipe):
     """Downsample Xarray dataset with coarsen"""
-    def __init__(self, source_datapipe: IterDataPipe, y_coarsen: int, x_coarsen: int, x_dim_name: str = "x_osgb", y_dim_name: str = "y_osgb"):
+
+    def __init__(
+        self,
+        source_datapipe: IterDataPipe,
+        y_coarsen: int,
+        x_coarsen: int,
+        x_dim_name: str = "x_osgb",
+        y_dim_name: str = "y_osgb",
+    ):
         """
         Downsample xarray dataset/dataarrays with coarsen
 
@@ -25,8 +33,10 @@ def __init__(self, source_datapipe: IterDataPipe, y_coarsen: int, x_coarsen: int
     def __iter__(self):
         """Coarsen the data on the specified dimensions"""
         for xr_data in self.source_datapipe:
-            yield xr_data.coarsen({
-                self.y_dim_name: self.y_coarsen,
-                self.x_dim_name: self.x_coarsen,},
+            yield xr_data.coarsen(
+                {
+                    self.y_dim_name: self.y_coarsen,
+                    self.x_dim_name: self.x_coarsen,
+                },
                 boundary="trim",
             ).mean()

ocf_datapipes/transform/xarray/normalize.py

@@ -1,19 +1,21 @@
-import xarray as xr
+from typing import Callable, Optional, Union
+
 import numpy as np
-from typing import Optional, Union, Callable
+import xarray as xr
 from torchdata.datapipes import functional_datapipe
 from torchdata.datapipes.iter import IterDataPipe
 
 
 @functional_datapipe("normalize")
 class NormalizeIterDataPipe(IterDataPipe):
     """Normalize the data in various methods"""
+
     def __init__(
         self,
         source_datapipe: IterDataPipe,
-        mean: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]]=None,
-        std: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]]=None,
-        max_value: Optional[Union[int, float]]=None,
+        mean: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]] = None,
+        std: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]] = None,
+        max_value: Optional[Union[int, float]] = None,
         calculate_mean_std_from_example: bool = False,
         normalize_fn: Optional[Callable] = None,
     ):

ocf_datapipes/transform/xarray/pv_power_rolling_window.py

@@ -9,6 +9,7 @@
 @functional_datapipe("pv_power_rolling_window")
 class PVPowerRollingWindowIterDataPipe(IterDataPipe):
     """Compute rolling mean of PV power."""
+
     def __init__(
         self,
         source_datapipe: IterDataPipe,

tests/convert/test_gsp.py

@@ -1,8 +1,7 @@
-from ocf_datapipes.convert import ConvertGSPToNumpyBatch
-from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration
-
 from datetime import timedelta
 
+from ocf_datapipes.convert import ConvertGSPToNumpyBatch
+from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration
 
 
 def test_convert_gsp_to_numpy_batch(gsp_dp):

tests/convert/test_nwp.py

@@ -1,15 +1,15 @@
+from datetime import timedelta
+
 from ocf_datapipes.convert import ConvertNWPToNumpyBatch
-from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration, ConvertToNWPTargetTime
 from ocf_datapipes.select import SelectLiveT0Time
-
-from datetime import timedelta
+from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration, ConvertToNWPTargetTime
 
 
 def test_convert_nwp_to_numpy_batch(nwp_dp):
     nwp_dp = AddT0IdxAndSamplePeriodDuration(
         nwp_dp, sample_period_duration=timedelta(minutes=60), history_duration=timedelta(minutes=60)
     )
-    t0_dp = SelectLiveT0Time(nwp_dp, dim_name='init_time_utc')
+    t0_dp = SelectLiveT0Time(nwp_dp, dim_name="init_time_utc")
 
     nwp_dp = ConvertToNWPTargetTime(
         nwp_dp,

tests/convert/test_pv.py

@@ -1,12 +1,14 @@
+from datetime import timedelta
+
 from ocf_datapipes.convert import ConvertPVToNumpyBatch
 from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration
 
-from datetime import timedelta
-
 
 def test_convert_passiv_to_numpy_batch(passiv_dp):
     passiv_dp = AddT0IdxAndSamplePeriodDuration(
-        passiv_dp, sample_period_duration=timedelta(minutes=5), history_duration=timedelta(minutes=60)
+        passiv_dp,
+        sample_period_duration=timedelta(minutes=5),
+        history_duration=timedelta(minutes=60),
     )
     passiv_dp = ConvertPVToNumpyBatch(passiv_dp)
     data = next(iter(passiv_dp))
@@ -15,7 +17,9 @@ def test_convert_passiv_to_numpy_batch(passiv_dp):
 
 def test_convert_pvoutput_to_numpy_batch(pvoutput_dp):
     pvoutput_dp = AddT0IdxAndSamplePeriodDuration(
-        pvoutput_dp, sample_period_duration=timedelta(minutes=5), history_duration=timedelta(minutes=60)
+        pvoutput_dp,
+        sample_period_duration=timedelta(minutes=5),
+        history_duration=timedelta(minutes=60),
     )
 
     pvoutput_dp = ConvertPVToNumpyBatch(pvoutput_dp)

tests/convert/test_satellite.py

@@ -1,8 +1,8 @@
+from datetime import timedelta
+
 from ocf_datapipes.convert import ConvertSatelliteToNumpyBatch
 from ocf_datapipes.transform.xarray import AddT0IdxAndSamplePeriodDuration
 
-from datetime import timedelta
-
 
 def test_convert_satellite_to_numpy_batch(sat_dp):
 

tests/end2end/test_power_perceiver_production.py

@@ -17,6 +17,7 @@
     ConvertSatelliteToNumpyBatch,
 )
 from ocf_datapipes.experimental import EnsureNNWPVariables, SetSystemIDsToOne
+from ocf_datapipes.production.power_perceiver import GSPIterator
 from ocf_datapipes.select import (
     LocationPicker,
     SelectLiveT0Time,
@@ -43,7 +44,6 @@
     ReprojectTopography,
 )
 from ocf_datapipes.utils.consts import NWP_MEAN, NWP_STD, SAT_MEAN, SAT_STD, BatchKey
-from ocf_datapipes.production.power_perceiver import GSPIterator
 
 
 def test_power_perceiver_production(sat_hrv_dp, passiv_dp, topo_dp, gsp_dp, nwp_dp):

tests/load/test_load_config.py

@@ -1,9 +1,7 @@
-
 from ocf_datapipes.load import OpenConfiguration
 
 
 def test_open_config():
     config_dp = OpenConfiguration("tests/config/test.yaml")
     configuration = next(iter(config_dp))
     print(configuration)
-

tests/production/test_pp_production.py

@@ -1,12 +1,13 @@
-from ocf_datapipes.production.power_perceiver import power_perceiver_production_datapipe
-from ocf_datapipes.utils.consts import BatchKey
-
-import ocf_datapipes
 import os
+
 import pytest
 
+import ocf_datapipes
+from ocf_datapipes.production.power_perceiver import power_perceiver_production_datapipe
+from ocf_datapipes.utils.consts import BatchKey
+
 
-@pytest.mark.skip('Need to set up laod PV from database first')
+@pytest.mark.skip("Need to set up laod PV from database first")
 def test_pp_production_datapipe():
 
     filename = os.path.join(os.path.dirname(ocf_datapipes.__file__), "../tests/config/test.yaml")

tests/select/test_select_live_t0_time_slice.py

@@ -58,7 +58,7 @@ def test_select_passiv(passiv_dp):
 
 def test_select_pvoutput(pvoutput_dp):
     time_len = len(next(iter(pvoutput_dp)).time_utc.values)
-    t0_dp = SelectLiveT0Time(pvoutput_dp, dim_name='time_utc')
+    t0_dp = SelectLiveT0Time(pvoutput_dp, dim_name="time_utc")
     pvoutput_dp = SelectLiveTimeSlice(
         pvoutput_dp,
         t0_datapipe=t0_dp,

tests/select/test_select_live_time_slice.py

@@ -1,13 +1,16 @@
 from datetime import timedelta
 
 from ocf_datapipes.select import SelectLiveT0Time, SelectLiveTimeSlice
+
 # from ocf_datapipes.transform.xarray import AddNWPTargetTime
 
 
 def test_select_hrv(sat_hrv_dp):
     time_len = len(next(iter(sat_hrv_dp)).time_utc.values)
     t0_dp = SelectLiveT0Time(sat_hrv_dp, dim_name="time_utc")
-    sat_hrv_dp = SelectLiveTimeSlice(sat_hrv_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp)
+    sat_hrv_dp = SelectLiveTimeSlice(
+        sat_hrv_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp
+    )
     data = next(iter(sat_hrv_dp))
     assert len(data.time_utc.values) == 13
     assert len(data.time_utc.values) < time_len
@@ -49,7 +52,9 @@ def test_select_gsp(gsp_dp):
 def test_select_passiv(passiv_dp):
     time_len = len(next(iter(passiv_dp)).time_utc.values)
     t0_dp = SelectLiveT0Time(passiv_dp, dim_name="time_utc")
-    passiv_dp = SelectLiveTimeSlice(passiv_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp)
+    passiv_dp = SelectLiveTimeSlice(
+        passiv_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp
+    )
     data = next(iter(passiv_dp))
     assert len(data.time_utc.values) == 13
     assert len(data.time_utc.values) < time_len
@@ -58,7 +63,9 @@ def test_select_passiv(passiv_dp):
 def test_select_pvoutput(pvoutput_dp):
     time_len = len(next(iter(pvoutput_dp)).time_utc.values)
     t0_dp = SelectLiveT0Time(pvoutput_dp, dim_name="time_utc")
-    pvoutput_dp = SelectLiveTimeSlice(pvoutput_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp)
+    pvoutput_dp = SelectLiveTimeSlice(
+        pvoutput_dp, history_duration=timedelta(minutes=60), t0_datapipe=t0_dp
+    )
     data = next(iter(pvoutput_dp))
     assert len(data.time_utc.values) == 13
     assert len(data.time_utc.values) < time_len

tests/transform/xarray/test_add_nwp_target_time.py

@@ -1,11 +1,18 @@
-from ocf_datapipes.transform.xarray import ConvertToNWPTargetTime
-from ocf_datapipes.select import SelectLiveT0Time
 from datetime import timedelta
 
+from ocf_datapipes.select import SelectLiveT0Time
+from ocf_datapipes.transform.xarray import ConvertToNWPTargetTime
+
+
 def test_add_nwp_target_time(nwp_dp):
     t0_dp = SelectLiveT0Time(nwp_dp, dim_name="init_time_utc")
-    nwp_dp = ConvertToNWPTargetTime(nwp_dp, t0_dp, sample_period_duration=timedelta(minutes=60), history_duration=timedelta(hours=2), forecast_duration=timedelta(hours=3))
+    nwp_dp = ConvertToNWPTargetTime(
+        nwp_dp,
+        t0_dp,
+        sample_period_duration=timedelta(minutes=60),
+        history_duration=timedelta(hours=2),
+        forecast_duration=timedelta(hours=3),
+    )
     data = next(iter(nwp_dp))
     assert "target_time_utc" in data.coords
     assert len(data.coords["target_time_utc"]) == 6
-

tests/transform/xarray/test_downsample.py

@@ -10,7 +10,13 @@ def test_nwp_downsample(nwp_dp):
 
 
 def test_sat_downsample(sat_dp):
-    sat_dp = Downsample(sat_dp, y_coarsen=16, x_coarsen=16, y_dim_name="y_geostationary", x_dim_name="x_geostationary")
+    sat_dp = Downsample(
+        sat_dp,
+        y_coarsen=16,
+        x_coarsen=16,
+        y_dim_name="y_geostationary",
+        x_dim_name="x_geostationary",
+    )
     data = next(iter(sat_dp))
     assert data.shape[-1] == 38
     assert data.shape[-2] == 18