Merge pull request #362 from openclimatefix/satellite-scalling

Satellite scalling

Author: peterdudfield

ocf_datapipes/config/model.py

@@ -502,6 +502,11 @@ class Satellite(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
         description="The temporal resolution (in minutes) of the data."
         "Note that this needs to be divisible by 5.",
     )
+    satellite_scaling_methods: Optional[List[str]] = Field(
+        ["mean_std"],
+        description="There are few ways to scale the satellite data. "
+        "1. None, 2. mean_std, 3. min_max",
+    )
 
 
 class HRVSatellite(DataSourceMixin, TimeResolutionMixin, DropoutMixin):

ocf_datapipes/training/pvnet_site.py

@@ -24,6 +24,8 @@
     NWP_MEANS,
     NWP_STDS,
     RSS_MEAN,
+    RSS_RAW_MAX,
+    RSS_RAW_MIN,
     RSS_STD,
 )
 from ocf_datapipes.utils.utils import (
@@ -273,7 +275,11 @@ def construct_sliced_data_pipeline(
             roi_height_pixels=conf_sat.satellite_image_size_pixels_height,
             roi_width_pixels=conf_sat.satellite_image_size_pixels_width,
         )
-        sat_datapipe = sat_datapipe.normalize(mean=RSS_MEAN, std=RSS_STD)
+        scaling_methods = conf_sat.satellite_scaling_methods
+        if "min_max" in scaling_methods:
+            sat_datapipe = sat_datapipe.normalize(min_values=RSS_RAW_MIN, max_values=RSS_RAW_MAX)
+        if "mean_std" in scaling_methods:
+            sat_datapipe = sat_datapipe.normalize(mean=RSS_MEAN, std=RSS_STD)
 
     if "pv" in datapipes_dict:
         # Recombine Sensor arrays - see function doc for further explanation

ocf_datapipes/transform/xarray/normalize.py

@@ -22,6 +22,8 @@ def __init__(
         max_value: Optional[Union[int, float]] = None,
         calculate_mean_std_from_example: bool = False,
         normalize_fn: Optional[Callable] = None,
+        min_values: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]] = None,
+        max_values: Optional[Union[xr.Dataset, xr.DataArray, np.ndarray]] = None,
     ):
         """
         Normalize the data with either given mean/std,
@@ -37,13 +39,17 @@ def __init__(
             calculate_mean_std_from_example: Whether to calculate the
                 mean/std from the input data or not
             normalize_fn: Callable function to apply to the data to normalize it
+            min_values: Min values for each channel
+            max_values: Max values for each channel
         """
         self.source_datapipe = source_datapipe
         self.mean = mean
         self.std = std
         self.max_value = max_value
         self.calculate_mean_std_from_example = calculate_mean_std_from_example
         self.normalize_fn = normalize_fn
+        self.min_values = min_values
+        self.max_values = max_values
 
     def __iter__(self) -> Union[xr.Dataset, xr.DataArray]:
         """Normalize the data depending on the init arguments"""
@@ -61,6 +67,8 @@ def __iter__(self) -> Union[xr.Dataset, xr.DataArray]:
                 # For Topo data for example
                 xr_data -= xr_data.mean().item()
                 xr_data /= xr_data.std().item()
+            elif (self.min_values is not None) and (self.max_values is not None):
+                xr_data = (xr_data - self.min_values) / (self.max_values - self.min_values)
             else:
                 try:
                     logger.debug(f"Normalizing by {self.normalize_fn}")

ocf_datapipes/utils/consts.py

@@ -413,6 +413,41 @@ def __getitem__(self, key):
 RSS_STD = _to_data_array(RSS_STD)
 RSS_MEAN = _to_data_array(RSS_MEAN)
 
+# normalizing from raw values
+
+RSS_RAW_MIN = {
+    "IR_016": -2.5118103,
+    "IR_039": -64.83977,
+    "IR_087": 63.404694,
+    "IR_097": 2.844452,
+    "IR_108": 199.10002,
+    "IR_120": -17.254883,
+    "IR_134": -26.29155,
+    "VIS006": -1.1009827,
+    "VIS008": -2.4184198,
+    "WV_062": 199.57048,
+    "WV_073": 198.95093,
+    "HRV": -1.2278595,
+}
+
+RSS_RAW_MAX = {
+    "IR_016": 69.60857,
+    "IR_039": 339.15588,
+    "IR_087": 340.26526,
+    "IR_097": 317.86752,
+    "IR_108": 313.2767,
+    "IR_120": 315.99194,
+    "IR_134": 274.82297,
+    "VIS006": 93.786545,
+    "VIS008": 101.34922,
+    "WV_062": 249.91806,
+    "WV_073": 286.96323,
+    "HRV": 103.90016,
+}
+
+RSS_RAW_MIN = _to_data_array(RSS_RAW_MIN)
+RSS_RAW_MAX = _to_data_array(RSS_RAW_MAX)
+
 
 # --------------------------- SENSORS --------------------------------