This repository was archived by the owner on Jun 2, 2025. It is now read-only.
File tree Expand file tree Collapse file tree 2 files changed +13
-3
lines changed Expand file tree Collapse file tree 2 files changed +13
-3
lines changed Original file line number Diff line number Diff line change 1
1
import fsspec
2
2
from pathy import Pathy
3
+ import logging
3
4
from pyaml_env import parse_config
4
5
from torchdata .datapipes import functional_datapipe
5
6
from torchdata .datapipes .iter import IterDataPipe
6
7
7
8
from ocf_datapipes .config .model import Configuration
8
9
10
+ logger = logging .getLogger (__name__ )
11
+
9
12
10
13
@functional_datapipe ("open_config" )
11
14
class OpenConfigurationIterDataPipe (IterDataPipe ):
12
15
def __init__ (self , configuration_filename : str ):
13
16
self .configuration_filename = configuration_filename
14
17
15
18
def __iter__ (self ):
19
+ logger .debug (f'Going to open { self .configuration_filename } ' )
16
20
with fsspec .open (self .configuration_filename , mode = "r" ) as stream :
17
21
configuration = parse_config (data = stream )
18
22
23
+ logger .debug (f'Converting to Configuration ({ configuration } )' )
19
24
configuration = Configuration (** configuration )
20
25
21
26
while True :
Original file line number Diff line number Diff line change 1
1
from ocf_datapipes .production .power_perceiver import power_perceiver_production_datapipe
2
2
from ocf_datapipes .utils .consts import BatchKey
3
3
4
+ import ocf_datapipes
5
+ import os
6
+
4
7
5
8
def test_pp_production_datapipe ():
6
- pp_dp = power_perceiver_production_datapipe (
7
- "/home/jacob/Development/ocf_datapipes/tests/config/test.yaml"
8
- )
9
+
10
+ filename = os .path .join (os .path .dirname (ocf_datapipes .__file__ ), "../tests/config/test.yaml" )
11
+
12
+ pp_dp = power_perceiver_production_datapipe (filename )
13
+
9
14
batch = next (iter (pp_dp ))
10
15
11
16
assert len (batch [BatchKey .hrvsatellite_time_utc ]) == 4
You can’t perform that action at this time.
0 commit comments