1
+ """Data Loader CLI STAC_API Ingestion Tool."""
1
2
import json
2
3
import os
4
+
3
5
import click
4
6
import requests
5
7
8
+
6
9
def load_data (data_dir , filename ):
7
10
"""Load json data from a file within the specified data directory."""
8
11
filepath = os .path .join (data_dir , filename )
@@ -12,6 +15,7 @@ def load_data(data_dir, filename):
12
15
with open (filepath ) as file :
13
16
return json .load (file )
14
17
18
+
15
19
def load_collection (base_url , collection_id , data_dir ):
16
20
"""Load a STAC collection into the database."""
17
21
collection = load_data (data_dir , "collection.json" )
@@ -27,14 +31,25 @@ def load_collection(base_url, collection_id, data_dir):
27
31
except requests .ConnectionError :
28
32
click .secho ("Failed to connect" , fg = "red" , err = True )
29
33
34
+
30
35
def load_items (base_url , collection_id , use_bulk , data_dir ):
31
36
"""Load STAC items into the database based on the method selected."""
32
37
# Attempt to dynamically find a suitable feature collection file
33
- feature_files = [file for file in os .listdir (data_dir ) if file .endswith ('.json' ) and file != "collection.json" ]
38
+ feature_files = [
39
+ file
40
+ for file in os .listdir (data_dir )
41
+ if file .endswith (".json" ) and file != "collection.json"
42
+ ]
34
43
if not feature_files :
35
- click .secho ("No feature collection files found in the specified directory." , fg = "red" , err = True )
44
+ click .secho (
45
+ "No feature collection files found in the specified directory." ,
46
+ fg = "red" ,
47
+ err = True ,
48
+ )
36
49
raise click .Abort ()
37
- feature_collection_file = feature_files [0 ] # Use the first found feature collection file
50
+ feature_collection_file = feature_files [
51
+ 0
52
+ ] # Use the first found feature collection file
38
53
feature_collection = load_data (data_dir , feature_collection_file )
39
54
40
55
load_collection (base_url , collection_id , data_dir )
@@ -43,12 +58,15 @@ def load_items(base_url, collection_id, use_bulk, data_dir):
43
58
else :
44
59
load_items_one_by_one (base_url , collection_id , feature_collection , data_dir )
45
60
61
+
46
62
def load_items_one_by_one (base_url , collection_id , feature_collection , data_dir ):
47
63
"""Load STAC items into the database one by one."""
48
64
for feature in feature_collection ["features" ]:
49
65
try :
50
66
feature ["collection" ] = collection_id
51
- resp = requests .post (f"{ base_url } /collections/{ collection_id } /items" , json = feature )
67
+ resp = requests .post (
68
+ f"{ base_url } /collections/{ collection_id } /items" , json = feature
69
+ )
52
70
if resp .status_code == 200 :
53
71
click .echo (f"Status code: { resp .status_code } " )
54
72
click .echo (f"Added item: { feature ['id' ]} " )
@@ -58,12 +76,15 @@ def load_items_one_by_one(base_url, collection_id, feature_collection, data_dir)
58
76
except requests .ConnectionError :
59
77
click .secho ("Failed to connect" , fg = "red" , err = True )
60
78
79
+
61
80
def load_items_bulk_insert (base_url , collection_id , feature_collection , data_dir ):
62
81
"""Load STAC items into the database via bulk insert."""
63
82
try :
64
83
for i , _ in enumerate (feature_collection ["features" ]):
65
84
feature_collection ["features" ][i ]["collection" ] = collection_id
66
- resp = requests .post (f"{ base_url } /collections/{ collection_id } /items" , json = feature_collection ) # Adjust this endpoint as necessary
85
+ resp = requests .post (
86
+ f"{ base_url } /collections/{ collection_id } /items" , json = feature_collection
87
+ ) # Adjust this endpoint as necessary
67
88
if resp .status_code == 200 :
68
89
click .echo (f"Status code: { resp .status_code } " )
69
90
click .echo ("Bulk inserted items successfully." )
@@ -76,14 +97,25 @@ def load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir
76
97
except requests .ConnectionError :
77
98
click .secho ("Failed to connect" , fg = "red" , err = True )
78
99
100
+
79
101
@click .command ()
80
102
@click .option ("--base-url" , required = True , help = "Base URL of the STAC API" )
81
- @click .option ("--collection-id" , default = "test-collection" , help = "ID of the collection to which items are added" )
103
+ @click .option (
104
+ "--collection-id" ,
105
+ default = "test-collection" ,
106
+ help = "ID of the collection to which items are added" ,
107
+ )
82
108
@click .option ("--use-bulk" , is_flag = True , help = "Use bulk insert method for items" )
83
- @click .option ("--data-dir" , type = click .Path (exists = True ), default = "sample_data/" , help = "Directory containing collection.json and feature collection file" )
109
+ @click .option (
110
+ "--data-dir" ,
111
+ type = click .Path (exists = True ),
112
+ default = "sample_data/" ,
113
+ help = "Directory containing collection.json and feature collection file" ,
114
+ )
84
115
def main (base_url , collection_id , use_bulk , data_dir ):
85
116
"""Load STAC items into the database."""
86
117
load_items (base_url , collection_id , use_bulk , data_dir )
87
118
119
+
88
120
if __name__ == "__main__" :
89
- main ()
121
+ main ()
0 commit comments