Skip to content

Commit 6e17191

Browse files
rahul003rahul003
committed
Merge branch 'smp13' of https://github.com/rahul003/sagemaker-python-sdk into smp13
2 parents 29d7497 + df708ef commit 6e17191

File tree

6 files changed

+99
-18
lines changed

6 files changed

+99
-18
lines changed

‎src/sagemaker/feature_store/feature_group.py

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -512,14 +512,11 @@ def as_hive_ddl(self, database: str = "sagemaker_featurestore", table_name: str
512512
if not table_name:
513513
table_name = self.name
514514

515-
s3_uri = self.describe().get("OfflineStoreConfig").get("S3StorageConfig").get("S3Uri")
516-
offline_store_s3_uri = os.path.join(
517-
s3_uri,
518-
self.sagemaker_session.account_id(),
519-
"sagemaker",
520-
self.sagemaker_session.boto_session.region_name,
521-
"offline-store",
522-
self.name,
515+
resolved_output_s3_uri = (
516+
self.describe()
517+
.get("OfflineStoreConfig")
518+
.get("S3StorageConfig")
519+
.get("ResolvedOutputS3Uri")
523520
)
524521

525522
ddl = f"CREATE EXTERNAL TABLE IF NOT EXISTS {database}.{table_name} (\n"
@@ -537,6 +534,6 @@ def as_hive_ddl(self, database: str = "sagemaker_featurestore", table_name: str
537534
" STORED AS\n"
538535
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
539536
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
540-
f"LOCATION '{offline_store_s3_uri}'"
537+
f"LOCATION '{resolved_output_s3_uri}'"
541538
)
542539
return ddl

‎src/sagemaker/fw_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
)
6161
SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSIONS = {
6262
"tensorflow": ["2.3.0", "2.3.1", "2.4.1"],
63-
"pytorch": ["1.6.0", "1.7.1"],
63+
"pytorch": ["1.6.0", "1.7.1", "1.8.0"],
6464
}
6565
SMDISTRIBUTED_SUPPORTED_STRATEGIES = ["dataparallel", "modelparallel"]
6666

‎src/sagemaker/image_uri_config/pytorch.json

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@
5555
"1.4": "1.4.0",
5656
"1.5": "1.5.0",
5757
"1.6": "1.6.0",
58-
"1.7": "1.7.1"
58+
"1.7": "1.7.1",
59+
"1.8": "1.8.0"
5960
},
6061
"versions": {
6162
"0.4.0": {
@@ -352,6 +353,39 @@
352353
"us-west-2": "763104351884"
353354
},
354355
"repository": "pytorch-inference"
356+
},
357+
"1.8.0": {
358+
"py_versions": [
359+
"py3",
360+
"py36"
361+
],
362+
"registries": {
363+
"af-south-1": "626614931356",
364+
"ap-east-1": "871362719292",
365+
"ap-northeast-1": "763104351884",
366+
"ap-northeast-2": "763104351884",
367+
"ap-south-1": "763104351884",
368+
"ap-southeast-1": "763104351884",
369+
"ap-southeast-2": "763104351884",
370+
"ca-central-1": "763104351884",
371+
"cn-north-1": "727897471807",
372+
"cn-northwest-1": "727897471807",
373+
"eu-central-1": "763104351884",
374+
"eu-north-1": "763104351884",
375+
"eu-west-1": "763104351884",
376+
"eu-west-2": "763104351884",
377+
"eu-west-3": "763104351884",
378+
"eu-south-1": "692866216735",
379+
"me-south-1": "217643126080",
380+
"sa-east-1": "763104351884",
381+
"us-east-1": "763104351884",
382+
"us-east-2": "763104351884",
383+
"us-gov-west-1": "442386744353",
384+
"us-iso-east-1": "886529160074",
385+
"us-west-1": "763104351884",
386+
"us-west-2": "763104351884"
387+
},
388+
"repository": "pytorch-inference"
355389
}
356390
}
357391
},
@@ -369,7 +403,8 @@
369403
"1.4": "1.4.0",
370404
"1.5": "1.5.0",
371405
"1.6": "1.6.0",
372-
"1.7": "1.7.1"
406+
"1.7": "1.7.1",
407+
"1.8": "1.8.0"
373408
},
374409
"versions": {
375410
"0.4.0": {
@@ -667,6 +702,39 @@
667702
"us-west-2": "763104351884"
668703
},
669704
"repository": "pytorch-training"
705+
},
706+
"1.8.0": {
707+
"py_versions": [
708+
"py3",
709+
"py36"
710+
],
711+
"registries": {
712+
"af-south-1": "626614931356",
713+
"ap-east-1": "871362719292",
714+
"ap-northeast-1": "763104351884",
715+
"ap-northeast-2": "763104351884",
716+
"ap-south-1": "763104351884",
717+
"ap-southeast-1": "763104351884",
718+
"ap-southeast-2": "763104351884",
719+
"ca-central-1": "763104351884",
720+
"cn-north-1": "727897471807",
721+
"cn-northwest-1": "727897471807",
722+
"eu-central-1": "763104351884",
723+
"eu-north-1": "763104351884",
724+
"eu-west-1": "763104351884",
725+
"eu-west-2": "763104351884",
726+
"eu-west-3": "763104351884",
727+
"eu-south-1": "692866216735",
728+
"me-south-1": "217643126080",
729+
"sa-east-1": "763104351884",
730+
"us-east-1": "763104351884",
731+
"us-east-2": "763104351884",
732+
"us-gov-west-1": "442386744353",
733+
"us-iso-east-1": "886529160074",
734+
"us-west-1": "763104351884",
735+
"us-west-2": "763104351884"
736+
},
737+
"repository": "pytorch-training"
670738
}
671739
}
672740
}

‎tests/integ/test_feature_store.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,7 @@ def create_table_ddl():
142142
" STORED AS\n"
143143
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
144144
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
145-
"LOCATION 's3://sagemaker-test-featurestore-{region}-{account}"
146-
"/{account}/sagemaker/us-east-2/offline-store/{feature_group_name}'"
145+
"LOCATION '{resolved_output_s3_uri}'"
147146
)
148147

149148

@@ -191,6 +190,12 @@ def test_create_feature_store(
191190
)
192191
_wait_for_feature_group_create(feature_group)
193192

193+
resolved_output_s3_uri = (
194+
feature_group.describe()
195+
.get("OfflineStoreConfig")
196+
.get("S3StorageConfig")
197+
.get("ResolvedOutputS3Uri")
198+
)
194199
# Ingest data
195200
feature_group.put_record(record=record)
196201
ingestion_manager = feature_group.ingest(
@@ -225,6 +230,7 @@ def test_create_feature_store(
225230
feature_group_name=feature_group_name,
226231
region=feature_store_session.boto_session.region_name,
227232
account=feature_store_session.account_id(),
233+
resolved_output_s3_uri=resolved_output_s3_uri,
228234
)
229235
== feature_group.as_hive_ddl()
230236
)

‎tests/unit/sagemaker/feature_store/test_feature_store.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ def create_table_ddl():
6666
" STORED AS\n"
6767
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
6868
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
69-
"LOCATION 's3://some-bucket"
70-
"/{account}/sagemaker/{region}/offline-store/{feature_group_name}'"
69+
"LOCATION 's3://resolved_output_s3_uri'"
7170
)
7271

7372

@@ -217,7 +216,12 @@ def test_as_hive_ddl_with_default_values(
217216
create_table_ddl, feature_group_dummy_definitions, sagemaker_session_mock
218217
):
219218
sagemaker_session_mock.describe_feature_group.return_value = {
220-
"OfflineStoreConfig": {"S3StorageConfig": {"S3Uri": "s3://some-bucket"}}
219+
"OfflineStoreConfig": {
220+
"S3StorageConfig": {
221+
"S3Uri": "s3://some-bucket",
222+
"ResolvedOutputS3Uri": "s3://resolved_output_s3_uri",
223+
}
224+
}
221225
}
222226
sagemaker_session_mock.account_id.return_value = "1234"
223227
sagemaker_session_mock.boto_session.region_name = "us-west-2"
@@ -238,7 +242,12 @@ def test_as_hive_ddl_with_default_values(
238242

239243
def test_as_hive_ddl(create_table_ddl, feature_group_dummy_definitions, sagemaker_session_mock):
240244
sagemaker_session_mock.describe_feature_group.return_value = {
241-
"OfflineStoreConfig": {"S3StorageConfig": {"S3Uri": "s3://some-bucket"}}
245+
"OfflineStoreConfig": {
246+
"S3StorageConfig": {
247+
"S3Uri": "s3://some-bucket",
248+
"ResolvedOutputS3Uri": "s3://resolved_output_s3_uri",
249+
}
250+
}
242251
}
243252
sagemaker_session_mock.account_id.return_value = "1234"
244253
sagemaker_session_mock.boto_session.region_name = "us-west-2"

‎tests/unit/test_fw_utils.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,7 @@ def test_validate_smdataparallel_args_not_raises():
636636
("ml.p3.16xlarge", "tensorflow", "2.4.1", "py3", smdataparallel_enabled),
637637
("ml.p3.16xlarge", "pytorch", "1.6.0", "py3", smdataparallel_enabled),
638638
("ml.p3.16xlarge", "pytorch", "1.7.1", "py3", smdataparallel_enabled),
639+
("ml.p3.16xlarge", "pytorch", "1.8.0", "py3", smdataparallel_enabled),
639640
]
640641
for instance_type, framework_name, framework_version, py_version, distribution in good_args:
641642
fw_utils._validate_smdataparallel_args(

0 commit comments

Comments
 (0)