Skip to content

change: use ResolvedOutputS3Uir for Hive DDL LOCATION #2213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 16, 2021
Merged

change: use ResolvedOutputS3Uir for Hive DDL LOCATION #2213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e57e5aa
Update feature_group.py
can-sun Mar 15, 2021
9213962
Update test_feature_store.py
can-sun Mar 15, 2021
d2012d9
Update feature_group.py
can-sun Mar 16, 2021
6e10bde
Merge branch 'master' into can-sun-patch-1
Mar 16, 2021
26e22b0
Merge branch 'master' into can-sun-patch-1
icywang86rui Mar 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions src/sagemaker/feature_store/feature_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,14 +512,11 @@ def as_hive_ddl(self, database: str = "sagemaker_featurestore", table_name: str
if not table_name:
table_name = self.name

s3_uri = self.describe().get("OfflineStoreConfig").get("S3StorageConfig").get("S3Uri")
offline_store_s3_uri = os.path.join(
s3_uri,
self.sagemaker_session.account_id(),
"sagemaker",
self.sagemaker_session.boto_session.region_name,
"offline-store",
self.name,
resolved_output_s3_uri = (
self.describe()
.get("OfflineStoreConfig")
.get("S3StorageConfig")
.get("ResolvedOutputS3Uri")
)

ddl = f"CREATE EXTERNAL TABLE IF NOT EXISTS {database}.{table_name} (\n"
Expand All @@ -537,6 +534,6 @@ def as_hive_ddl(self, database: str = "sagemaker_featurestore", table_name: str
" STORED AS\n"
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
f"LOCATION '{offline_store_s3_uri}'"
f"LOCATION '{resolved_output_s3_uri}'"
)
return ddl
10 changes: 8 additions & 2 deletions tests/integ/test_feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,7 @@ def create_table_ddl():
" STORED AS\n"
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
"LOCATION 's3://sagemaker-test-featurestore-{region}-{account}"
"/{account}/sagemaker/us-east-2/offline-store/{feature_group_name}'"
"LOCATION '{resolved_output_s3_uri}'"
)


Expand Down Expand Up @@ -191,6 +190,12 @@ def test_create_feature_store(
)
_wait_for_feature_group_create(feature_group)

resolved_output_s3_uri = (
feature_group.describe()
.get("OfflineStoreConfig")
.get("S3StorageConfig")
.get("ResolvedOutputS3Uri")
)
# Ingest data
feature_group.put_record(record=record)
ingestion_manager = feature_group.ingest(
Expand Down Expand Up @@ -225,6 +230,7 @@ def test_create_feature_store(
feature_group_name=feature_group_name,
region=feature_store_session.boto_session.region_name,
account=feature_store_session.account_id(),
resolved_output_s3_uri=resolved_output_s3_uri,
)
== feature_group.as_hive_ddl()
)
Expand Down
17 changes: 13 additions & 4 deletions tests/unit/sagemaker/feature_store/test_feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ def create_table_ddl():
" STORED AS\n"
" INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"
" OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n"
"LOCATION 's3://some-bucket"
"/{account}/sagemaker/{region}/offline-store/{feature_group_name}'"
"LOCATION 's3://resolved_output_s3_uri'"
)


Expand Down Expand Up @@ -217,7 +216,12 @@ def test_as_hive_ddl_with_default_values(
create_table_ddl, feature_group_dummy_definitions, sagemaker_session_mock
):
sagemaker_session_mock.describe_feature_group.return_value = {
"OfflineStoreConfig": {"S3StorageConfig": {"S3Uri": "s3://some-bucket"}}
"OfflineStoreConfig": {
"S3StorageConfig": {
"S3Uri": "s3://some-bucket",
"ResolvedOutputS3Uri": "s3://resolved_output_s3_uri",
}
}
}
sagemaker_session_mock.account_id.return_value = "1234"
sagemaker_session_mock.boto_session.region_name = "us-west-2"
Expand All @@ -238,7 +242,12 @@ def test_as_hive_ddl_with_default_values(

def test_as_hive_ddl(create_table_ddl, feature_group_dummy_definitions, sagemaker_session_mock):
sagemaker_session_mock.describe_feature_group.return_value = {
"OfflineStoreConfig": {"S3StorageConfig": {"S3Uri": "s3://some-bucket"}}
"OfflineStoreConfig": {
"S3StorageConfig": {
"S3Uri": "s3://some-bucket",
"ResolvedOutputS3Uri": "s3://resolved_output_s3_uri",
}
}
}
sagemaker_session_mock.account_id.return_value = "1234"
sagemaker_session_mock.boto_session.region_name = "us-west-2"
Expand Down