Skip to content

Commit 659c90b

Browse files
mufaddal-rohawalamufaddal-rohawala
authored
Merge branch 'master-jumpstart' into feat/client-cache-for-jumpstart-models
2 parents 8f8b1aa + 2beb91e commit 659c90b

File tree

18 files changed

+755
-168
lines changed

18 files changed

+755
-168
lines changed

‎CHANGELOG.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,40 @@
11
# Changelog
22

3+
## v2.71.0 (2021-12-06)
4+
5+
### Features
6+
7+
* Add support for TF 2.6
8+
* Adding PT 17/18 Repo
9+
* Add profile_name support for Feature Store ingestion
10+
11+
### Bug Fixes and Other Changes
12+
13+
* Fix non-existent variable name
14+
* Add TF 2.6.2 on training
15+
* Recreate static lineage test data
16+
17+
## v2.70.0 (2021-12-02)
18+
19+
### Features
20+
21+
* update boto3 minor version >= 1.20.18
22+
* Add support for SageMaker lineage queries
23+
* add CV shap explainability for SageMaker Clarify
24+
* add NLP support for SageMaker Clarify
25+
* Add support for ModelMonitor/Clarify integration in model building pipelines
26+
* adding support for transformers 4.11 for SM Training Compiler
27+
* SM Training Compiler with an UI to enable/disable compilation for HuggingFace DLCs to speedup training
28+
29+
### Bug Fixes and Other Changes
30+
31+
* pin coveragepy
32+
* Add support for PyTorch 1.9.1
33+
* Update s3 path of scheduling analysis config on ClarifyCheckStep
34+
* documentation/logging to indicate correct place for DEBUG artifacts from SM trcomp
35+
* validate requested transformers version and use the best available version
36+
* Install custom pkgs
37+
338
## v2.69.0 (2021-11-12)
439

540
### Features

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.69.1.dev0
1+
2.71.1.dev0

‎src/sagemaker/feature_store/feature_group.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,15 @@ class IngestionManagerPandas:
163163
max_workers (int): number of threads to create.
164164
max_processes (int): number of processes to create. Each process spawns
165165
``max_workers`` threads.
166+
profile_name (str): the profile credential should be used for ``PutRecord``
167+
(default: None).
166168
"""
167169

168170
feature_group_name: str = attr.ib()
169171
sagemaker_fs_runtime_client_config: Config = attr.ib()
170172
max_workers: int = attr.ib(default=1)
171173
max_processes: int = attr.ib(default=1)
174+
profile_name: str = attr.ib(default=None)
172175
_async_result: AsyncResult = attr.ib(default=None)
173176
_processing_pool: ProcessingPool = attr.ib(default=None)
174177
_failed_indices: List[int] = attr.ib(factory=list)
@@ -180,6 +183,7 @@ def _ingest_single_batch(
180183
client_config: Config,
181184
start_index: int,
182185
end_index: int,
186+
profile_name: str = None,
183187
) -> List[int]:
184188
"""Ingest a single batch of DataFrame rows into FeatureStore.
185189
@@ -190,6 +194,8 @@ def _ingest_single_batch(
190194
client to perform boto calls.
191195
start_index (int): starting position to ingest in this batch.
192196
end_index (int): ending position to ingest in this batch.
197+
profile_name (str): the profile credential should be used for ``PutRecord``
198+
(default: None).
193199
194200
Returns:
195201
List of row indices that failed to be ingested.
@@ -198,7 +204,7 @@ def _ingest_single_batch(
198204
if "max_attempts" not in retry_config and "total_max_attempts" not in retry_config:
199205
client_config = copy.deepcopy(client_config)
200206
client_config.retries = {"max_attempts": 10, "mode": "standard"}
201-
sagemaker_featurestore_runtime_client = boto3.Session().client(
207+
sagemaker_featurestore_runtime_client = boto3.Session(profile_name=profile_name).client(
202208
service_name="sagemaker-featurestore-runtime", config=client_config
203209
)
204210

@@ -287,6 +293,7 @@ def _run_multi_process(self, data_frame: DataFrame, wait=True, timeout=None):
287293
data_frame[start_index:end_index],
288294
start_index,
289295
timeout,
296+
self.profile_name,
290297
)
291298
]
292299

@@ -311,6 +318,7 @@ def _run_multi_threaded(
311318
data_frame: DataFrame,
312319
row_offset=0,
313320
timeout=None,
321+
profile_name=None,
314322
) -> List[int]:
315323
"""Start the ingestion process.
316324
@@ -321,6 +329,8 @@ def _run_multi_threaded(
321329
wait (bool): whether to wait for the ingestion to finish or not.
322330
timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
323331
if timeout is reached.
332+
profile_name (str): the profile credential should be used for ``PutRecord``
333+
(default: None).
324334
325335
Returns:
326336
List of row indices that failed to be ingested.
@@ -342,6 +352,7 @@ def _run_multi_threaded(
342352
start_index=start_index,
343353
end_index=end_index,
344354
client_config=sagemaker_fs_runtime_client_config,
355+
profile_name=profile_name,
345356
)
346357
] = (start_index + row_offset, end_index + row_offset)
347358

@@ -581,6 +592,7 @@ def ingest(
581592
max_processes: int = 1,
582593
wait: bool = True,
583594
timeout: Union[int, float] = None,
595+
profile_name: str = None,
584596
) -> IngestionManagerPandas:
585597
"""Ingest the content of a pandas DataFrame to feature store.
586598
@@ -599,6 +611,11 @@ def ingest(
599611
They can also be found from the IngestionManagerPandas' ``failed_rows`` function after
600612
the exception is thrown.
601613
614+
`profile_name` argument is an optional one. It will use the default credential if None is
615+
passed. This `profile_name` is used in the sagemaker_featurestore_runtime client only. See
616+
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html for more
617+
about the default credential.
618+
602619
Args:
603620
data_frame (DataFrame): data_frame to be ingested to feature store.
604621
max_workers (int): number of threads to be created.
@@ -607,6 +624,8 @@ def ingest(
607624
wait (bool): whether to wait for the ingestion to finish or not.
608625
timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
609626
if timeout is reached.
627+
profile_name (str): the profile credential should be used for ``PutRecord``
628+
(default: None).
610629
611630
Returns:
612631
An instance of IngestionManagerPandas.
@@ -622,6 +641,7 @@ def ingest(
622641
sagemaker_fs_runtime_client_config=self.sagemaker_session.sagemaker_featurestore_runtime_client.meta.config,
623642
max_workers=max_workers,
624643
max_processes=max_processes,
644+
profile_name=profile_name,
625645
)
626646

627647
manager.run(data_frame=data_frame, wait=wait, timeout=timeout)

‎src/sagemaker/fw_utils.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,20 @@
5959
"local_gpu",
6060
)
6161
SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSIONS = {
62-
"tensorflow": ["2.3", "2.3.1", "2.3.2", "2.4", "2.4.1", "2.4.3", "2.5", "2.5.0", "2.5.1"],
62+
"tensorflow": [
63+
"2.3",
64+
"2.3.1",
65+
"2.3.2",
66+
"2.4",
67+
"2.4.1",
68+
"2.4.3",
69+
"2.5",
70+
"2.5.0",
71+
"2.5.1",
72+
"2.6",
73+
"2.6.0",
74+
"2.6.2",
75+
],
6376
"pytorch": ["1.6", "1.6.0", "1.7", "1.7.1", "1.8", "1.8.0", "1.8.1", "1.9", "1.9.0", "1.9.1"],
6477
}
6578
SMDISTRIBUTED_SUPPORTED_STRATEGIES = ["dataparallel", "modelparallel"]

‎src/sagemaker/image_uri_config/neo-pytorch.json

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
"1.1.0": "1.4",
88
"1.2.0": "1.4",
99
"1.3.0": "1.4",
10-
"1.4.0": "1.4"
10+
"1.4.0": "1.4",
11+
"1.7.0": "1.7",
12+
"1.7.1": "1.7",
13+
"1.8.0": "1.8",
14+
"1.8.1": "1.8"
1115
},
1216
"versions": {
1317
"1.4": {
@@ -99,6 +103,66 @@
99103
"us-west-2": "301217895009"
100104
},
101105
"repository": "sagemaker-inference-pytorch"
106+
},
107+
"1.7": {
108+
"py_versions": ["py3"],
109+
"registries": {
110+
"af-south-1": "774647643957",
111+
"ap-east-1": "110948597952",
112+
"ap-northeast-1": "941853720454",
113+
"ap-northeast-2": "151534178276",
114+
"ap-northeast-3": "925152966179",
115+
"ap-south-1": "763008648453",
116+
"ap-southeast-1": "324986816169",
117+
"ap-southeast-2": "355873309152",
118+
"ca-central-1": "464438896020",
119+
"cn-north-1": "472730292857",
120+
"cn-northwest-1": "474822919863",
121+
"eu-central-1": "746233611703",
122+
"eu-north-1": "601324751636",
123+
"eu-south-1": "966458181534",
124+
"eu-west-1": "802834080501",
125+
"eu-west-2": "205493899709",
126+
"eu-west-3": "254080097072",
127+
"me-south-1": "836785723513",
128+
"sa-east-1": "756306329178",
129+
"us-east-1": "785573368785",
130+
"us-east-2": "007439368137",
131+
"us-gov-west-1": "263933020539",
132+
"us-west-1": "710691900526",
133+
"us-west-2": "301217895009"
134+
},
135+
"repository": "sagemaker-inference-pytorch"
136+
},
137+
"1.8": {
138+
"py_versions": ["py3"],
139+
"registries": {
140+
"af-south-1": "774647643957",
141+
"ap-east-1": "110948597952",
142+
"ap-northeast-1": "941853720454",
143+
"ap-northeast-2": "151534178276",
144+
"ap-northeast-3": "925152966179",
145+
"ap-south-1": "763008648453",
146+
"ap-southeast-1": "324986816169",
147+
"ap-southeast-2": "355873309152",
148+
"ca-central-1": "464438896020",
149+
"cn-north-1": "472730292857",
150+
"cn-northwest-1": "474822919863",
151+
"eu-central-1": "746233611703",
152+
"eu-north-1": "601324751636",
153+
"eu-south-1": "966458181534",
154+
"eu-west-1": "802834080501",
155+
"eu-west-2": "205493899709",
156+
"eu-west-3": "254080097072",
157+
"me-south-1": "836785723513",
158+
"sa-east-1": "756306329178",
159+
"us-east-1": "785573368785",
160+
"us-east-2": "007439368137",
161+
"us-gov-west-1": "263933020539",
162+
"us-west-1": "710691900526",
163+
"us-west-2": "301217895009"
164+
},
165+
"repository": "sagemaker-inference-pytorch"
102166
}
103167
}
104168
}

‎src/sagemaker/image_uri_config/tensorflow.json

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,8 @@
278278
"2.2": "2.2.2",
279279
"2.3": "2.3.2",
280280
"2.4": "2.4.3",
281-
"2.5": "2.5.1"
281+
"2.5": "2.5.1",
282+
"2.6": "2.6.0"
282283
},
283284
"versions": {
284285
"1.10.0": {
@@ -1312,6 +1313,36 @@
13121313
"us-west-2": "763104351884"
13131314
},
13141315
"repository": "tensorflow-inference"
1316+
},
1317+
"2.6.0": {
1318+
"registries": {
1319+
"af-south-1": "626614931356",
1320+
"ap-east-1": "871362719292",
1321+
"ap-northeast-1": "763104351884",
1322+
"ap-northeast-2": "763104351884",
1323+
"ap-northeast-3": "364406365360",
1324+
"ap-south-1": "763104351884",
1325+
"ap-southeast-1": "763104351884",
1326+
"ap-southeast-2": "763104351884",
1327+
"ca-central-1": "763104351884",
1328+
"cn-north-1": "727897471807",
1329+
"cn-northwest-1": "727897471807",
1330+
"eu-central-1": "763104351884",
1331+
"eu-north-1": "763104351884",
1332+
"eu-south-1": "692866216735",
1333+
"eu-west-1": "763104351884",
1334+
"eu-west-2": "763104351884",
1335+
"eu-west-3": "763104351884",
1336+
"me-south-1": "217643126080",
1337+
"sa-east-1": "763104351884",
1338+
"us-east-1": "763104351884",
1339+
"us-east-2": "763104351884",
1340+
"us-gov-west-1": "442386744353",
1341+
"us-iso-east-1": "886529160074",
1342+
"us-west-1": "763104351884",
1343+
"us-west-2": "763104351884"
1344+
},
1345+
"repository": "tensorflow-inference"
13151346
}
13161347
}
13171348
},
@@ -1338,7 +1369,8 @@
13381369
"2.2": "2.2.2",
13391370
"2.3": "2.3.2",
13401371
"2.4": "2.4.3",
1341-
"2.5": "2.5.1"
1372+
"2.5": "2.5.1",
1373+
"2.6": "2.6.2"
13421374
},
13431375
"versions": {
13441376
"1.10.0": {
@@ -2531,6 +2563,72 @@
25312563
"us-west-2": "763104351884"
25322564
},
25332565
"repository": "tensorflow-training"
2566+
},
2567+
"2.6.0": {
2568+
"py_versions": [
2569+
"py38"
2570+
],
2571+
"registries": {
2572+
"af-south-1": "626614931356",
2573+
"ap-east-1": "871362719292",
2574+
"ap-northeast-1": "763104351884",
2575+
"ap-northeast-2": "763104351884",
2576+
"ap-northeast-3": "364406365360",
2577+
"ap-south-1": "763104351884",
2578+
"ap-southeast-1": "763104351884",
2579+
"ap-southeast-2": "763104351884",
2580+
"ca-central-1": "763104351884",
2581+
"cn-north-1": "727897471807",
2582+
"cn-northwest-1": "727897471807",
2583+
"eu-central-1": "763104351884",
2584+
"eu-north-1": "763104351884",
2585+
"eu-south-1": "692866216735",
2586+
"eu-west-1": "763104351884",
2587+
"eu-west-2": "763104351884",
2588+
"eu-west-3": "763104351884",
2589+
"me-south-1": "217643126080",
2590+
"sa-east-1": "763104351884",
2591+
"us-east-1": "763104351884",
2592+
"us-east-2": "763104351884",
2593+
"us-gov-west-1": "442386744353",
2594+
"us-iso-east-1": "886529160074",
2595+
"us-west-1": "763104351884",
2596+
"us-west-2": "763104351884"
2597+
},
2598+
"repository": "tensorflow-training"
2599+
},
2600+
"2.6.2": {
2601+
"py_versions": [
2602+
"py38"
2603+
],
2604+
"registries": {
2605+
"af-south-1": "626614931356",
2606+
"ap-east-1": "871362719292",
2607+
"ap-northeast-1": "763104351884",
2608+
"ap-northeast-2": "763104351884",
2609+
"ap-northeast-3": "364406365360",
2610+
"ap-south-1": "763104351884",
2611+
"ap-southeast-1": "763104351884",
2612+
"ap-southeast-2": "763104351884",
2613+
"ca-central-1": "763104351884",
2614+
"cn-north-1": "727897471807",
2615+
"cn-northwest-1": "727897471807",
2616+
"eu-central-1": "763104351884",
2617+
"eu-north-1": "763104351884",
2618+
"eu-south-1": "692866216735",
2619+
"eu-west-1": "763104351884",
2620+
"eu-west-2": "763104351884",
2621+
"eu-west-3": "763104351884",
2622+
"me-south-1": "217643126080",
2623+
"sa-east-1": "763104351884",
2624+
"us-east-1": "763104351884",
2625+
"us-east-2": "763104351884",
2626+
"us-gov-west-1": "442386744353",
2627+
"us-iso-east-1": "886529160074",
2628+
"us-west-1": "763104351884",
2629+
"us-west-2": "763104351884"
2630+
},
2631+
"repository": "tensorflow-training"
25342632
}
25352633
}
25362634
}

0 commit comments

Comments
 (0)