Skip to content

Commit 0a61992

Browse files
evakravievakravi
committed
Merge remote-tracking branch 'origin' into feat/jumpstart-default-payloads
2 parents 214e458 + 3d0f660 commit 0a61992

File tree

18 files changed

+255
-31
lines changed

18 files changed

+255
-31
lines changed

‎CHANGELOG.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
# Changelog
22

3+
## v2.189.0 (2023-10-03)
4+
5+
### Features
6+
7+
* add feature processor APIs to public doc
8+
* s3 prefix model data for JumpStartModel
9+
* Model Package support for updating approval
10+
11+
### Bug Fixes and Other Changes
12+
13+
* Add bucket owner check
14+
* transform step unit test
15+
* Release TGI 1.1.0 Image
16+
317
## v2.188.0 (2023-09-26)
418

519
### Features

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.188.1.dev0
1+
2.189.1.dev0

‎doc/api/inference/deserializers.rst

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
Deserializers
22
-------------
33

4-
.. automodule:: sagemaker.deserializers
4+
.. automodule:: sagemaker.base_deserializers
55
:members:
66
:undoc-members:
77
:show-inheritance:
88

9+
.. automodule:: sagemaker.deserializers
10+
:members:
11+
:undoc-members:
12+
:show-inheritance:

‎doc/api/inference/serializers.rst

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
Serializers
22
-----------
33

4-
.. automodule:: sagemaker.serializers
4+
.. automodule:: sagemaker.base_serializers
55
:members:
66
:undoc-members:
77
:show-inheritance:
88

9+
.. automodule:: sagemaker.serializers
10+
:members:
11+
:undoc-members:
12+
:show-inheritance:

‎doc/api/prep_data/feature_store.rst

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,3 +120,41 @@ Feature Store
120120
.. autoclass:: sagemaker.feature_store.feature_store.FeatureStore
121121
:members:
122122
:show-inheritance:
123+
124+
125+
@feature_processor Decorator
126+
****************************
127+
128+
.. autodecorator:: sagemaker.feature_store.feature_processor.feature_processor
129+
130+
131+
Feature Processor Data Source
132+
*****************************
133+
134+
.. autoclass:: sagemaker.feature_store.feature_processor.FeatureGroupDataSource
135+
:members:
136+
:show-inheritance:
137+
138+
.. autoclass:: sagemaker.feature_store.feature_processor.CSVDataSource
139+
:members:
140+
:show-inheritance:
141+
142+
.. autoclass:: sagemaker.feature_store.feature_processor.ParquetDataSource
143+
:members:
144+
:show-inheritance:
145+
146+
147+
Feature Processor Scheduler
148+
***************************
149+
150+
.. automethod:: sagemaker.feature_store.feature_processor.to_pipeline
151+
152+
.. automethod:: sagemaker.feature_store.feature_processor.schedule
153+
154+
.. automethod:: sagemaker.feature_store.feature_processor.execute
155+
156+
.. automethod:: sagemaker.feature_store.feature_processor.delete_schedule
157+
158+
.. automethod:: sagemaker.feature_store.feature_processor.describe
159+
160+
.. automethod:: sagemaker.feature_store.feature_processor.list_pipelines

‎doc/conf.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@
109109
# autosectionlabel
110110
autosectionlabel_prefix_document = True
111111

112+
autodoc_mock_imports = ["pyspark", "feature_store_pyspark", "py4j"]
113+
112114

113115
def setup(app):
114116
sys.stdout.write("Generating JumpStart model table...")

‎doc/frameworks/huggingface/sagemaker.huggingface.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,11 @@ Hugging Face Predictor
3232
:members:
3333
:undoc-members:
3434
:show-inheritance:
35+
36+
Hugging Face Processor
37+
----------------------
38+
39+
.. autoclass:: sagemaker.huggingface.processing.HuggingFaceProcessor
40+
:members:
41+
:undoc-members:
42+
:show-inheritance:

‎doc/frameworks/mxnet/sagemaker.mxnet.rst

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,33 @@ MXNet Classes
33

44

55
MXNet Estimator
6-
---------------------------
6+
---------------
77

88
.. autoclass:: sagemaker.mxnet.estimator.MXNet
99
:members:
1010
:undoc-members:
1111
:show-inheritance:
1212

1313
MXNet Model
14-
---------------------------
14+
-----------
1515

1616
.. autoclass:: sagemaker.mxnet.model.MXNetModel
1717
:members:
1818
:undoc-members:
1919
:show-inheritance:
2020

2121
MXNet Predictor
22-
---------------------------
22+
---------------
2323

2424
.. autoclass:: sagemaker.mxnet.model.MXNetPredictor
2525
:members:
2626
:undoc-members:
2727
:show-inheritance:
28+
29+
MXNet Processor
30+
---------------
31+
32+
.. autoclass:: sagemaker.mxnet.processing.MXNetProcessor
33+
:members:
34+
:undoc-members:
35+
:show-inheritance:

‎doc/frameworks/pytorch/sagemaker.pytorch.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,11 @@ PyTorch Predictor
2424
:members:
2525
:undoc-members:
2626
:show-inheritance:
27+
28+
PyTorch Processor
29+
-----------------
30+
31+
.. autoclass:: sagemaker.pytorch.processing.PyTorchProcessor
32+
:members:
33+
:undoc-members:
34+
:show-inheritance:

‎doc/frameworks/tensorflow/sagemaker.tensorflow.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,12 @@ TensorFlow Serving Predictor
3333
:members:
3434
:undoc-members:
3535
:show-inheritance:
36+
37+
38+
TensorFlow Processor
39+
--------------------
40+
41+
.. autoclass:: sagemaker.tensorflow.processing.TensorFlowProcessor
42+
:members:
43+
:undoc-members:
44+
:show-inheritance:

‎doc/workflows/airflow/using_workflow.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@ the configuration dictionary in the SageMaker Python SDK. Currently, the followi
2222
* ``SageMakerTransformOperator``
2323
* ``SageMakerEndpointConfigOperator``
2424
* ``SageMakerEndpointOperator``
25+
* ``SageMakerProcessingOperator``
26+
* ``SageMakerDeleteModelOperator``
27+
* ``SageMakerStartPipelineOperator``
28+
* ``SageMakerStopPipelineOperator``
29+
* ``SageMakerRegisterModelVersionOperator``
30+
* ``SageMakerAutoMLOperator``
31+
* ``SageMakerCreateExperimentOperator``
32+
33+
You can look into `the Apache Airflow documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/operators/sagemaker.html>`_ for the full
34+
list of operators with examples.
2535

2636
2. PythonOperator: Airflow built-in operator that executes Python callables. You can use the PythonOperator to execute
2737
operations in the SageMaker Python SDK to create a SageMaker workflow.

‎src/sagemaker/feature_store/feature_processor/feature_processor.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,10 @@ def feature_processor(
4949
names nand types in [params: Dict[str, Any], spark: SparkSession]. Outputs: a single return
5050
value of type pyspark.sql.DataFrame. The function can have any name.
5151
52-
Example:
52+
**Example:**
53+
54+
.. code-block:: python
55+
5356
@feature_processor(
5457
inputs=[FeatureGroupDataSource("input-fg"), CSVDataSource("s3://bucket/prefix)],
5558
output='arn:aws:sagemaker:us-west-2:123456789012:feature-group/output-fg'
@@ -60,7 +63,10 @@ def transform(
6063
) -> DataFrame:
6164
return ...
6265
63-
More concisely:
66+
**More concisely:**
67+
68+
.. code-block:: python
69+
6470
@feature_processor(
6571
inputs=[FeatureGroupDataSource("input-fg"), CSVDataSource("s3://bucket/prefix)],
6672
output='arn:aws:sagemaker:us-west-2:123456789012:feature-group/output-fg'

‎src/sagemaker/image_uri_config/huggingface-llm.json

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
"0.6": "0.6.0",
66
"0.8": "0.8.2",
77
"0.9": "0.9.3",
8-
"1.0": "1.0.3"
8+
"1.0": "1.0.3",
9+
"1.1": "1.1.0"
910
},
1011
"versions": {
1112
"0.6.0": {
@@ -175,6 +176,48 @@
175176
"tag_prefix": "2.0.1-tgi1.0.3",
176177
"repository": "huggingface-pytorch-tgi-inference",
177178
"container_version": {"gpu": "cu118-ubuntu20.04"}
179+
},
180+
"1.1.0": {
181+
"py_versions": ["py39"],
182+
"registries": {
183+
"af-south-1": "626614931356",
184+
"il-central-1": "780543022126",
185+
"ap-east-1": "871362719292",
186+
"ap-northeast-1": "763104351884",
187+
"ap-northeast-2": "763104351884",
188+
"ap-northeast-3": "364406365360",
189+
"ap-south-1": "763104351884",
190+
"ap-south-2": "772153158452",
191+
"ap-southeast-1": "763104351884",
192+
"ap-southeast-2": "763104351884",
193+
"ap-southeast-3": "907027046896",
194+
"ap-southeast-4": "457447274322",
195+
"ca-central-1": "763104351884",
196+
"cn-north-1": "727897471807",
197+
"cn-northwest-1": "727897471807",
198+
"eu-central-1": "763104351884",
199+
"eu-central-2": "380420809688",
200+
"eu-north-1": "763104351884",
201+
"eu-west-1": "763104351884",
202+
"eu-west-2": "763104351884",
203+
"eu-west-3": "763104351884",
204+
"eu-south-1": "692866216735",
205+
"eu-south-2": "503227376785",
206+
"me-south-1": "217643126080",
207+
"me-central-1": "914824155844",
208+
"sa-east-1": "763104351884",
209+
"us-east-1": "763104351884",
210+
"us-east-2": "763104351884",
211+
"us-gov-east-1": "446045086412",
212+
"us-gov-west-1": "442386744353",
213+
"us-iso-east-1": "886529160074",
214+
"us-isob-east-1": "094389454867",
215+
"us-west-1": "763104351884",
216+
"us-west-2": "763104351884"
217+
},
218+
"tag_prefix": "2.0.1-tgi1.1.0",
219+
"repository": "huggingface-pytorch-tgi-inference",
220+
"container_version": {"gpu": "cu118-ubuntu20.04"}
178221
}
179222
}
180223
}

‎src/sagemaker/session.py

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ def __init__(
233233
self._default_bucket_name_override = default_bucket
234234
# this may also be set again inside :func:`_initialize` if it is None
235235
self.default_bucket_prefix = default_bucket_prefix
236+
self._default_bucket_set_by_sdk = False
236237

237238
self.s3_resource = None
238239
self.s3_client = None
@@ -545,8 +546,12 @@ def default_bucket(self):
545546
default_bucket = self._default_bucket_name_override
546547
if not default_bucket:
547548
default_bucket = generate_default_sagemaker_bucket_name(self.boto_session)
549+
self._default_bucket_set_by_sdk = True
548550

549-
self._create_s3_bucket_if_it_does_not_exist(bucket_name=default_bucket, region=region)
551+
self._create_s3_bucket_if_it_does_not_exist(
552+
bucket_name=default_bucket,
553+
region=region,
554+
)
550555

551556
self._default_bucket = default_bucket
552557

@@ -620,6 +625,28 @@ def _create_s3_bucket_if_it_does_not_exist(self, bucket_name, region):
620625
else:
621626
raise
622627

628+
if self._default_bucket_set_by_sdk:
629+
# make sure the s3 bucket is configured in users account.
630+
expected_bucket_owner_id = self.account_id()
631+
try:
632+
s3.meta.client.head_bucket(
633+
Bucket=bucket_name, ExpectedBucketOwner=expected_bucket_owner_id
634+
)
635+
except ClientError as e:
636+
error_code = e.response["Error"]["Code"]
637+
message = e.response["Error"]["Message"]
638+
if error_code == "403" and message == "Forbidden":
639+
LOGGER.error(
640+
"Since default_bucket param was not set, SageMaker Python SDK tried to use "
641+
"%s bucket. "
642+
"This bucket cannot be configured to use as it is not owned by Account %s. "
643+
"To unblock it's recommended to use custom default_bucket "
644+
"parameter in sagemaker.Session",
645+
bucket_name,
646+
expected_bucket_owner_id,
647+
)
648+
raise
649+
623650
def _append_sagemaker_config_tags(self, tags: list, config_path_to_tags: str):
624651
"""Appends tags specified in the sagemaker_config to the given list of tags.
625652

‎tests/integ/sagemaker/jumpstart/retrieve_uri/test_transfer_learning.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from sagemaker import script_uris
2929
from sagemaker import model_uris
3030
from sagemaker import hyperparameters
31+
import pytest
3132

3233

3334
def test_jumpstart_transfer_learning_retrieve_functions(setup):
@@ -71,7 +72,12 @@ def test_jumpstart_transfer_learning_retrieve_functions(setup):
7172
base_name="huggingface",
7273
)
7374

74-
training_job.create_training_job()
75+
try:
76+
training_job.create_training_job()
77+
except Exception as e:
78+
if "ResourceInUse" in str(e):
79+
pytest.xfail("Known issue with conflicting resources.")
80+
raise
7581
training_job.wait_until_training_job_complete()
7682

7783
# inference

‎tests/unit/sagemaker/image_uris/test_huggingface_llm.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,14 @@
4444
"us-west-1": "763104351884",
4545
"us-west-2": "763104351884",
4646
}
47-
HF_VERSIONS = ["0.6.0", "0.8.2", "0.9.3", "1.0.3"]
47+
HF_VERSIONS = ["0.6.0", "0.8.2", "0.9.3", "1.0.3", "1.1.0"]
4848
LMI_VERSIONS = ["0.23.0"]
4949
HF_VERSIONS_MAPPING = {
5050
"0.6.0": "2.0.0-tgi0.6.0-gpu-py39-cu118-ubuntu20.04",
5151
"0.8.2": "2.0.0-tgi0.8.2-gpu-py39-cu118-ubuntu20.04",
5252
"0.9.3": "2.0.1-tgi0.9.3-gpu-py39-cu118-ubuntu20.04",
5353
"1.0.3": "2.0.1-tgi1.0.3-gpu-py39-cu118-ubuntu20.04",
54+
"1.1.0": "2.0.1-tgi1.1.0-gpu-py39-cu118-ubuntu20.04",
5455
}
5556
LMI_VERSIONS_MAPPING = {"0.23.0": "deepspeed0.9.5-cu118"}
5657

‎tests/unit/sagemaker/workflow/test_transform_step.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def test_transform_step_with_transformer(model_name, data, output_path, pipeline
127127
)
128128

129129
step_def = json.loads(pipeline.definition())["Steps"][0]
130-
assert step_def["Arguments"]["TransformJobName"] == "TestTransformJobPrefix"
130+
assert step_def["Arguments"]["TransformJobName"].startswith("TestTransformJobPrefix")
131131
assert step_def == {
132132
"Name": "MyTransformStep",
133133
"Type": "Transform",

0 commit comments

Comments
 (0)