Merge branch 'master' into fix/unique-resource-names-js-integ-tests

Author: evakravi

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v2.189.0 (2023-10-03)
+
+### Features
+
+ * add feature processor APIs to public doc
+ * s3 prefix model data for JumpStartModel
+ * Model Package support for updating approval
+
+### Bug Fixes and Other Changes
+
+ * Add bucket owner check
+ * transform step  unit test
+ * Release TGI 1.1.0 Image
+
 ## v2.188.0 (2023-09-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.188.1.dev0
+2.189.1.dev0

doc/api/prep_data/feature_store.rst

@@ -120,3 +120,41 @@ Feature Store
 .. autoclass:: sagemaker.feature_store.feature_store.FeatureStore
     :members:
     :show-inheritance:
+
+
+@feature_processor Decorator
+****************************
+
+.. autodecorator:: sagemaker.feature_store.feature_processor.feature_processor
+
+
+Feature Processor Data Source
+*****************************
+
+.. autoclass:: sagemaker.feature_store.feature_processor.FeatureGroupDataSource
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_processor.CSVDataSource
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_processor.ParquetDataSource
+    :members:
+    :show-inheritance:
+
+
+Feature Processor Scheduler
+***************************
+
+.. automethod:: sagemaker.feature_store.feature_processor.to_pipeline
+
+.. automethod:: sagemaker.feature_store.feature_processor.schedule
+
+.. automethod:: sagemaker.feature_store.feature_processor.execute
+
+.. automethod:: sagemaker.feature_store.feature_processor.delete_schedule
+
+.. automethod:: sagemaker.feature_store.feature_processor.describe
+
+.. automethod:: sagemaker.feature_store.feature_processor.list_pipelines

doc/conf.py

@@ -109,6 +109,8 @@
 # autosectionlabel
 autosectionlabel_prefix_document = True
 
+autodoc_mock_imports = ["pyspark", "feature_store_pyspark", "py4j"]
+
 
 def setup(app):
     sys.stdout.write("Generating JumpStart model table...")

src/sagemaker/feature_store/feature_processor/feature_processor.py

@@ -49,7 +49,10 @@ def feature_processor(
     names nand types in [params: Dict[str, Any], spark: SparkSession]. Outputs: a single return
     value of type pyspark.sql.DataFrame. The function can have any name.
 
-    Example:
+    **Example:**
+
+    .. code-block:: python
+
         @feature_processor(
             inputs=[FeatureGroupDataSource("input-fg"), CSVDataSource("s3://bucket/prefix)],
             output='arn:aws:sagemaker:us-west-2:123456789012:feature-group/output-fg'
@@ -60,7 +63,10 @@ def transform(
         ) -> DataFrame:
             return ...
 
-    More concisely:
+    **More concisely:**
+
+    .. code-block:: python
+
         @feature_processor(
             inputs=[FeatureGroupDataSource("input-fg"), CSVDataSource("s3://bucket/prefix)],
             output='arn:aws:sagemaker:us-west-2:123456789012:feature-group/output-fg'

src/sagemaker/image_uri_config/huggingface-llm.json

@@ -5,7 +5,8 @@
             "0.6": "0.6.0",
             "0.8": "0.8.2",
             "0.9": "0.9.3",
-            "1.0": "1.0.3"
+            "1.0": "1.0.3",
+            "1.1": "1.1.0"
         },
         "versions": {
             "0.6.0": {
@@ -175,6 +176,48 @@
                 "tag_prefix": "2.0.1-tgi1.0.3",
                 "repository": "huggingface-pytorch-tgi-inference",
                 "container_version": {"gpu": "cu118-ubuntu20.04"}
+            },
+            "1.1.0": {
+                "py_versions": ["py39"],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "il-central-1": "780543022126",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "me-south-1": "217643126080",
+                    "me-central-1": "914824155844",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-isob-east-1": "094389454867",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "tag_prefix": "2.0.1-tgi1.1.0",
+                "repository": "huggingface-pytorch-tgi-inference",
+                "container_version": {"gpu": "cu118-ubuntu20.04"}
             }
         }
     }

src/sagemaker/session.py

@@ -233,6 +233,7 @@ def __init__(
         self._default_bucket_name_override = default_bucket
         # this may also be set again inside :func:`_initialize` if it is None
         self.default_bucket_prefix = default_bucket_prefix
+        self._default_bucket_set_by_sdk = False
 
         self.s3_resource = None
         self.s3_client = None
@@ -545,8 +546,12 @@ def default_bucket(self):
         default_bucket = self._default_bucket_name_override
         if not default_bucket:
             default_bucket = generate_default_sagemaker_bucket_name(self.boto_session)
+            self._default_bucket_set_by_sdk = True
 
-        self._create_s3_bucket_if_it_does_not_exist(bucket_name=default_bucket, region=region)
+        self._create_s3_bucket_if_it_does_not_exist(
+            bucket_name=default_bucket,
+            region=region,
+        )
 
         self._default_bucket = default_bucket
 
@@ -620,6 +625,28 @@ def _create_s3_bucket_if_it_does_not_exist(self, bucket_name, region):
                 else:
                     raise
 
+        if self._default_bucket_set_by_sdk:
+            # make sure the s3 bucket is configured in users account.
+            expected_bucket_owner_id = self.account_id()
+            try:
+                s3.meta.client.head_bucket(
+                    Bucket=bucket_name, ExpectedBucketOwner=expected_bucket_owner_id
+                )
+            except ClientError as e:
+                error_code = e.response["Error"]["Code"]
+                message = e.response["Error"]["Message"]
+                if error_code == "403" and message == "Forbidden":
+                    LOGGER.error(
+                        "Since default_bucket param was not set, SageMaker Python SDK tried to use "
+                        "%s bucket. "
+                        "This bucket cannot be configured to use as it is not owned by Account %s. "
+                        "To unblock it's recommended to use custom default_bucket "
+                        "parameter in sagemaker.Session",
+                        bucket_name,
+                        expected_bucket_owner_id,
+                    )
+                    raise
+
     def _append_sagemaker_config_tags(self, tags: list, config_path_to_tags: str):
         """Appends tags specified in the sagemaker_config to the given list of tags.
 

tests/unit/sagemaker/image_uris/test_huggingface_llm.py

@@ -44,13 +44,14 @@
     "us-west-1": "763104351884",
     "us-west-2": "763104351884",
 }
-HF_VERSIONS = ["0.6.0", "0.8.2", "0.9.3", "1.0.3"]
+HF_VERSIONS = ["0.6.0", "0.8.2", "0.9.3", "1.0.3", "1.1.0"]
 LMI_VERSIONS = ["0.23.0"]
 HF_VERSIONS_MAPPING = {
     "0.6.0": "2.0.0-tgi0.6.0-gpu-py39-cu118-ubuntu20.04",
     "0.8.2": "2.0.0-tgi0.8.2-gpu-py39-cu118-ubuntu20.04",
     "0.9.3": "2.0.1-tgi0.9.3-gpu-py39-cu118-ubuntu20.04",
     "1.0.3": "2.0.1-tgi1.0.3-gpu-py39-cu118-ubuntu20.04",
+    "1.1.0": "2.0.1-tgi1.1.0-gpu-py39-cu118-ubuntu20.04",
 }
 LMI_VERSIONS_MAPPING = {"0.23.0": "deepspeed0.9.5-cu118"}
 

tests/unit/sagemaker/workflow/test_transform_step.py

@@ -127,7 +127,7 @@ def test_transform_step_with_transformer(model_name, data, output_path, pipeline
     )
 
     step_def = json.loads(pipeline.definition())["Steps"][0]
-    assert step_def["Arguments"]["TransformJobName"] == "TestTransformJobPrefix"
+    assert step_def["Arguments"]["TransformJobName"].startswith("TestTransformJobPrefix")
     assert step_def == {
         "Name": "MyTransformStep",
         "Type": "Transform",

tests/unit/test_default_bucket.py

@@ -12,16 +12,22 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
+import datetime
 import pytest
 from botocore.exceptions import ClientError
-from mock import MagicMock, patch
+from mock import MagicMock
 import sagemaker
 
 ACCOUNT_ID = "123"
 REGION = "us-west-2"
 DEFAULT_BUCKET_NAME = "sagemaker-{}-{}".format(REGION, ACCOUNT_ID)
 
 
+@pytest.fixture
+def datetime_obj():
+    return datetime.datetime(2017, 6, 16, 15, 55, 0)
+
+
 @pytest.fixture()
 def sagemaker_session():
     boto_mock = MagicMock(name="boto_session", region_name=REGION)
@@ -50,23 +56,53 @@ def test_default_bucket_s3_create_call(sagemaker_session):
     assert sagemaker_session._default_bucket == bucket_name
 
 
-def test_default_bucket_s3_needs_access(sagemaker_session):
-    with patch("logging.Logger.error") as mocked_error_log:
-        with pytest.raises(ClientError):
-            error = ClientError(
-                error_response={"Error": {"Code": "403", "Message": "Forbidden"}},
-                operation_name="foo",
-            )
-            sagemaker_session.boto_session.resource(
-                "s3"
-            ).meta.client.head_bucket.side_effect = error
-            sagemaker_session.default_bucket()
-            mocked_error_log.assert_called_once_with(
-                "Bucket %s exists, but access is forbidden. Please try again after "
-                "adding appropriate access.",
-                DEFAULT_BUCKET_NAME,
-            )
-        assert sagemaker_session._default_bucket is None
+def test_default_bucket_s3_needs_access(sagemaker_session, caplog):
+    with pytest.raises(ClientError):
+        error = ClientError(
+            error_response={"Error": {"Code": "403", "Message": "Forbidden"}},
+            operation_name="foo",
+        )
+        sagemaker_session.boto_session.resource("s3").meta.client.head_bucket.side_effect = error
+        sagemaker_session.default_bucket()
+    error_message = (
+        " exists, but access is forbidden. Please try again after adding appropriate access."
+    )
+    assert error_message in caplog.text
+    assert sagemaker_session._default_bucket is None
+
+
+def test_default_bucket_s3_needs_bucket_owner_access(sagemaker_session, datetime_obj, caplog):
+    with pytest.raises(ClientError):
+        error = ClientError(
+            error_response={"Error": {"Code": "403", "Message": "Forbidden"}},
+            operation_name="foo",
+        )
+        sagemaker_session.boto_session.resource("s3").meta.client.head_bucket.side_effect = error
+        # bucket exists
+        sagemaker_session.boto_session.resource("s3").Bucket(
+            name=DEFAULT_BUCKET_NAME
+        ).creation_date = datetime_obj
+        sagemaker_session.default_bucket()
+
+    error_message = "This bucket cannot be configured to use as it is not owned by Account"
+    assert error_message in caplog.text
+    assert sagemaker_session._default_bucket is None
+
+
+def test_default_bucket_s3_custom_bucket_input(sagemaker_session, datetime_obj, caplog):
+    sagemaker_session._default_bucket_name_override = "custom-bucket-override"
+    error = ClientError(
+        error_response={"Error": {"Code": "403", "Message": "Forbidden"}},
+        operation_name="foo",
+    )
+    sagemaker_session.boto_session.resource("s3").meta.client.head_bucket.side_effect = error
+    # bucket exists
+    sagemaker_session.boto_session.resource("s3").Bucket(
+        name=DEFAULT_BUCKET_NAME
+    ).creation_date = datetime_obj
+    # This should not raise ClientError as no head_bucket call is expected for custom bucket
+    sagemaker_session.default_bucket()
+    assert sagemaker_session._default_bucket == "custom-bucket-override"
 
 
 def test_default_already_cached(sagemaker_session):