Merge branch 'master' into change/remove-setuptools-deprecation

Author: jjerphan

CHANGELOG.md

@@ -1,5 +1,70 @@
 # Changelog
 
+## v2.194.0 (2023-10-19)
+
+### Features
+
+ * Added register step in Jumpstart model
+ * jumpstart instance specific metric definitions
+
+### Bug Fixes and Other Changes
+
+ * Updates for DJL 0.24.0 Release
+ * use getter for resource-metadata dict
+ * add method to Model class to check if repack is needed
+
+## v2.193.0 (2023-10-18)
+
+### Features
+
+ * jumpstart model artifact instance type variants
+ * jumpstart instance specific hyperparameters
+ * Feature Processor event based triggers (#1132)
+ * Support job checkpoint in remote function
+ * jumpstart model package arn instance type variants
+
+### Bug Fixes and Other Changes
+
+ * Fix hyperlinks in feature_processor.scheduler parameter descriptions
+ * add image_uris_unit_test pytest mark
+ * bump apache-airflow to `v2.7.2`
+ * clone distribution in validate_distribution
+ * fix flaky Inference Recommender integration tests
+
+### Documentation Changes
+
+ * Update PipelineModel.register documentation
+ * specify that input_shape in no longer required for torch 2.0 mod…
+
+## v2.192.1 (2023-10-13)
+
+### Bug Fixes and Other Changes
+
+ * update local mode schema
+ * import error in unsupported js regions
+ * Update Ec2 instance type to g5.4xlarge in test_huggingface_torch_distributed.py
+
+## v2.192.0 (2023-10-11)
+
+### Features
+
+ * jumpstart estimator enable infra check flag
+ * jumpstart default payloads
+ * allow non-python files in job dependencies
+ * allow configuring docker container in local mode
+
+### Bug Fixes and Other Changes
+
+ * js tagging s3 prefix
+ * Batch transform: Add support for split_type == "None" in local mode
+ * use correct line endings and s3 uris on windows
+ * Fixed bug in _create_training_details
+ * DJL Neuronx 0.24.0
+
+### Documentation Changes
+
+ * Include FeatureGroup's load_feature_definitions API documentation
+
 ## v2.191.0 (2023-10-05)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.191.1.dev0
+2.194.1.dev0

doc/amazon_sagemaker_featurestore.rst

@@ -230,9 +230,11 @@ The following code from the fraud detection example shows a minimal
        enable_online_store=True
    )
 
-Creating a feature group takes time as the data is loaded. You will need
-to wait until it is created before you can use it. You can check status
-using the following method.
+Creating a feature group takes time as the data is loaded. You will
+need to wait until it is created before you can use it. You can
+check status using the following method. Note that it can take
+approximately 10-15 minutes to provision an online ``FeatureGroup``
+with the ``InMemory`` ``StorageType``.
 
 .. code:: python
 
@@ -480,7 +482,9 @@ Feature Store `DatasetBuilder API Reference
 .. rubric:: Delete a feature group
    :name: bCe9CA61b78
 
-You can delete a feature group with the ``delete`` function.
+You can delete a feature group with the ``delete`` function. Note that it
+can take approximately 10-15 minutes to delete an online ``FeatureGroup``
+with the ``InMemory`` ``StorageType``.
 
 .. code:: python
 

requirements/extras/test_requirements.txt

@@ -12,7 +12,7 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.7.1
+apache-airflow==2.7.2
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0

src/sagemaker/base_predictor.py

@@ -14,7 +14,7 @@
 from __future__ import print_function, absolute_import
 
 import abc
-from typing import Any, Tuple
+from typing import Any, Optional, Tuple, Union
 
 from sagemaker.deprecations import (
     deprecated_class,
@@ -32,6 +32,9 @@
     StreamDeserializer,
     StringDeserializer,
 )
+from sagemaker.jumpstart.payload_utils import PayloadSerializer
+from sagemaker.jumpstart.types import JumpStartSerializablePayload
+from sagemaker.jumpstart.utils import get_jumpstart_content_bucket
 from sagemaker.model_monitor import (
     DataCaptureConfig,
     DefaultModelMonitor,
@@ -201,20 +204,44 @@ def _create_request_args(
         custom_attributes=None,
     ):
         """Placeholder docstring"""
+
+        jumpstart_serialized_data: Optional[Union[str, bytes]] = None
+        jumpstart_accept: Optional[str] = None
+        jumpstart_content_type: Optional[str] = None
+
+        if isinstance(data, JumpStartSerializablePayload):
+            s3_client = self.sagemaker_session.s3_client
+            region = self.sagemaker_session._region_name
+            bucket = get_jumpstart_content_bucket(region)
+
+            jumpstart_serialized_data = PayloadSerializer(
+                bucket=bucket, region=region, s3_client=s3_client
+            ).serialize(data)
+            jumpstart_content_type = data.content_type
+            jumpstart_accept = data.accept
+
         args = dict(initial_args) if initial_args else {}
 
         if "EndpointName" not in args:
             args["EndpointName"] = self.endpoint_name
 
         if "ContentType" not in args:
-            args["ContentType"] = (
-                self.content_type
-                if isinstance(self.content_type, str)
-                else ", ".join(self.content_type)
-            )
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_content_type:
+                args["ContentType"] = jumpstart_content_type
+            else:
+                args["ContentType"] = (
+                    self.content_type
+                    if isinstance(self.content_type, str)
+                    else ", ".join(self.content_type)
+                )
 
         if "Accept" not in args:
-            args["Accept"] = self.accept if isinstance(self.accept, str) else ", ".join(self.accept)
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_accept:
+                args["Accept"] = jumpstart_accept
+            else:
+                args["Accept"] = (
+                    self.accept if isinstance(self.accept, str) else ", ".join(self.accept)
+                )
 
         if target_model:
             args["TargetModel"] = target_model
@@ -228,7 +255,11 @@ def _create_request_args(
         if custom_attributes:
             args["CustomAttributes"] = custom_attributes
 
-        data = self.serializer.serialize(data)
+        data = (
+            jumpstart_serialized_data
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_serialized_data
+            else self.serializer.serialize(data)
+        )
 
         args["Body"] = data
         return args

src/sagemaker/config/config_schema.py

@@ -106,6 +106,7 @@
 LOCAL_CODE = "local_code"
 SERVING_PORT = "serving_port"
 CONTAINER_CONFIG = "container_config"
+CONTAINER_ROOT = "container_root"
 REGION_NAME = "region_name"
 
 
@@ -1090,6 +1091,9 @@ def _simple_path(*args: str):
                 SERVING_PORT: {
                     TYPE: "integer",
                 },
+                CONTAINER_ROOT: {
+                    TYPE: "string",
+                },
                 CONTAINER_CONFIG: {
                     TYPE: OBJECT,
                 },

src/sagemaker/djl_inference/model.py

@@ -781,7 +781,7 @@ def serving_image_uri(self, region_name):
             str: The appropriate image URI based on the given parameters.
         """
         if not self.djl_version:
-            self.djl_version = "0.23.0"
+            self.djl_version = "0.24.0"
 
         return image_uris.retrieve(
             self._framework(),

src/sagemaker/estimator.py

@@ -3082,6 +3082,7 @@ def __init__(
             hyperparameters=hyperparameters,
             instance_groups=instance_groups,
             training_repository_access_mode=training_repository_access_mode,
+            enable_infra_check=enable_infra_check,
             training_repository_credentials_provider_arn=training_repository_credentials_provider_arn,  # noqa: E501 # pylint: disable=line-too-long
             container_entry_point=container_entry_point,
             container_arguments=container_arguments,

src/sagemaker/feature_store/feature_processor/__init__.py

@@ -30,8 +30,16 @@
     to_pipeline,
     schedule,
     describe,
+    put_trigger,
+    delete_trigger,
+    enable_trigger,
+    disable_trigger,
     delete_schedule,
     list_pipelines,
     execute,
     TransformationCode,
+    FeatureProcessorPipelineEvents,
+)
+from sagemaker.feature_store.feature_processor._enums import (  # noqa: F401
+    FeatureProcessorPipelineExecutionStatus,
 )

src/sagemaker/feature_store/feature_processor/_constants.py

@@ -17,6 +17,7 @@
 
 DEFAULT_INSTANCE_TYPE = "ml.m5.xlarge"
 DEFAULT_SCHEDULE_STATE = "ENABLED"
+DEFAULT_TRIGGER_STATE = "ENABLED"
 UNDERSCORE = "_"
 RESOURCE_NOT_FOUND_EXCEPTION = "ResourceNotFoundException"
 RESOURCE_NOT_FOUND = "ResourceNotFound"
@@ -36,6 +37,8 @@
 FEATURE_PROCESSOR_TAG_KEY = "sm-fs-fe:created-from"
 FEATURE_PROCESSOR_TAG_VALUE = "fp-to-pipeline"
 FEATURE_GROUP_ARN_REGEX_PATTERN = r"arn:(.*?):sagemaker:(.*?):(.*?):feature-group/(.*?)$"
+PIPELINE_ARN_REGEX_PATTERN = r"arn:(.*?):sagemaker:(.*?):(.*?):pipeline/(.*?)$"
+EVENTBRIDGE_RULE_ARN_REGEX_PATTERN = r"arn:(.*?):events:(.*?):(.*?):rule/(.*?)$"
 SAGEMAKER_WHL_FILE_S3_PATH = "s3://ada-private-beta/sagemaker-2.151.1.dev0-py2.py3-none-any.whl"
 S3_DATA_DISTRIBUTION_TYPE = "FullyReplicated"
 PIPELINE_CONTEXT_NAME_TAG_KEY = "sm-fs-fe:feature-engineering-pipeline-context-name"
@@ -45,3 +48,7 @@
     PIPELINE_CONTEXT_NAME_TAG_KEY,
     PIPELINE_VERSION_CONTEXT_NAME_TAG_KEY,
 ]
+BASE_EVENT_PATTERN = {
+    "source": ["aws.sagemaker"],
+    "detail": {"currentPipelineExecutionStatus": [], "pipelineArn": []},
+}

src/sagemaker/feature_store/feature_processor/_enums.py

@@ -21,3 +21,13 @@ class FeatureProcessorMode(Enum):
 
     PYSPARK = "pyspark"  # Execute a pyspark job.
     PYTHON = "python"  # Execute a regular python script.
+
+
+class FeatureProcessorPipelineExecutionStatus(Enum):
+    """Enum of feature_processor pipeline execution status."""
+
+    EXECUTING = "Executing"
+    STOPPING = "Stopping"
+    STOPPED = "Stopped"
+    FAILED = "Failed"
+    SUCCEEDED = "Succeeded"