Merge branch 'aws:master' into master

Author: haixiw

.gitignore

@@ -2,7 +2,7 @@
 build
 src/*.egg-info
 .cache
-.coverage
+.coverage*
 sagemaker_venv*
 *.egg-info
 .tox

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## v2.188.0 (2023-09-26)
+
+### Features
+
+ * jumpstart instance type variants
+ * New methods to ingest and create Feature Groups
+
+### Bug Fixes and Other Changes
+
+ * auto ml integ tests and add flaky test markers
+ * Enhance unit-tests to automatically consume image URIs config registries from config JSONs
+
+## v2.187.0 (2023-09-19)
+
+### Features
+
+ * add HealthCheckConfig support
+ * SkipModelValidation in modelRegistry
+
+### Bug Fixes and Other Changes
+
+ * Update fw_utils.py - support 2.0.1 container for DDP and Torch distri…
+ * bump apache-airflow to v2.7.1
+
+## v2.186.0 (2023-09-14)
+
+### Features
+
+ * TGI 1.0.3 Image URI Config
+
+## v2.185.0 (2023-09-12)
+
+### Features
+
+ * Local Mode - Add Support for Docker Compose V2
+
+### Bug Fixes and Other Changes
+
+ * handle bad jumpstart default session
+ * Add Data Wrangler TLV and version 3.x images
+
+## v2.184.0.post0 (2023-09-11)
+
+### Documentation Changes
+
+ * add interactive apps rst file
+
 ## v2.184.0 (2023-09-07)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.184.1.dev0
+2.188.1.dev0

doc/api/utility/featuregroup_utils.rst

@@ -0,0 +1,7 @@
+FeatureGroup Utilities
+----------------------
+
+.. automodule:: sagemaker.feature_store.feature_utils
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/overview.rst

@@ -1550,14 +1550,15 @@ them to your local environment. This is a great way to test your deep learning s
 managed training or hosting environments. Local Mode is supported for frameworks images (TensorFlow, MXNet, Chainer, PyTorch,
 and Scikit-Learn) and images you supply yourself.
 
-You can install necessary dependencies for this feature using pip; local mode also requires docker-compose which you can
-install using the following steps (More info - https://github.com/docker/compose#where-to-get-docker-compose ):
+You can install necessary dependencies for this feature using pip.
 
 ::
 
     pip install 'sagemaker[local]' --upgrade
-    curl -L "https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-    chmod +x /usr/local/bin/docker-compose
+
+
+Additionally, Local Mode also requires Docker Compose V2. Follow the guidelines in https://docs.docker.com/compose/install/ to install.
+Make sure to have a Compose Version compatible with your Docker Engine installation. Check Docker Engine release notes https://docs.docker.com/engine/release-notes to find a compatible version.
 
 If you want to keep everything local, and not use Amazon S3 either, you can enable "local code" in one of two ways:
 

requirements/extras/test_requirements.txt

@@ -12,7 +12,7 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.7.0
+apache-airflow==2.7.1
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0

src/sagemaker/chainer/model.py

@@ -148,8 +148,8 @@ def __init__(
 
     def register(
         self,
-        content_types: List[Union[str, PipelineVariable]],
-        response_types: List[Union[str, PipelineVariable]],
+        content_types: List[Union[str, PipelineVariable]] = None,
+        response_types: List[Union[str, PipelineVariable]] = None,
         inference_instances: Optional[List[Union[str, PipelineVariable]]] = None,
         transform_instances: Optional[List[Union[str, PipelineVariable]]] = None,
         model_package_name: Optional[Union[str, PipelineVariable]] = None,
@@ -169,6 +169,7 @@ def register(
         framework_version: Optional[Union[str, PipelineVariable]] = None,
         nearest_model_name: Optional[Union[str, PipelineVariable]] = None,
         data_input_configuration: Optional[Union[str, PipelineVariable]] = None,
+        skip_model_validation: Optional[Union[str, PipelineVariable]] = None,
     ):
         """Creates a model package for creating SageMaker models or listing on Marketplace.
 
@@ -216,6 +217,8 @@ def register(
                 benchmarked by Amazon SageMaker Inference Recommender (default: None).
             data_input_configuration (str or PipelineVariable): Input object for the model
                 (default: None).
+            skip_model_validation (str or PipelineVariable): Indicates if you want to skip model
+                validation. Values can be "All" or "None" (default: None).
 
         Returns:
             str: A string of SageMaker Model Package ARN.
@@ -254,6 +257,7 @@ def register(
             framework_version=framework_version or self.framework_version,
             nearest_model_name=nearest_model_name,
             data_input_configuration=data_input_configuration,
+            skip_model_validation=skip_model_validation,
         )
 
     def prepare_container_def(

src/sagemaker/environment_variables.py

@@ -20,6 +20,7 @@
 from sagemaker.jumpstart import utils as jumpstart_utils
 from sagemaker.jumpstart import artifacts
 from sagemaker.jumpstart.constants import DEFAULT_JUMPSTART_SAGEMAKER_SESSION
+from sagemaker.jumpstart.enums import JumpStartScriptScope
 from sagemaker.session import Session
 
 logger = logging.getLogger(__name__)
@@ -33,6 +34,8 @@ def retrieve_default(
     tolerate_deprecated_model: bool = False,
     include_aws_sdk_env_vars: bool = True,
     sagemaker_session: Session = DEFAULT_JUMPSTART_SAGEMAKER_SESSION,
+    instance_type: Optional[str] = None,
+    script: JumpStartScriptScope = JumpStartScriptScope.INFERENCE,
 ) -> Dict[str, str]:
     """Retrieves the default container environment variables for the model matching the arguments.
 
@@ -58,6 +61,10 @@ def retrieve_default(
             object, used for SageMaker interactions. If not
             specified, one is created using the default AWS configuration
             chain. (Default: sagemaker.jumpstart.constants.DEFAULT_JUMPSTART_SAGEMAKER_SESSION).
+        instance_type (str): An instance type to optionally supply in order to get environment
+            variables specific for the instance type.
+        script (JumpStartScriptScope): The JumpStart script for which to retrieve environment
+            variables.
     Returns:
         dict: The variables to use for the model.
 
@@ -78,4 +85,6 @@ def retrieve_default(
         tolerate_deprecated_model,
         include_aws_sdk_env_vars,
         sagemaker_session=sagemaker_session,
+        instance_type=instance_type,
+        script=script,
     )

src/sagemaker/estimator.py

@@ -173,6 +173,7 @@ def __init__(
         instance_groups: Optional[List[InstanceGroup]] = None,
         training_repository_access_mode: Optional[Union[str, PipelineVariable]] = None,
         training_repository_credentials_provider_arn: Optional[Union[str, PipelineVariable]] = None,
+        enable_infra_check: Optional[Union[bool, PipelineVariable]] = None,
         container_entry_point: Optional[List[str]] = None,
         container_arguments: Optional[List[str]] = None,
         disable_output_compression: bool = False,
@@ -536,6 +537,8 @@ def __init__(
                 a training job.
             disable_output_compression (bool): Optional. When set to true, Model is uploaded
                 to Amazon S3 without compression after training finishes.
+            enable_infra_check (bool or PipelineVariable): Optional.
+                Specifies whether it is running Sagemaker built-in infra check jobs.
         """
         instance_count = renamed_kwargs(
             "train_instance_count", "instance_count", instance_count, kwargs
@@ -665,6 +668,7 @@ def __init__(
             training_repository_credentials_provider_arn
         )
 
+        self.enable_infra_check = enable_infra_check
         # container entry point / arguments configs
         self.container_entry_point = container_entry_point
         self.container_arguments = container_arguments
@@ -1661,8 +1665,8 @@ def deploy(
 
     def register(
         self,
-        content_types,
-        response_types,
+        content_types=None,
+        response_types=None,
         inference_instances=None,
         transform_instances=None,
         image_uri=None,
@@ -1684,6 +1688,7 @@ def register(
         framework_version=None,
         nearest_model_name=None,
         data_input_configuration=None,
+        skip_model_validation=None,
         **kwargs,
     ):
         """Creates a model package for creating SageMaker models or listing on Marketplace.
@@ -1729,6 +1734,8 @@ def register(
             nearest_model_name (str): Name of a pre-trained machine learning benchmarked by
                 Amazon SageMaker Inference Recommender (default: None).
             data_input_configuration (str): Input object for the model (default: None).
+            skip_model_validation (str): Indicates if you want to skip model validation.
+                Values can be "All" or "None" (default: None).
             **kwargs: Passed to invocation of ``create_model()``. Implementations may customize
                 ``create_model()`` to accept ``**kwargs`` to customize model creation during
                 deploy. For more, see the implementation docs.
@@ -1772,6 +1779,7 @@ def register(
             framework_version=framework_version,
             nearest_model_name=nearest_model_name,
             data_input_configuration=data_input_configuration,
+            skip_model_validation=skip_model_validation,
         )
 
     @property
@@ -1900,6 +1908,11 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
                 "EnableInterContainerTrafficEncryption"
             ]
 
+        if "InfraCheckConfig" in job_details:
+            init_params["enable_infra_check"] = job_details["InfraCheckConfig"].get(
+                "EnableInfraCheck"
+            )
+
         subnets, security_group_ids = vpc_utils.from_dict(job_details.get(vpc_utils.VPC_CONFIG_KEY))
         if subnets:
             init_params["subnets"] = subnets
@@ -2442,6 +2455,10 @@ def _get_train_args(cls, estimator, inputs, experiment_config):
                 ] = estimator.training_repository_credentials_provider_arn
             train_args["training_image_config"] = training_image_config
 
+        if estimator.enable_infra_check is not None:
+            infra_check_config = {"EnableInfraCheck": estimator.enable_infra_check}
+            train_args["infra_check_config"] = infra_check_config
+
         if estimator.container_entry_point is not None:
             train_args["container_entry_point"] = estimator.container_entry_point
 
@@ -2657,6 +2674,7 @@ def __init__(
         container_entry_point: Optional[List[str]] = None,
         container_arguments: Optional[List[str]] = None,
         disable_output_compression: bool = False,
+        enable_infra_check: Optional[Union[bool, PipelineVariable]] = None,
         **kwargs,
     ):
         """Initialize an ``Estimator`` instance.
@@ -3016,6 +3034,8 @@ def __init__(
                 a training job.
             disable_output_compression (bool): Optional. When set to true, Model is uploaded
                 to Amazon S3 without compression after training finishes.
+            enable_infra_check (bool or PipelineVariable): Optional.
+                Specifies whether it is running Sagemaker built-in infra check jobs.
         """
         self.image_uri = image_uri
         self._hyperparameters = hyperparameters.copy() if hyperparameters else {}

src/sagemaker/feature_store/feature_group.py

@@ -138,9 +138,14 @@ def get_query_execution(self) -> Dict[str, Any]:
             query_execution_id=self._current_query_execution_id
         )
 
-    def as_dataframe(self) -> DataFrame:
+    def as_dataframe(self, **kwargs) -> DataFrame:
         """Download the result of the current query and load it into a DataFrame.
 
+        Args:
+            **kwargs (object): key arguments used for the method pandas.read_csv to be able to
+                    have a better tuning on data. For more info read:
+                    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html
+
         Returns:
             A pandas DataFrame contains the query result.
         """
@@ -161,7 +166,9 @@ def as_dataframe(self) -> DataFrame:
             query_execution_id=self._current_query_execution_id,
             filename=output_filename,
         )
-        return pd.read_csv(output_filename, delimiter=",")
+
+        kwargs.pop("delimiter", None)
+        return pd.read_csv(filepath_or_buffer=output_filename, delimiter=",", **kwargs)
 
 
 @attr.s