Merge branch 'master' into clarify-headers

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v2.46.0 (2021-06-15)
+
+### Features
+
+ * Add HF transformer version 4.6.1
+
+### Bug Fixes and Other Changes
+
+ * encode localmode payload to UTF-8
+ * call DescribeDomain as fallback in get_execution_role
+ * parameterize PT and TF version for HuggingFace tests
+
+### Documentation Changes
+
+ * Add import statement in Batch Transform Overview doc
+
 ## v2.45.0 (2021-06-07)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.45.1.dev0
+2.46.1.dev0

src/sagemaker/huggingface/estimator.py

@@ -66,7 +66,7 @@ def __init__(
                 must point to a file located at the root of ``source_dir``.
             transformers_version (str): Transformers version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
-                ``image_uri`` is provided. The current supported version is ``4.4.2``.
+                ``image_uri`` is provided. The current supported version is ``4.6.1``.
             tensorflow_version (str): TensorFlow version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
                 ``pytorch_version`` is provided. The current supported version is ``1.6.0``.

src/sagemaker/inputs.py

@@ -70,6 +70,8 @@ def __init__(
                         a local directory.
                     * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via
                         a Unix-named pipe.
+                    * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                        downloading the entire dataset before training begins.
 
             attribute_names (list[str]): A list of one or more attribute names to use that are
                 found in a specified AugmentedManifestFile.

src/sagemaker/model.py

@@ -195,6 +195,7 @@ def _get_model_package_args(
         marketplace_cert=False,
         approval_status=None,
         description=None,
+        tags=None,
     ):
         """Get arguments for session.create_model_package method.
 
@@ -250,6 +251,8 @@ def _get_model_package_args(
             model_package_args["approval_status"] = approval_status
         if description is not None:
             model_package_args["description"] = description
+        if tags is not None:
+            model_package_args["tags"] = tags
         return model_package_args
 
     def _init_sagemaker_session_if_does_not_exist(self, instance_type):

src/sagemaker/session.py

@@ -467,6 +467,8 @@ def train(  # noqa: C901
                 a directory in the Docker container.
                 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
                 Unix-named pipe.
+                * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                downloading the entire dataset before training begins.
             input_config (list): A list of Channel objects. Each channel is a named input source.
                 Please refer to the format details described:
                 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
@@ -609,6 +611,8 @@ def _get_train_request(  # noqa: C901
                 a directory in the Docker container.
                 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
                 Unix-named pipe.
+                * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                downloading the entire dataset before training begins.
             input_config (list): A list of Channel objects. Each channel is a named input source.
                 Please refer to the format details described:
                 https://botocore.readthedocs.io/en/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
@@ -1897,6 +1901,8 @@ def tune(  # noqa: C901
                 a directory in the Docker container.
                 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
                 Unix-named pipe.
+                * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                downloading the entire dataset before training begins.
             metric_definitions (list[dict]): A list of dictionaries that defines the metric(s)
                 used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
                 the name of the metric, and 'Regex' for the regular expression used to extract the
@@ -2180,6 +2186,8 @@ def _map_training_config(
                     a directory in the Docker container.
                 * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via a
                     Unix-named pipe.
+                * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                    downloading the entire dataset before training begins.
             role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training
                 jobs and APIs that create Amazon SageMaker endpoints use this role to access
                 training data and model artifacts. You must grant sufficient permissions to
@@ -2716,6 +2724,7 @@ def _get_create_model_package_request(
         marketplace_cert=False,
         approval_status="PendingManualApproval",
         description=None,
+        tags=None,
     ):
         """Get request dictionary for CreateModelPackage API.
 
@@ -2753,6 +2762,8 @@ def _get_create_model_package_request(
             request_dict["ModelPackageGroupName"] = model_package_group_name
         if description is not None:
             request_dict["ModelPackageDescription"] = description
+        if tags is not None:
+            request_dict["Tags"] = tags
         if model_metrics:
             request_dict["ModelMetrics"] = model_metrics
         if metadata_properties:

src/sagemaker/workflow/_utils.py

@@ -225,6 +225,7 @@ def __init__(
         compile_model_family=None,
         description=None,
         depends_on: List[str] = None,
+        tags=None,
         **kwargs,
     ):
         """Constructor of a register model step.
@@ -264,6 +265,7 @@ def __init__(
         self.inference_instances = inference_instances
         self.transform_instances = transform_instances
         self.model_package_group_name = model_package_group_name
+        self.tags = tags
         self.model_metrics = model_metrics
         self.metadata_properties = metadata_properties
         self.approval_status = approval_status
@@ -324,10 +326,12 @@ def arguments(self) -> RequestType:
             metadata_properties=self.metadata_properties,
             approval_status=self.approval_status,
             description=self.description,
+            tags=self.tags,
         )
         request_dict = model.sagemaker_session._get_create_model_package_request(
             **model_package_args
         )
+
         # these are not available in the workflow service and will cause rejection
         if "CertifyForMarketplace" in request_dict:
             request_dict.pop("CertifyForMarketplace")

src/sagemaker/workflow/step_collections.py

@@ -67,6 +67,7 @@ def __init__(
         image_uri=None,
         compile_model_family=None,
         description=None,
+        tags=None,
         **kwargs,
     ):
         """Construct steps `_RepackModelStep` and `_RegisterModelStep` based on the estimator.
@@ -94,6 +95,10 @@ def __init__(
             compile_model_family (str): The instance family for the compiled model. If
                 specified, a compiled model is used (default: None).
             description (str): Model Package description (default: None).
+            tags (List[dict[str, str]]): The list of tags to attach to the model package group. Note
+                that tags will only be applied to newly created model package groups; if the
+                name of an existing group is passed to "model_package_group_name",
+                tags will not be applied.
             **kwargs: additional arguments to `create_model`.
         """
         steps: List[Step] = []
@@ -134,6 +139,7 @@ def __init__(
             image_uri=image_uri,
             compile_model_family=compile_model_family,
             description=description,
+            tags=tags,
             **kwargs,
         )
         if not repack_model:

src/sagemaker/workflow/steps.py

@@ -16,16 +16,12 @@
 import abc
 
 from enum import Enum
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import attr
 
 from sagemaker.estimator import EstimatorBase, _TrainingJob
-from sagemaker.inputs import (
-    CreateModelInput,
-    TrainingInput,
-    TransformInput,
-)
+from sagemaker.inputs import CreateModelInput, TrainingInput, TransformInput, FileSystemInput
 from sagemaker.model import Model
 from sagemaker.processing import (
     ProcessingInput,
@@ -145,7 +141,7 @@ def __init__(
         self,
         name: str,
         estimator: EstimatorBase,
-        inputs: TrainingInput = None,
+        inputs: Union[TrainingInput, dict, str, FileSystemInput] = None,
         cache_config: CacheConfig = None,
         depends_on: List[str] = None,
     ):
@@ -157,7 +153,23 @@ def __init__(
         Args:
             name (str): The name of the training step.
             estimator (EstimatorBase): A `sagemaker.estimator.EstimatorBase` instance.
-            inputs (TrainingInput): A `sagemaker.inputs.TrainingInput` instance. Defaults to `None`.
+            inputs (str or dict or sagemaker.inputs.TrainingInput
+                or sagemaker.inputs.FileSystemInput): Information
+                about the training data. This can be one of three types:
+
+                * (str) the S3 location where training data is saved, or a file:// path in
+                    local mode.
+                * (dict[str, str] or dict[str, sagemaker.inputs.TrainingInput]) If using multiple
+                    channels for training data, you can specify a dict mapping channel names to
+                    strings or :func:`~sagemaker.inputs.TrainingInput` objects.
+                * (sagemaker.inputs.TrainingInput) - channel configuration for S3 data sources
+                    that can provide additional information as well as the path to the training
+                    dataset.
+                    See :func:`sagemaker.inputs.TrainingInput` for full details.
+                * (sagemaker.inputs.FileSystemInput) - channel configuration for
+                    a file system data source that can provide additional information as well as
+                    the path to the training dataset.
+
             cache_config (CacheConfig):  A `sagemaker.workflow.steps.CacheConfig` instance.
             depends_on (List[str]): A list of step names this `sagemaker.workflow.steps.TrainingStep`
                 depends on

tests/unit/sagemaker/workflow/test_step_collections.py

@@ -182,6 +182,7 @@ def test_register_model(estimator, model_metrics):
         approval_status="Approved",
         description="description",
         depends_on=["TestStep"],
+        tags=[{"Key": "myKey", "Value": "myValue"}],
     )
     assert ordered(register_model.request_dicts()) == ordered(
         [
@@ -210,6 +211,7 @@ def test_register_model(estimator, model_metrics):
                     },
                     "ModelPackageDescription": "description",
                     "ModelPackageGroupName": "mpg",
+                    "Tags": [{"Key": "myKey", "Value": "myValue"}],
                 },
             },
         ]