Merge branch 'aws:master' into master

Author: rohangujarathi

CHANGELOG.md

@@ -1,5 +1,62 @@
 # Changelog
 
+## v2.46.1 (2021-06-22)
+
+### Bug Fixes and Other Changes
+
+ * Register model step tags
+
+### Documentation Changes
+
+ * update to include new batch_get_record api call
+ * Correct type annotation for TrainingStep inputs
+ * introduce input mode FastFile
+ * update hf transformer version
+
+## v2.46.0 (2021-06-15)
+
+### Features
+
+ * Add HF transformer version 4.6.1
+
+### Bug Fixes and Other Changes
+
+ * encode localmode payload to UTF-8
+ * call DescribeDomain as fallback in get_execution_role
+ * parameterize PT and TF version for HuggingFace tests
+
+### Documentation Changes
+
+ * Add import statement in Batch Transform Overview doc
+
+## v2.45.0 (2021-06-07)
+
+### Features
+
+ * Add support for Callback steps in model building pipelines
+
+## v2.44.0 (2021-06-01)
+
+### Features
+
+ * support endpoint_name_prefix, seed and version for Clarify
+
+## v2.43.0 (2021-05-31)
+
+### Features
+
+ * add xgboost framework version 1.3-1
+
+### Bug Fixes and Other Changes
+
+ * remove duplicated tags in _append_project_tags
+
+## v2.42.1 (2021-05-27)
+
+### Bug Fixes and Other Changes
+
+ * default value removed if zero for integer param
+
 ## v2.42.0 (2021-05-24)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.42.1.dev0
+2.46.2.dev0

doc/amazon_sagemaker_featurestore.rst

@@ -291,6 +291,13 @@ example identifier to retrieve the record.
    record_identifier_value = str(2990130)
    featurestore_runtime.get_record(FeatureGroupName=transaction_feature_group_name, RecordIdentifierValueAsString=record_identifier_value)
 
+You can use the ``batch_get_record`` function to retrieve multiple records simultaneously from your feature store. The following example uses this API to retrieve a batch of records.
+
+.. code:: python
+
+   record_identifier_values = ["573291", "109382", "828400", "124013"]
+   featurestore_runtime.batch_get_record(Identifiers=[{"FeatureGroupName": transaction_feature_group_name, "RecordIdentifiersValueAsString": record_identifier_values}])
+
 An example response from the fraud detection example:
 
 .. code:: python

doc/frameworks/pytorch/using_pytorch.rst

@@ -31,7 +31,7 @@ To train a PyTorch model by using the SageMaker Python SDK:
 Prepare a PyTorch Training Script
 =================================
 
-Your PyTorch training script must be a Python 2.7 or 3.5 compatible source file.
+Your PyTorch training script must be a Python 3.6 compatible source file.
 
 Prepare your script in a separate source file than the notebook, terminal session, or source file you're
 using to submit the script to SageMaker via a ``PyTorch`` Estimator. This will be discussed in further detail below.

doc/overview.rst

@@ -703,6 +703,8 @@ Alternatively, if you already have a SageMaker model, you can create an instance
 
 .. code:: python
 
+    from sagemaker.transformer import Transformer
+
     transformer = Transformer(model_name='my-previously-trained-model',
                               instance_count=1,
                               instance_type='ml.m4.xlarge')

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -6,7 +6,6 @@ ConditionStep
 
 .. autoclass:: sagemaker.workflow.condition_step.ConditionStep
 
-.. autoclass:: sagemaker.workflow.condition_step.JsonGet
 
 Conditions
 ----------
@@ -55,6 +54,7 @@ Functions
 ---------
 
 .. autoclass:: sagemaker.workflow.functions.Join
+.. autoclass:: sagemaker.workflow.functions.JsonGet
 
 Parameters
 ----------

src/sagemaker/_studio.py

@@ -46,6 +46,7 @@ def _append_project_tags(tags=None, working_dir=None):
         return tags
 
     all_tags = tags or []
+    additional_tags = [tag for tag in additional_tags if tag not in all_tags]
     all_tags.extend(additional_tags)
 
     return all_tags

src/sagemaker/clarify.py

@@ -1007,6 +1007,8 @@ def register(
         if compile_model_family is not None:
             model = self._compiled_models[compile_model_family]
         else:
+            if "model_kms_key" not in kwargs:
+                kwargs["model_kms_key"] = self.output_kms_key
             model = self.create_model(image_uri=image_uri, **kwargs)
         model.name = model_name
         return model.register(

src/sagemaker/estimator.py

@@ -21,6 +21,7 @@
 
 from __future__ import absolute_import
 
+import copy
 import logging
 import math
 import os
@@ -193,6 +194,10 @@ def _ingest_single_batch(
         Returns:
             List of row indices that failed to be ingested.
         """
+        retry_config = client_config.retries
+        if "max_attempts" not in retry_config and "total_max_attempts" not in retry_config:
+            client_config = copy.deepcopy(client_config)
+            client_config.retries = {"max_attempts": 10, "mode": "standard"}
         sagemaker_featurestore_runtime_client = boto3.Session().client(
             service_name="sagemaker-featurestore-runtime", config=client_config
         )

src/sagemaker/feature_store/feature_group.py

@@ -66,7 +66,7 @@ def __init__(
                 must point to a file located at the root of ``source_dir``.
             transformers_version (str): Transformers version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
-                ``image_uri`` is provided. The current supported version is ``4.4.2``.
+                ``image_uri`` is provided. The current supported version is ``4.6.1``.
             tensorflow_version (str): TensorFlow version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
                 ``pytorch_version`` is provided. The current supported version is ``1.6.0``.

src/sagemaker/huggingface/estimator.py

@@ -3,7 +3,8 @@
         "processors": ["gpu"],
         "version_aliases": {
             "4.4": "4.4.2",
-            "4.5": "4.5.0"
+            "4.5": "4.5.0",
+            "4.6": "4.6.1"
         },
         "versions": {
             "4.4.2": {
@@ -137,6 +138,103 @@
                     },
                     "repository": "huggingface-tensorflow-training"
                 }
+            },
+            "4.6.1": {
+                "version_aliases": {
+                    "pytorch1.6": "pytorch1.6.0",
+                    "pytorch1.7": "pytorch1.7.1",
+                    "tensorflow2.4": "tensorflow2.4.1"
+                },
+                "pytorch1.6.0": {
+                    "py_versions": ["py36"],
+                    "registries": {
+                        "af-south-1": "626614931356",
+                        "ap-east-1": "871362719292",
+                        "ap-northeast-1": "763104351884",
+                        "ap-northeast-2": "763104351884",
+                        "ap-south-1": "763104351884",
+                        "ap-southeast-1": "763104351884",
+                        "ap-southeast-2": "763104351884",
+                        "ca-central-1": "763104351884",
+                        "cn-north-1": "727897471807",
+                        "cn-northwest-1": "727897471807",
+                        "eu-central-1": "763104351884",
+                        "eu-north-1": "763104351884",
+                        "eu-west-1": "763104351884",
+                        "eu-west-2": "763104351884",
+                        "eu-west-3": "763104351884",
+                        "eu-south-1": "692866216735",
+                        "me-south-1": "217643126080",
+                        "sa-east-1": "763104351884",
+                        "us-east-1": "763104351884",
+                        "us-east-2": "763104351884",
+                        "us-gov-west-1": "442386744353",
+                        "us-iso-east-1": "886529160074",
+                        "us-west-1": "763104351884",
+                        "us-west-2": "763104351884"
+                    },
+                    "repository": "huggingface-pytorch-training"
+                },
+                "pytorch1.7.1": {
+                    "py_versions": ["py36"],
+                    "registries": {
+                        "af-south-1": "626614931356",
+                        "ap-east-1": "871362719292",
+                        "ap-northeast-1": "763104351884",
+                        "ap-northeast-2": "763104351884",
+                        "ap-south-1": "763104351884",
+                        "ap-southeast-1": "763104351884",
+                        "ap-southeast-2": "763104351884",
+                        "ca-central-1": "763104351884",
+                        "cn-north-1": "727897471807",
+                        "cn-northwest-1": "727897471807",
+                        "eu-central-1": "763104351884",
+                        "eu-north-1": "763104351884",
+                        "eu-west-1": "763104351884",
+                        "eu-west-2": "763104351884",
+                        "eu-west-3": "763104351884",
+                        "eu-south-1": "692866216735",
+                        "me-south-1": "217643126080",
+                        "sa-east-1": "763104351884",
+                        "us-east-1": "763104351884",
+                        "us-east-2": "763104351884",
+                        "us-gov-west-1": "442386744353",
+                        "us-iso-east-1": "886529160074",
+                        "us-west-1": "763104351884",
+                        "us-west-2": "763104351884"
+                    },
+                    "repository": "huggingface-pytorch-training"
+                },
+                "tensorflow2.4.1": {
+                    "py_versions": ["py37"],
+                    "registries": {
+                        "af-south-1": "626614931356",
+                        "ap-east-1": "871362719292",
+                        "ap-northeast-1": "763104351884",
+                        "ap-northeast-2": "763104351884",
+                        "ap-south-1": "763104351884",
+                        "ap-southeast-1": "763104351884",
+                        "ap-southeast-2": "763104351884",
+                        "ca-central-1": "763104351884",
+                        "cn-north-1": "727897471807",
+                        "cn-northwest-1": "727897471807",
+                        "eu-central-1": "763104351884",
+                        "eu-north-1": "763104351884",
+                        "eu-south-1": "692866216735",
+                        "eu-west-1": "763104351884",
+                        "eu-west-2": "763104351884",
+                        "eu-west-3": "763104351884",
+                        "me-south-1": "217643126080",
+                        "sa-east-1": "763104351884",
+                        "us-east-1": "763104351884",
+                        "us-east-2": "763104351884",
+                        "us-gov-west-1": "442386744353",
+                        "us-iso-east-1": "886529160074",
+                        "us-west-1": "763104351884",
+                        "us-west-2": "763104351884"
+                    },
+                    "repository": "huggingface-tensorflow-training"
+                }
             }
         }
     }

src/sagemaker/image_uri_config/huggingface.json

@@ -183,6 +183,35 @@
                 "us-west-2": "246618743249"
             },
             "repository": "sagemaker-xgboost"
+        },
+        "1.3-1": {
+            "registries": {
+                "af-south-1": "510948584623",
+                "ap-east-1": "651117190479",
+                "ap-northeast-1": "354813040037",
+                "ap-northeast-2": "366743142698",
+                "ap-south-1": "720646828776",
+                "ap-southeast-1": "121021644041",
+                "ap-southeast-2": "783357654285",
+                "ca-central-1": "341280168497",
+                "cn-north-1": "450853457545",
+                "cn-northwest-1": "451049120500",
+                "eu-central-1": "492215442770",
+                "eu-north-1": "662702820516",
+                "eu-west-1": "141502667606",
+                "eu-west-2": "764974769150",
+                "eu-west-3": "659782779980",
+                "eu-south-1": "978288397137",
+                "me-south-1": "801668240914",
+                "sa-east-1": "737474898029",
+                "us-east-1": "683313688378",
+                "us-east-2": "257758044811",
+                "us-gov-west-1": "414596584902",
+                "us-iso-east-1": "833128469047",
+                "us-west-1": "746614075791",
+                "us-west-2": "246618743249"
+            },
+            "repository": "sagemaker-xgboost"
         }
     }
 }

src/sagemaker/image_uri_config/xgboost.json

@@ -70,6 +70,8 @@ def __init__(
                         a local directory.
                     * 'Pipe' - Amazon SageMaker streams data directly from S3 to the container via
                         a Unix-named pipe.
+                    * 'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                        downloading the entire dataset before training begins.
 
             attribute_names (list[str]): A list of one or more attribute names to use that are
                 found in a specified AugmentedManifestFile.

src/sagemaker/inputs.py

@@ -469,6 +469,9 @@ def invoke_endpoint(
         if InferenceId is not None:
             headers["X-Amzn-SageMaker-Inference-Id"] = InferenceId
 
+        # The http client encodes all strings using latin-1, which is not what we want.
+        if isinstance(Body, str):
+            Body = Body.encode("utf-8")
         r = self.http.request("POST", url, body=Body, preload_content=False, headers=headers)
 
         return {"Body": r, "ContentType": Accept}

src/sagemaker/local/local_session.py

@@ -195,6 +195,7 @@ def _get_model_package_args(
         marketplace_cert=False,
         approval_status=None,
         description=None,
+        tags=None,
     ):
         """Get arguments for session.create_model_package method.
 
@@ -250,6 +251,8 @@ def _get_model_package_args(
             model_package_args["approval_status"] = approval_status
         if description is not None:
             model_package_args["description"] = description
+        if tags is not None:
+            model_package_args["tags"] = tags
         return model_package_args
 
     def _init_sagemaker_session_if_does_not_exist(self, instance_type):