Merge pull request aws#37 from verdimrc/pr-framework-processor-round-02

Merge framework-processor-round-02 to framework-processor branch

Author: verdimrc

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v2.42.1 (2021-05-27)
+
+### Bug Fixes and Other Changes
+
+ * default value removed if zero for integer param
+
+## v2.42.0 (2021-05-24)
+
+### Features
+
+ * support for custom pipeline execution name
+ * Add data ingestion only data-wrangler flow recipe generation helper function
+
+### Bug Fixes and Other Changes
+
+ * add kms key for processing job code upload
+ * remove failing notebooks from notebook pr test
+ * fix in and not in condition bug
+ * Update overview.rst
+
+### Documentation Changes
+
+ * Update "Ask a question" contact link
+ * Update smdp docs with sparse_as_dense support
+
 ## v2.41.0 (2021-05-17)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.41.1.dev0
+2.42.2.dev0

src/sagemaker/_studio.py

@@ -46,6 +46,7 @@ def _append_project_tags(tags=None, working_dir=None):
         return tags
 
     all_tags = tags or []
+    additional_tags = [tag for tag in additional_tags if tag not in all_tags]
     all_tags.extend(additional_tags)
 
     return all_tags

src/sagemaker/huggingface/__init__.py

@@ -14,3 +14,4 @@
 from __future__ import absolute_import
 
 from sagemaker.huggingface.estimator import HuggingFace  # noqa: F401
+from sagemaker.huggingface.processing import HuggingFaceProcessor  # noqa:F401

src/sagemaker/huggingface/processing.py

@@ -0,0 +1,132 @@
+# Copyright 2019-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains code related to HuggingFace Processors which are used for Processing jobs.
+
+These jobs let customers perform data pre-processing, post-processing, feature engineering,
+data validation, and model evaluation and interpretation on SageMaker.
+"""
+from __future__ import absolute_import
+
+from sagemaker.processing import FrameworkProcessor
+from sagemaker.huggingface.estimator import HuggingFace
+
+
+class HuggingFaceProcessor(FrameworkProcessor):
+    """Handles Amazon SageMaker processing tasks for jobs using HuggingFace containers."""
+
+    estimator_cls = HuggingFace
+
+    def __init__(
+        self,
+        role,
+        instance_count,
+        instance_type,
+        transformers_version=None,
+        tensorflow_version=None,
+        pytorch_version=None,
+        py_version="py36",
+        image_uri=None,
+        command=None,
+        volume_size_in_gb=30,
+        volume_kms_key=None,
+        output_kms_key=None,
+        code_location=None,
+        max_runtime_in_seconds=None,
+        base_job_name=None,
+        sagemaker_session=None,
+        env=None,
+        tags=None,
+        network_config=None,
+    ):
+        """This processor executes a Python script in a HuggingFace execution environment.
+
+        Unless ``image_uri`` is specified, the environment is an Amazon-built Docker container
+        that executes functions defined in the supplied ``code`` Python script.
+
+        The arguments have the same meaning as in ``FrameworkProcessor``, with the following
+        exceptions.
+
+        Args:
+            transformers_version (str): Transformers version you want to use for
+                executing your model training code. Defaults to ``None``. Required unless
+                ``image_uri`` is provided. The current supported version is ``4.4.2``.
+            tensorflow_version (str): TensorFlow version you want to use for
+                executing your model training code. Defaults to ``None``. Required unless
+                ``pytorch_version`` is provided. The current supported version is ``1.6.0``.
+            pytorch_version (str): PyTorch version you want to use for
+                executing your model training code. Defaults to ``None``. Required unless
+                ``tensorflow_version`` is provided. The current supported version is ``2.4.1``.
+            py_version (str): Python version you want to use for executing your model training
+                code. Defaults to ``None``. Required unless ``image_uri`` is provided.  If
+                using PyTorch, the current supported version is ``py36``. If using TensorFlow,
+                the current supported version is ``py37``.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.processing.FrameworkProcessor`.
+        """
+        self.pytorch_version = pytorch_version
+        self.tensorflow_version = tensorflow_version
+        super().__init__(
+            self.estimator_cls,
+            transformers_version,
+            role,
+            instance_count,
+            instance_type,
+            py_version,
+            image_uri,
+            command,
+            volume_size_in_gb,
+            volume_kms_key,
+            output_kms_key,
+            code_location,
+            max_runtime_in_seconds,
+            base_job_name,
+            sagemaker_session,
+            env,
+            tags,
+            network_config,
+        )
+
+    def _create_estimator(
+        self,
+        entry_point="",
+        source_dir=None,
+        dependencies=None,
+        git_config=None,
+    ):
+        """Override default estimator factory function for HuggingFace's different parameters
+
+        HuggingFace estimators have 3 framework version parameters instead of one: The version for
+        Transformers, PyTorch, and TensorFlow.
+        """
+        return self.estimator_cls(
+            transformers_version=self.framework_version,
+            tensorflow_version=self.tensorflow_version,
+            pytorch_version=self.pytorch_version,
+            py_version=self.py_version,
+            entry_point=entry_point,
+            source_dir=source_dir,
+            dependencies=dependencies,
+            git_config=git_config,
+            code_location=self.code_location,
+            enable_network_isolation=False,
+            image_uri=self.image_uri,
+            role=self.role,
+            instance_count=self.instance_count,
+            instance_type=self.instance_type,
+            sagemaker_session=self.sagemaker_session,
+            debugger_hook_config=False,
+            disable_profiler=True,
+        )

src/sagemaker/image_uri_config/xgboost.json

@@ -183,6 +183,35 @@
                 "us-west-2": "246618743249"
             },
             "repository": "sagemaker-xgboost"
+        },
+        "1.3-1": {
+            "registries": {
+                "af-south-1": "510948584623",
+                "ap-east-1": "651117190479",
+                "ap-northeast-1": "354813040037",
+                "ap-northeast-2": "366743142698",
+                "ap-south-1": "720646828776",
+                "ap-southeast-1": "121021644041",
+                "ap-southeast-2": "783357654285",
+                "ca-central-1": "341280168497",
+                "cn-north-1": "450853457545",
+                "cn-northwest-1": "451049120500",
+                "eu-central-1": "492215442770",
+                "eu-north-1": "662702820516",
+                "eu-west-1": "141502667606",
+                "eu-west-2": "764974769150",
+                "eu-west-3": "659782779980",
+                "eu-south-1": "978288397137",
+                "me-south-1": "801668240914",
+                "sa-east-1": "737474898029",
+                "us-east-1": "683313688378",
+                "us-east-2": "257758044811",
+                "us-gov-west-1": "414596584902",
+                "us-iso-east-1": "833128469047",
+                "us-west-1": "746614075791",
+                "us-west-2": "246618743249"
+            },
+            "repository": "sagemaker-xgboost"
         }
     }
 }

src/sagemaker/local/local_session.py

@@ -475,10 +475,30 @@ def invoke_endpoint(
 
 
 class LocalSession(Session):
-    """A LocalSession class definition."""
+    """A SageMaker ``Session`` class for Local Mode.
 
-    def __init__(self, boto_session=None, s3_endpoint_url=None):
+    This class provides alternative Local Mode implementations for the functionality of
+    :class:`~sagemaker.session.Session`.
+    """
+
+    def __init__(self, boto_session=None, s3_endpoint_url=None, disable_local_code=False):
+        """Create a Local SageMaker Session.
+
+        Args:
+            boto_session (boto3.session.Session): The underlying Boto3 session which AWS service
+                calls are delegated to (default: None). If not provided, one is created with
+                default AWS configuration chain.
+            s3_endpoint_url (str): Override the default endpoint URL for Amazon S3, if set
+                (default: None).
+            disable_local_code (bool): Set ``True`` to override the default AWS configuration
+                chain to disable the ``local.local_code`` setting, which may not be supported for
+                some SDK features (default: False).
+        """
         self.s3_endpoint_url = s3_endpoint_url
+        # We use this local variable to avoid disrupting the __init__->_initialize API of the
+        # parent class... But overwriting it after constructor won't do anything, so prefix _ to
+        # discourage external use:
+        self._disable_local_code = disable_local_code
 
         super(LocalSession, self).__init__(boto_session)
 
@@ -530,6 +550,8 @@ def _initialize(
                 raise e
 
             self.config = yaml.load(open(sagemaker_config_file, "r"))
+            if self._disable_local_code and "local" in self.config:
+                self.config["local"]["local_code"] = False
 
     def logs_for_job(self, job_name, wait=False, poll=5, log_type="All"):
         """A no-op method meant to override the sagemaker client.

src/sagemaker/mxnet/__init__.py

@@ -10,8 +10,9 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-"""Placeholder docstring"""
+"""Classes for using MXNet with Amazon SageMaker."""
 from __future__ import absolute_import  # noqa: F401
 
 from sagemaker.mxnet.estimator import MXNet  # noqa: F401
 from sagemaker.mxnet.model import MXNetModel, MXNetPredictor  # noqa: F401
+from sagemaker.mxnet.processing import MXNetProcessor  # noqa: F401

src/sagemaker/mxnet/processing.py

@@ -29,15 +29,16 @@ class MXNetProcessor(FrameworkProcessor):
     def __init__(
         self,
         framework_version,  # New arg
-        s3_prefix,  # New arg
         role,
         instance_count,
         instance_type,
         py_version="py3",  # New kwarg
         image_uri=None,
+        command=None,
         volume_size_in_gb=30,
         volume_kms_key=None,
         output_kms_key=None,
+        code_location=None,  # New arg
         max_runtime_in_seconds=None,
         base_job_name=None,
         sagemaker_session=None,
@@ -49,27 +50,28 @@ def __init__(
 
         Unless ``image_uri`` is specified, the MXNet environment is an
         Amazon-built Docker container that executes functions defined in the supplied
-        ``entry_point`` Python script.
+        ``code`` Python script.
 
         The arguments have the exact same meaning as in ``FrameworkProcessor``.
 
         .. tip::
 
             You can find additional parameters for initializing this class at
-            :class:`~smallmatter.ds.FrameworkProcessor`.
+            :class:`~sagemaker.processing.FrameworkProcessor`.
         """
         super().__init__(
             self.estimator_cls,
             framework_version,
-            s3_prefix,
             role,
             instance_count,
             instance_type,
             py_version,
             image_uri,
+            command,
             volume_size_in_gb,
             volume_kms_key,
             output_kms_key,
+            code_location,
             max_runtime_in_seconds,
             base_job_name,
             sagemaker_session,