Merge branch 'master' into doc-xgboost-byom

Author: laurenyu

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## v1.71.1 (2020-07-27)
+
+### Bug Fixes and Other Changes
+
+ * remove redundant information from the user_agent string.
+
+### Testing and Release Infrastructure
+
+ * use unique model name in TFS integ tests
+ * use pytest-cov instead of coverage
+
+## v1.71.0 (2020-07-23)
+
+### Features
+
+ * Add mpi support for mxnet estimator api
+
+### Bug Fixes and Other Changes
+
+ * use 'sagemaker' logger instead of root logger
+ * account for "py36" and "py37" in image tag parsing
+
 ## v1.70.2 (2020-07-22)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-1.70.3.dev0
+1.71.2.dev0

buildspec-unittests.yml

@@ -7,11 +7,11 @@ phases:
       - TOX_PARALLEL_NO_SPINNER=1
       - PY_COLORS=0
       - start_time=`date +%s`
-      - tox -e flake8,pylint,twine,black-check
+      - tox -e flake8,pylint,twine,black-check --parallel all
       - ./ci-scripts/displaytime.sh 'flake8,pylint,twine,black-check' $start_time
 
       - start_time=`date +%s`
-      - tox -e sphinx,doc8
+      - tox -e sphinx,doc8 --parallel all
       - ./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
 
       # run unit tests

doc/amazon_sagemaker_processing.rst

@@ -10,14 +10,14 @@ Amazon SageMaker Processing allows you to run steps for data pre- or post-proces
 Background
 ==========
 
-Amazon SageMaker lets developers and data scientists train and deploy machine learning models. With Amazon SageMaker Processing, you can run processing jobs on for data processing steps in your machine learning pipeline, which accept data from Amazon S3 as input, and put data into Amazon S3 as output.
+Amazon SageMaker lets developers and data scientists train and deploy machine learning models. With Amazon SageMaker Processing, you can run processing jobs for data processing steps in your machine learning pipeline. Processing jobs accept data from Amazon S3 as input and store data into Amazon S3 as output.
 
 .. image:: ./amazon_sagemaker_processing_image1.png
 
 Setup
 =====
 
-The fastest way to run get started with Amazon SageMaker Processing is by running a Jupyter notebook. You can follow the `Getting Started with Amazon SageMaker`_ guide to start running notebooks on Amazon SageMaker.
+The fastest way to get started with Amazon SageMaker Processing is by running a Jupyter notebook. You can follow the `Getting Started with Amazon SageMaker`_ guide to start running notebooks on Amazon SageMaker.
 
 .. _Getting Started with Amazon SageMaker: https://docs.aws.amazon.com/sagemaker/latest/dg/gs.html
 

src/sagemaker/amazon/amazon_estimator.py

@@ -665,7 +665,7 @@ def get_image_uri(region_name, repo_name, repo_version=1):
 
 def _warn_newer_xgboost_image():
     """Print a warning when there is a newer XGBoost image"""
-    logging.warning(
+    logger.warning(
         "There is a more up to date SageMaker XGBoost image. "
         "To use the newer image, please set 'repo_version'="
         "'%s'. For example:\n"

src/sagemaker/analytics.py

@@ -23,11 +23,12 @@
 from sagemaker.session import Session
 from sagemaker.utils import DeferredError
 
+logger = logging.getLogger(__name__)
 
 try:
     import pandas as pd
 except ImportError as e:
-    logging.warning("pandas failed to import. Analytics features will be impaired or broken.")
+    logger.warning("pandas failed to import. Analytics features will be impaired or broken.")
     # Any subsequent attempt to use pandas will raise the ImportError
     pd = DeferredError(e)
 
@@ -251,15 +252,13 @@ def training_job_summaries(self, force_refresh=False):
         output = []
         next_args = {}
         for count in range(100):
-            logging.debug("Calling list_training_jobs_for_hyper_parameter_tuning_job %d", count)
+            logger.debug("Calling list_training_jobs_for_hyper_parameter_tuning_job %d", count)
             raw_result = self._sage_client.list_training_jobs_for_hyper_parameter_tuning_job(
                 HyperParameterTuningJobName=self.name, MaxResults=100, **next_args
             )
             new_output = raw_result["TrainingJobSummaries"]
             output.extend(new_output)
-            logging.debug(
-                "Got %d more TrainingJobs. Total so far: %d", len(new_output), len(output)
-            )
+            logger.debug("Got %d more TrainingJobs. Total so far: %d", len(new_output), len(output))
             if ("NextToken" in raw_result) and (len(new_output) > 0):
                 next_args["NextToken"] = raw_result["NextToken"]
             else:
@@ -373,7 +372,7 @@ def _fetch_metric(self, metric_name):
         }
         raw_cwm_data = self._cloudwatch.get_metric_statistics(**request)["Datapoints"]
         if len(raw_cwm_data) == 0:
-            logging.warning("Warning: No metrics called %s found", metric_name)
+            logger.warning("Warning: No metrics called %s found", metric_name)
             return
 
         # Process data: normalize to starting time, and sort.

src/sagemaker/estimator.py

@@ -59,6 +59,8 @@
 from sagemaker.utils import base_name_from_image, name_from_base, get_config_value
 from sagemaker import vpc_utils
 
+logger = logging.getLogger(__name__)
+
 
 class EstimatorBase(with_metaclass(ABCMeta, object)):
     """Handle end-to-end Amazon SageMaker training and deployment tasks.
@@ -510,6 +512,10 @@ def compile_model(
         framework_version=None,
         compile_max_run=15 * 60,
         tags=None,
+        target_platform_os=None,
+        target_platform_arch=None,
+        target_platform_accelerator=None,
+        compiler_options=None,
         **kwargs
     ):
         """Compile a Neo model using the input model.
@@ -534,6 +540,21 @@ def compile_model(
             tags (list[dict]): List of tags for labeling a compilation job. For
                 more, see
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
+            target_platform_os (str): Target Platform OS, for example: 'LINUX'.
+                For allowed strings see
+                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
+                It can be used instead of target_instance_family.
+            target_platform_arch (str): Target Platform Architecture, for example: 'X86_64'.
+                For allowed strings see
+                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
+                It can be used instead of target_instance_family.
+            target_platform_accelerator (str, optional): Target Platform Accelerator,
+                for example: 'NVIDIA'. For allowed strings see
+                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
+                It can be used instead of target_instance_family.
+            compiler_options (dict, optional): Additional parameters for compiler.
+                Compiler Options are TargetPlatform / target_instance_family specific. See
+                https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html for details.
             **kwargs: Passed to invocation of ``create_model()``.
                 Implementations may customize ``create_model()`` to accept
                 ``**kwargs`` to customize model creation during deploy. For
@@ -563,6 +584,10 @@ def compile_model(
             compile_max_run,
             framework=framework,
             framework_version=framework_version,
+            target_platform_os=target_platform_os,
+            target_platform_arch=target_platform_arch,
+            target_platform_accelerator=target_platform_accelerator,
+            compiler_options=compiler_options,
         )
         return self._compiled_models[target_instance_family]
 
@@ -730,7 +755,7 @@ def model_data(self):
                 TrainingJobName=self.latest_training_job.name
             )["ModelArtifacts"]["S3ModelArtifacts"]
         else:
-            logging.warning(
+            logger.warning(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )
@@ -826,7 +851,7 @@ def delete_endpoint(self):
         Raises:
             botocore.exceptions.ClientError: If the endpoint does not exist.
         """
-        logging.warning(
+        logger.warning(
             "estimator.delete_endpoint() will be deprecated in SageMaker Python SDK v2. "
             "Please use the delete_endpoint() function on your predictor instead."
         )
@@ -908,7 +933,7 @@ def transformer(
         tags = tags or self.tags
 
         if self.latest_training_job is None:
-            logging.warning(
+            logger.warning(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )
@@ -1029,7 +1054,7 @@ def start_new(cls, estimator, inputs, experiment_config):
 
         if isinstance(inputs, s3_input):
             if "InputMode" in inputs.config:
-                logging.debug(
+                logger.debug(
                     "Selecting s3_input's input_mode (%s) for TrainingInputMode.",
                     inputs.config["InputMode"],
                 )
@@ -1279,7 +1304,7 @@ def __init__(
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
                 (default: ``None``).
         """
-        logging.warning(parameter_v2_rename_warning("image_name", "image_uri"))
+        logger.warning(parameter_v2_rename_warning("image_name", "image_uri"))
         self.image_name = image_name
         self.hyperparam_dict = hyperparameters.copy() if hyperparameters else {}
         super(Estimator, self).__init__(
@@ -1653,7 +1678,7 @@ def __init__(
         self.code_location = code_location
         self.image_name = image_name
         if image_name is not None:
-            logging.warning(parameter_v2_rename_warning("image_name", "image_uri"))
+            logger.warning(parameter_v2_rename_warning("image_name", "image_uri"))
 
         self.uploaded_code = None
 
@@ -2029,7 +2054,7 @@ def transformer(
             if env is not None:
                 transform_env.update(env)
         else:
-            logging.warning(
+            logger.warning(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )

src/sagemaker/fw_registry.py

@@ -12,10 +12,13 @@
 # language governing permissions and limitations under the License.
 """Placeholder docstring"""
 from __future__ import absolute_import
+
 import logging
 
 from sagemaker.utils import get_ecr_image_uri_prefix
 
+logger = logging.getLogger(__name__)
+
 image_registry_map = {
     "us-west-1": {
         "sparkml-serving": "746614075791",
@@ -142,7 +145,7 @@ def registry(region_name, framework=None):
         account_id = image_registry_map[region_name][framework]
         return get_ecr_image_uri_prefix(account_id, region_name)
     except KeyError:
-        logging.error("The specific image or region does not exist")
+        logger.error("The specific image or region does not exist")
         raise
 
 

src/sagemaker/fw_utils.py

@@ -493,6 +493,7 @@ def _list_files_to_compress(script, directory):
 def framework_name_from_image(image_name):
     # noinspection LongLine
     """Extract the framework and Python version from the image name.
+
     Args:
         image_name (str): Image URI, which should be one of the following forms:
             legacy:
@@ -503,25 +504,32 @@ def framework_name_from_image(image_name):
             '<account>.dkr.ecr.<region>.amazonaws.com/sagemaker-<fw>:<fw_version>-<device>-<py_ver>'
             current:
             '<account>.dkr.ecr.<region>.amazonaws.com/sagemaker-rl-<fw>:<rl_toolkit><rl_version>-<device>-<py_ver>'
+            current:
+            '<account>.dkr.ecr.<region>.amazonaws.com/<fw>-<image_scope>:<fw_version>-<device>-<py_ver>'
+
     Returns:
         tuple: A tuple containing:
-            str: The framework name str: The Python version str: The image tag
-            str: If the image is script mode
-        """
+
+            - str: The framework name
+            - str: The Python version
+            - str: The image tag
+            - str: If the TensorFlow image is script mode
+    """
     sagemaker_pattern = re.compile(ECR_URI_PATTERN)
     sagemaker_match = sagemaker_pattern.match(image_name)
     if sagemaker_match is None:
         return None, None, None, None
+
     # extract framework, python version and image tag
     # We must support both the legacy and current image name format.
     name_pattern = re.compile(
-        r"^(?:sagemaker(?:-rl)?-)?(tensorflow|mxnet|chainer|pytorch|scikit-learn|xgboost)(?:-)?(scriptmode|training)?:(.*)-(.*?)-(py2|py3)$"  # noqa: E501 # pylint: disable=line-too-long
+        r"""^(?:sagemaker(?:-rl)?-)?
+        (tensorflow|mxnet|chainer|pytorch|scikit-learn|xgboost)(?:-)?
+        (scriptmode|training)?
+        :(.*)-(.*?)-(py2|py3[67]?)$""",
+        re.VERBOSE,
     )
-    legacy_name_pattern = re.compile(r"^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$")
-
     name_match = name_pattern.match(sagemaker_match.group(9))
-    legacy_match = legacy_name_pattern.match(sagemaker_match.group(9))
-
     if name_match is not None:
         fw, scriptmode, ver, device, py = (
             name_match.group(1),
@@ -531,20 +539,25 @@ def framework_name_from_image(image_name):
             name_match.group(5),
         )
         return fw, py, "{}-{}-{}".format(ver, device, py), scriptmode
+
+    legacy_name_pattern = re.compile(r"^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$")
+    legacy_match = legacy_name_pattern.match(sagemaker_match.group(9))
     if legacy_match is not None:
         return (legacy_match.group(1), legacy_match.group(2), legacy_match.group(4), None)
     return None, None, None, None
 
 
 def framework_version_from_tag(image_tag):
     """Extract the framework version from the image tag.
+
     Args:
         image_tag (str): Image tag, which should take the form
             '<framework_version>-<device>-<py_version>'
+
     Returns:
         str: The framework version.
     """
-    tag_pattern = re.compile("^(.*)-(cpu|gpu)-(py2|py3)$")
+    tag_pattern = re.compile("^(.*)-(cpu|gpu)-(py2|py3[67]?)$")
     tag_match = tag_pattern.match(image_tag)
     return None if tag_match is None else tag_match.group(1)
 

src/sagemaker/local/entities.py

@@ -25,16 +25,15 @@
 from sagemaker.local.utils import copy_directory_structure, move_to_destination
 from sagemaker.utils import DeferredError, get_config_value
 
+logger = logging.getLogger(__name__)
+
 try:
     import urllib3
 except ImportError as e:
-    logging.warning("urllib3 failed to import. Local mode features will be impaired or broken.")
+    logger.warning("urllib3 failed to import. Local mode features will be impaired or broken.")
     # Any subsequent attempt to use urllib3 will raise the ImportError
     urllib3 = DeferredError(e)
 
-
-logger = logging.getLogger(__name__)
-
 _UNUSED_ARN = "local:arn-does-not-matter"
 HEALTH_CHECK_TIMEOUT_LIMIT = 120
 

src/sagemaker/local/image.py

@@ -473,7 +473,7 @@ def _generate_compose_file(self, command, additional_volumes=None, additional_en
         try:
             import yaml
         except ImportError as e:
-            logging.error(sagemaker.utils._module_import_error("yaml", "Local mode", "local"))
+            logger.error(sagemaker.utils._module_import_error("yaml", "Local mode", "local"))
             raise e
 
         yaml_content = yaml.dump(content, default_flow_style=False)

src/sagemaker/local/local_session.py

@@ -327,7 +327,7 @@ def __init__(self, config=None):
         try:
             import urllib3
         except ImportError as e:
-            logging.error(_module_import_error("urllib3", "Local mode", "local"))
+            logger.error(_module_import_error("urllib3", "Local mode", "local"))
             raise e
 
         self.http = urllib3.PoolManager()
@@ -423,7 +423,7 @@ def _initialize(self, boto_session, sagemaker_client, sagemaker_runtime_client):
             try:
                 import yaml
             except ImportError as e:
-                logging.error(_module_import_error("yaml", "Local mode", "local"))
+                logger.error(_module_import_error("yaml", "Local mode", "local"))
                 raise e
 
             self.config = yaml.load(open(sagemaker_config_file, "r"))