Merge branch 'master' of https://github.com/aws/sagemaker-python-sdk into xgboost_get_image_uri

Author: Kim

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## v1.58.4 (2020-05-20)
+
+### Bug Fixes and Other Changes
+
+ * update AutoML default max_candidate value to use the service default
+ * add describe_transform_job in session class
+
+### Documentation Changes
+
+ * clarify support for requirements.txt in Tensorflow docs
+
+### Testing and Release Infrastructure
+
+ * wait for DisassociateTrialComponent to take effect in experiment integ test cleanup
+
 ## v1.58.3 (2020-05-19)
 
 ### Bug Fixes and Other Changes

README.rst

@@ -205,7 +205,7 @@ By using MXNet SageMaker Estimators, you can train and host MXNet models on Amaz
 
 Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.6.0``.
 
-Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``.
+Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.5.1``.
 
 We recommend that you use the latest supported version, because that's where we focus most of our development efforts.
 

VERSION

@@ -1 +1 @@
-1.58.4.dev0
+1.58.5.dev0

doc/overview.rst

@@ -732,6 +732,26 @@ You can install all necessary for this feature dependencies using pip:
 
     pip install 'sagemaker[local]' --upgrade
 
+If you want to keep everything local, and not use Amazon S3 either, you can enable "local code" in one of two ways:
+
+- Create a file at ``~/.sagemaker/config.yaml`` that contains:
+
+.. code:: yaml
+
+    local:
+      local_code: true
+
+- Create a ``LocalSession`` and configure it directly:
+
+.. code:: python
+
+    from sagemaker.local import LocalSession
+
+    sagemaker_session = LocalSession()
+    sagemaker_session.config = {'local': {'local_code': True}}
+
+    # pass sagemaker_session to your estimator or model
+
 We can take the example in  `Using Estimators <#using-estimators>`__ , and use either ``local`` or ``local_gpu`` as the instance type.
 
 .. code:: python

src/sagemaker/__init__.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+import logging
+import sys
 import importlib_metadata
 
 from sagemaker import estimator, parameter, tuner  # noqa: F401
@@ -61,3 +63,10 @@
 from sagemaker.automl.candidate_estimator import CandidateEstimator, CandidateStep  # noqa: F401
 
 __version__ = importlib_metadata.version("sagemaker")
+
+if sys.version[0] == "2":
+    logging.getLogger("sagemaker").warning(
+        "SageMaker Python SDK v2 will no longer support Python 2. "
+        "Please see https://github.com/aws/sagemaker-python-sdk/issues/1459 "
+        "for more information"
+    )

src/sagemaker/amazon/amazon_estimator.py

@@ -616,6 +616,11 @@ def get_image_uri(region_name, repo_name, repo_version=1):
         repo_name:
         repo_version:
     """
+    logger.warning(
+        "'get_image_uri' method will be deprecated in favor of 'ImageURIProvider' class "
+        "in SageMaker Python SDK v2."
+    )
+
     repo_version = str(repo_version)
 
     if repo_name == XGBOOST_NAME:

src/sagemaker/estimator.py

@@ -39,6 +39,7 @@
     UploadedCode,
     validate_source_dir,
     _region_supports_debugger,
+    parameter_v2_rename_warning,
 )
 from sagemaker.job import _Job
 from sagemaker.local import LocalSession
@@ -1273,6 +1274,7 @@ def __init__(
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
                 (default: ``None``).
         """
+        logging.warning(parameter_v2_rename_warning("image_name", "image_uri"))
         self.image_name = image_name
         self.hyperparam_dict = hyperparameters.copy() if hyperparameters else {}
         super(Estimator, self).__init__(
@@ -1637,6 +1639,8 @@ def __init__(
         self.container_log_level = container_log_level
         self.code_location = code_location
         self.image_name = image_name
+        if image_name is not None:
+            logging.warning(parameter_v2_rename_warning("image_name", "image_uri"))
 
         self.uploaded_code = None
 
@@ -1722,11 +1726,14 @@ def _stage_user_code_in_s3(self):
             code_bucket = self.sagemaker_session.default_bucket()
             code_s3_prefix = "{}/{}".format(self._current_job_name, "source")
             kms_key = None
-
         elif self.code_location is None:
             code_bucket, _ = parse_s3_url(self.output_path)
             code_s3_prefix = "{}/{}".format(self._current_job_name, "source")
             kms_key = self.output_kms_key
+        elif local_mode:
+            code_bucket, key_prefix = parse_s3_url(self.code_location)
+            code_s3_prefix = "/".join(filter(None, [key_prefix, self._current_job_name, "source"]))
+            kms_key = None
         else:
             code_bucket, key_prefix = parse_s3_url(self.code_location)
             code_s3_prefix = "/".join(filter(None, [key_prefix, self._current_job_name, "source"]))

src/sagemaker/fw_utils.py

@@ -34,7 +34,10 @@
 instantiated with positional or keyword arguments.
 """
 
-EMPTY_FRAMEWORK_VERSION_WARNING = "No framework_version specified, defaulting to version {}."
+EMPTY_FRAMEWORK_VERSION_WARNING = (
+    "No framework_version specified, defaulting to version {}. "
+    "framework_version will be required in SageMaker Python SDK v2."
+)
 LATER_FRAMEWORK_VERSION_WARNING = (
     "This is not the latest supported version. "
     "If you would like to use version {latest}, "
@@ -52,6 +55,10 @@
     "fully leverage all GPU cores; the parameter server will be configured to run "
     "only one worker per host regardless of the number of GPUs."
 )
+PARAMETER_V2_RENAME_WARNING = (
+    "Parameter {v1_parameter_name} will be renamed to {v2_parameter_name} "
+    "in SageMaker Python SDK v2."
+)
 
 
 EMPTY_FRAMEWORK_VERSION_ERROR = (
@@ -253,6 +260,11 @@ def create_image_uri(
     Returns:
         str: The appropriate image URI based on the given parameters.
     """
+    logger.warning(
+        "'create_image_uri' will be deprecated in favor of 'ImageURIProvider' class "
+        "in SageMaker Python SDK v2."
+    )
+
     optimized_families = optimized_families or []
 
     if py_version and py_version not in VALID_PY_VERSIONS:
@@ -647,6 +659,17 @@ def python_deprecation_warning(framework, latest_supported_version):
     )
 
 
+def parameter_v2_rename_warning(v1_parameter_name, v2_parameter_name):
+    """
+    Args:
+        v1_parameter_name: parameter name used in SageMaker Python SDK v1
+        v2_parameter_name: parameter name used in SageMaker Python SDK v2
+    """
+    return PARAMETER_V2_RENAME_WARNING.format(
+        v1_parameter_name=v1_parameter_name, v2_parameter_name=v2_parameter_name
+    )
+
+
 def _region_supports_debugger(region_name):
     """Returns boolean indicating whether the region supports Amazon SageMaker Debugger.
 

src/sagemaker/inputs.py

@@ -13,9 +13,13 @@
 """Amazon SageMaker channel configurations for S3 data sources and file system data sources"""
 from __future__ import absolute_import, print_function
 
+import logging
+
 FILE_SYSTEM_TYPES = ["FSxLustre", "EFS"]
 FILE_SYSTEM_ACCESS_MODES = ["ro", "rw"]
 
+logger = logging.getLogger("sagemaker")
+
 
 class s3_input(object):
     """Amazon SageMaker channel configurations for S3 data sources.
@@ -76,6 +80,9 @@ def __init__(
                 this channel. See the SageMaker API documentation for more info:
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_ShuffleConfig.html
         """
+        logger.warning(
+            "'s3_input' class will be renamed to 'TrainingInput' in SageMaker Python SDK v2."
+        )
 
         self.config = {
             "DataSource": {"S3DataSource": {"S3DataType": s3_data_type, "S3Uri": s3_data}}

src/sagemaker/model.py

@@ -108,6 +108,8 @@ def __init__(
             model_kms_key (str): KMS key ARN used to encrypt the repacked
                 model archive file if the model is repacked
         """
+        LOGGER.warning(fw_utils.parameter_v2_rename_warning("image", "image_uri"))
+
         self.model_data = model_data
         self.image = image
         self.role = role

src/sagemaker/mxnet/README.rst

@@ -6,7 +6,7 @@ With the SageMaker Python SDK, you can train and host MXNet models on Amazon Sag
 
 Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.6.0``.
 
-Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``.
+Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.5.1``.
 
 Supported versions of MXNet for Inferentia: ``1.5.1``.
 

src/sagemaker/mxnet/estimator.py

@@ -21,6 +21,7 @@
     framework_version_from_tag,
     empty_framework_version_warning,
     python_deprecation_warning,
+    parameter_v2_rename_warning,
     is_version_equal_or_higher,
     warn_if_parameter_server_with_multi_gpu,
 )
@@ -129,6 +130,7 @@ def __init__(
             )
 
         if distributions is not None:
+            logger.warning(parameter_v2_rename_warning("distributions", "distribution"))
             train_instance_type = kwargs.get("train_instance_type")
             warn_if_parameter_server_with_multi_gpu(
                 training_instance_type=train_instance_type, distributions=distributions

src/sagemaker/s3.py

@@ -13,11 +13,27 @@
 """This module contains Enums and helper methods related to S3."""
 from __future__ import print_function, absolute_import
 
+import logging
 import os
 
 from six.moves.urllib.parse import urlparse
 from sagemaker.session import Session
 
+logger = logging.getLogger("sagemaker")
+
+SESSION_V2_RENAME_MESSAGE = (
+    "Parameter 'session' will be renamed to 'sagemaker_session' in SageMaker Python SDK v2."
+)
+
+
+def _session_v2_rename_warning(session):
+    """
+    Args:
+        session (sagemaker.session.Session):
+    """
+    if session is not None:
+        logger.warning(SESSION_V2_RENAME_MESSAGE)
+
 
 def parse_s3_url(url):
     """Returns an (s3 bucket, key name/prefix) tuple from a url with an s3
@@ -54,6 +70,9 @@ def upload(local_path, desired_s3_uri, kms_key=None, session=None):
             The S3 uri of the uploaded file(s).
 
         """
+        if session is not None:
+            _session_v2_rename_warning(session)
+
         sagemaker_session = session or Session()
         bucket, key_prefix = parse_s3_url(url=desired_s3_uri)
         if kms_key is not None:
@@ -80,6 +99,9 @@ def upload_string_as_file_body(body, desired_s3_uri=None, kms_key=None, session=
             str: The S3 uri of the uploaded file(s).
 
         """
+        if session is not None:
+            _session_v2_rename_warning(session)
+
         sagemaker_session = session or Session()
         bucket, key = parse_s3_url(desired_s3_uri)
 
@@ -107,6 +129,9 @@ def download(s3_uri, local_path, kms_key=None, session=None):
                 using the default AWS configuration chain.
 
         """
+        if session is not None:
+            _session_v2_rename_warning(session)
+
         sagemaker_session = session or Session()
         bucket, key_prefix = parse_s3_url(url=s3_uri)
         if kms_key is not None:
@@ -131,6 +156,9 @@ def read_file(s3_uri, session=None):
             str: The body of the file.
 
         """
+        if session is not None:
+            _session_v2_rename_warning(session)
+
         sagemaker_session = session or Session()
         bucket, key_prefix = parse_s3_url(url=s3_uri)
 
@@ -149,6 +177,9 @@ def list(s3_uri, session=None):
             [str]: The list of S3 URIs in the given S3 base uri.
 
         """
+        if session is not None:
+            _session_v2_rename_warning(session)
+
         sagemaker_session = session or Session()
         bucket, key_prefix = parse_s3_url(url=s3_uri)
 

src/sagemaker/session.py

@@ -181,6 +181,12 @@ def upload_data(self, path, bucket=None, key_prefix="data", extra_args=None):
                 ``s3://{bucket name}/{key_prefix}``.
         """
         # Generate a tuple for each file that we want to upload of the form (local_path, s3_key).
+        LOGGER.warning(
+            "'upload_data' method will be deprecated in favor of 'S3Uploader' class "
+            "(https://sagemaker.readthedocs.io/en/stable/s3.html#sagemaker.s3.S3Uploader) "
+            "in SageMaker Python SDK v2."
+        )
+
         files = []
         key_suffix = None
         if os.path.isdir(path):
@@ -230,6 +236,12 @@ def upload_string_as_file_body(self, body, bucket, key, kms_key=None):
             str: The S3 URI of the uploaded file.
                 The URI format is: ``s3://{bucket name}/{key}``.
         """
+        LOGGER.warning(
+            "'upload_string_as_file_body' method will be deprecated in favor of 'S3Uploader' class "
+            "(https://sagemaker.readthedocs.io/en/stable/s3.html#sagemaker.s3.S3Uploader) "
+            "in SageMaker Python SDK v2."
+        )
+
         if self.s3_resource is None:
             s3 = self.boto_session.resource("s3", region_name=self.boto_region_name)
         else:
@@ -3323,6 +3335,7 @@ def get_execution_role(sagemaker_session=None):
     Returns:
         (str): The role ARN
     """
+
     if not sagemaker_session:
         sagemaker_session = Session()
     arn = sagemaker_session.get_caller_identity_arn()

src/sagemaker/tensorflow/estimator.py

@@ -311,6 +311,7 @@ def __init__(
             )
 
         if distributions is not None:
+            logger.warning(fw.parameter_v2_rename_warning("distribution", distributions))
             train_instance_type = kwargs.get("train_instance_type")
             fw.warn_if_parameter_server_with_multi_gpu(
                 training_instance_type=train_instance_type, distributions=distributions
@@ -385,7 +386,9 @@ def _validate_args(
 
         if (not self._script_mode_enabled()) and self._only_script_mode_supported():
             logger.warning(
-                "Legacy mode is deprecated in versions 1.13 and higher. Using script mode instead."
+                "Legacy mode is deprecated in versions 1.13 and higher. Using script mode instead. "
+                "Legacy mode and its training parameters will be deprecated in "
+                "SageMaker Python SDK v2. Please use TF 1.13 or higher and script mode."
             )
             self.script_mode = True
 

src/sagemaker/transformer.py

@@ -135,6 +135,7 @@ def transform(
 
                 * 'ManifestFile' - the S3 URI points to a single manifest file listing each S3
                     object to use as an input for the transform job.
+
             content_type (str): MIME type of the input data (default: None).
             compression_type (str): Compression type of the input data, if
                 compressed (default: None). Valid values: 'Gzip', None.