Merge branch 'zwei' into deploy-new-resources

Author: laurenyu

doc/overview.rst

@@ -116,7 +116,7 @@ Here is an end to end example of how to use a SageMaker Estimator:
     # Deletes the SageMaker model
     mxnet_predictor.delete_model()
 
-The example above will eventually delete both the SageMaker endpoint and endpoint configuration through `delete_endpoint()`. If you want to keep your SageMaker endpoint configuration, use the value False for the `delete_endpoint_config` parameter, as shown below.
+The example above will eventually delete both the SageMaker endpoint and endpoint configuration through ``delete_endpoint()``. If you want to keep your SageMaker endpoint configuration, use the value ``False`` for the ``delete_endpoint_config`` parameter, as shown below.
 
 .. code:: python
 
@@ -180,10 +180,10 @@ Here is an example:
         train_input = algo.sagemaker_session.upload_data(path='/path/to/your/data')
 
         algo.fit({'training': train_input})
-        algo.deploy(1, 'ml.m4.xlarge')
+        predictor = algo.deploy(1, 'ml.m4.xlarge')
 
         # When you are done using your endpoint
-        algo.delete_endpoint()
+        predictor.delete_endpoint()
 
 Use Scripts Stored in a Git Repository
 --------------------------------------
@@ -609,7 +609,7 @@ Here is a basic example of how to use it:
     response = my_predictor.predict(my_prediction_data)
 
     # Tear down the SageMaker endpoint
-    my_tuner.delete_endpoint()
+    my_predictor.delete_endpoint()
 
 This example shows a hyperparameter tuning job that creates up to 100 training jobs, running up to 10 training jobs at a time.
 Each training job's learning rate is a value between 0.05 and 0.06, but this value will differ between training jobs.

src/sagemaker/chainer/estimator.py

@@ -273,12 +273,10 @@ class constructor
             init_params["image_name"] = image_name
             return init_params
 
-        training_job_name = init_params["base_job_name"]
-
         if framework != cls.__framework_name__:
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )
         return init_params

src/sagemaker/cli/compatibility/v2/ast_transformer.py

@@ -18,6 +18,7 @@
 from sagemaker.cli.compatibility.v2 import modifiers
 
 FUNCTION_CALL_MODIFIERS = [
+    modifiers.predictors.PredictorConstructorRefactor(),
     modifiers.framework_version.FrameworkVersionEnforcer(),
     modifiers.tf_legacy_mode.TensorFlowLegacyModeConstructorUpgrader(),
     modifiers.tf_legacy_mode.TensorBoardParameterRemover(),
@@ -28,7 +29,10 @@
 
 IMPORT_MODIFIERS = [modifiers.tfs.TensorFlowServingImportRenamer()]
 
-IMPORT_FROM_MODIFIERS = [modifiers.tfs.TensorFlowServingImportFromRenamer()]
+IMPORT_FROM_MODIFIERS = [
+    modifiers.predictors.PredictorImportFromRenamer(),
+    modifiers.tfs.TensorFlowServingImportFromRenamer(),
+]
 
 
 class ASTTransformer(ast.NodeTransformer):

src/sagemaker/cli/compatibility/v2/modifiers/__init__.py

@@ -17,6 +17,7 @@
     airflow,
     deprecated_params,
     framework_version,
+    predictors,
     tf_legacy_mode,
     tfs,
 )

src/sagemaker/cli/compatibility/v2/modifiers/predictors.py

@@ -0,0 +1,146 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Classes to modify Predictor code to be compatible
+with version 2.0 and later of the SageMaker Python SDK.
+"""
+from __future__ import absolute_import
+
+import ast
+
+from sagemaker.cli.compatibility.v2.modifiers.modifier import Modifier
+
+BASE_PREDICTOR = "RealTimePredictor"
+PREDICTORS = {
+    "FactorizationMachinesPredictor": ("sagemaker", "sagemaker.amazon.factorization_machines"),
+    "IPInsightsPredictor": ("sagemaker", "sagemaker.amazon.ipinsights"),
+    "KMeansPredictor": ("sagemaker", "sagemaker.amazon.kmeans"),
+    "KNNPredictor": ("sagemaker", "sagemaker.amazon.knn"),
+    "LDAPredictor": ("sagemaker", "sagemaker.amazon.lda"),
+    "LinearLearnerPredictor": ("sagemaker", "sagemaker.amazon.linear_learner"),
+    "NTMPredictor": ("sagemaker", "sagemaker.amazon.ntm"),
+    "PCAPredictor": ("sagemaker", "sagemaker.amazon.pca"),
+    "RandomCutForestPredictor": ("sagemaker", "sagemaker.amazon.randomcutforest"),
+    "RealTimePredictor": ("sagemaker", "sagemaker.predictor"),
+    "SparkMLPredictor": ("sagemaker.sparkml", "sagemaker.sparkml.model"),
+}
+
+
+class PredictorConstructorRefactor(Modifier):
+    """A class to refactor *Predictor class and refactor endpoint attribute."""
+
+    def node_should_be_modified(self, node):
+        """Checks if the ``ast.Call`` node instantiates a class of interest.
+
+        This looks for the following calls:
+
+        - ``sagemaker.<my>.<namespace>.<MyPredictor>``
+        - ``sagemaker.<namespace>.<MyPredictor>``
+        - ``<MyPredictor>``
+
+        Args:
+            node (ast.Call): a node that represents a function call. For more,
+                see https://docs.python.org/3/library/ast.html#abstract-grammar.
+
+        Returns:
+            bool: If the ``ast.Call`` instantiates a class of interest.
+        """
+        return any(_matching(node, name, namespaces) for name, namespaces in PREDICTORS.items())
+
+    def modify_node(self, node):
+        """Modifies the ``ast.Call`` node to call ``Predictor`` instead.
+
+        Also renames ``endpoint`` attribute to ``endpoint_name``.
+
+        Args:
+            node (ast.Call): a node that represents a *Predictor constructor.
+        """
+        _rename_class(node)
+        _rename_endpoint(node)
+
+
+def _matching(node, name, namespaces):
+    """Determines if the node matches the constructor name in the right namespace"""
+    if _matching_name(node, name):
+        return True
+
+    if not _matching_attr(node, name):
+        return False
+
+    return any(_matching_namespace(node, namespace) for namespace in namespaces)
+
+
+def _matching_name(node, name):
+    """Determines if the node is an ast.Name node with a matching name"""
+    return isinstance(node.func, ast.Name) and node.func.id == name
+
+
+def _matching_attr(node, name):
+    """Determines if the node is an ast.Attribute node with a matching name"""
+    return isinstance(node.func, ast.Attribute) and node.func.attr == name
+
+
+def _matching_namespace(node, namespace):
+    """Determines if the node corresponds to a matching namespace"""
+    names = namespace.split(".")
+    name, value = names.pop(), node.func.value
+    while isinstance(value, ast.Attribute) and len(names) > 0:
+        if value.attr != name:
+            return False
+        name, value = names.pop(), value.value
+
+    return isinstance(value, ast.Name) and value.id == name
+
+
+def _rename_class(node):
+    """Renames the RealTimePredictor base class to Predictor"""
+    if _matching_name(node, BASE_PREDICTOR):
+        node.func.id = "Predictor"
+    elif _matching_attr(node, BASE_PREDICTOR):
+        node.func.attr = "Predictor"
+
+
+def _rename_endpoint(node):
+    """Renames keyword endpoint argument to endpoint_name"""
+    for keyword in node.keywords:
+        if keyword.arg == "endpoint":
+            keyword.arg = "endpoint_name"
+            break
+
+
+class PredictorImportFromRenamer(Modifier):
+    """A class to update import statements of ``RealTimePredictor``."""
+
+    def node_should_be_modified(self, node):
+        """Checks if the import statement imports ``RealTimePredictor`` from the correct module.
+
+        Args:
+            node (ast.ImportFrom): a node that represents a ``from ... import ... `` statement.
+                For more, see https://docs.python.org/3/library/ast.html#abstract-grammar.
+
+        Returns:
+            bool: If the import statement imports ``RealTimePredictor`` from the correct module.
+        """
+        return node.module in PREDICTORS[BASE_PREDICTOR] and any(
+            name.name == BASE_PREDICTOR for name in node.names
+        )
+
+    def modify_node(self, node):
+        """Changes the ``ast.ImportFrom`` node's name from ``RealTimePredictor`` to ``Predictor``.
+
+        Args:
+            node (ast.ImportFrom): a node that represents a ``from ... import ... `` statement.
+                For more, see https://docs.python.org/3/library/ast.html#abstract-grammar.
+        """
+        for name in node.names:
+            if name.name == BASE_PREDICTOR:
+                name.name = "Predictor"

src/sagemaker/estimator.py

@@ -56,7 +56,7 @@
 from sagemaker.session import Session
 from sagemaker.session import s3_input
 from sagemaker.transformer import Transformer
-from sagemaker.utils import base_name_from_image, name_from_base, get_config_value
+from sagemaker.utils import base_from_name, base_name_from_image, name_from_base, get_config_value
 from sagemaker import vpc_utils
 
 
@@ -627,7 +627,7 @@ def attach(cls, training_job_name, sagemaker_session=None, model_channel_name="m
 
         estimator = cls(sagemaker_session=sagemaker_session, **init_params)
         estimator.latest_training_job = _TrainingJob(
-            sagemaker_session=sagemaker_session, job_name=init_params["base_job_name"]
+            sagemaker_session=sagemaker_session, job_name=training_job_name
         )
         estimator._current_job_name = estimator.latest_training_job.name
         estimator.latest_training_job.wait()
@@ -791,7 +791,7 @@ class constructor
         init_params["train_volume_size"] = job_details["ResourceConfig"]["VolumeSizeInGB"]
         init_params["train_max_run"] = job_details["StoppingCondition"]["MaxRuntimeInSeconds"]
         init_params["input_mode"] = job_details["AlgorithmSpecification"]["TrainingInputMode"]
-        init_params["base_job_name"] = job_details["TrainingJobName"]
+        init_params["base_job_name"] = base_from_name(job_details["TrainingJobName"])
         init_params["output_path"] = job_details["OutputDataConfig"]["S3OutputPath"]
         init_params["output_kms_key"] = job_details["OutputDataConfig"]["KmsKeyId"]
         if "EnableNetworkIsolation" in job_details:
@@ -835,15 +835,6 @@ class constructor
 
         return init_params
 
-    def delete_endpoint(self):
-        """Delete an Amazon SageMaker ``Endpoint``.
-
-        Raises:
-            botocore.exceptions.ClientError: If the endpoint does not exist.
-        """
-        self._ensure_latest_training_job(error_message="Endpoint was not created yet")
-        self.sagemaker_session.delete_endpoint(self.latest_training_job.name)
-
     def transformer(
         self,
         instance_count,

src/sagemaker/mxnet/estimator.py

@@ -276,12 +276,10 @@ class constructor
             init_params["image_name"] = image_name
             return init_params
 
-        training_job_name = init_params["base_job_name"]
-
         if framework != cls.__framework_name__:
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )
 

src/sagemaker/pytorch/estimator.py

@@ -222,12 +222,10 @@ class constructor
             init_params["image_name"] = image_name
             return init_params
 
-        training_job_name = init_params["base_job_name"]
-
         if framework != cls.__framework_name__:
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )
 

src/sagemaker/rl/estimator.py

@@ -315,10 +315,9 @@ class constructor
         toolkit, toolkit_version = cls._toolkit_and_version_from_tag(tag)
 
         if not cls._is_combination_supported(toolkit, toolkit_version, framework):
-            training_job_name = init_params["base_job_name"]
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )
 

src/sagemaker/sklearn/estimator.py

@@ -244,12 +244,10 @@ class constructor
             init_params["image_name"] = image_name
             return init_params
 
-        training_job_name = init_params["base_job_name"]
-
         if framework and framework != cls.__framework_name__:
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )
 

src/sagemaker/tensorflow/estimator.py

@@ -221,11 +221,10 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
         if not script_mode:
             init_params["image_name"] = image_name
 
-        training_job_name = init_params["base_job_name"]
         if framework != cls.__framework_name__:
             raise ValueError(
                 "Training job: {} didn't use image for requested framework".format(
-                    training_job_name
+                    job_details["TrainingJobName"]
                 )
             )