breaking: create new inference resources during estimator.deploy() or estimator.transformer()

Author: laurenyu

src/sagemaker/chainer/estimator.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from sagemaker import utils
 from sagemaker.estimator import Framework
 from sagemaker.fw_utils import (
     framework_name_from_image,
@@ -210,7 +211,8 @@ def create_model(
             kwargs["image"] = self.image_name
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         return ChainerModel(
             self.model_data,

src/sagemaker/estimator.py

@@ -140,7 +140,7 @@ def __init__(
                 training output (default: None).
             base_job_name (str): Prefix for training job name when the
                 :meth:`~sagemaker.estimator.EstimatorBase.fit` method launches.
-                If not specified, the estimator generates a default job name,
+                If not specified, the estimator generates a default job name
                 based on the training image name and current timestamp.
             sagemaker_session (sagemaker.session.Session): Session object which
                 manages interactions with Amazon SageMaker APIs and any other
@@ -328,6 +328,15 @@ def prepare_workflow_for_training(self, job_name=None):
         """
         self._prepare_for_training(job_name=job_name)
 
+    def _ensure_base_job_name(self):
+        # honor supplied base_job_name or generate it
+        if self.base_job_name:
+            return
+        elif isinstance(self, sagemaker.algorithm.AlgorithmEstimator):
+            self.base_job_name = self.algorithm_arn.split("/")[-1]  # pylint: disable=no-member
+        else:
+            self.base_job_name = base_name_from_image(self.train_image())
+
     def _prepare_for_training(self, job_name=None):
         """Set any values in the estimator that need to be set before training.
 
@@ -339,15 +348,8 @@ def _prepare_for_training(self, job_name=None):
         if job_name is not None:
             self._current_job_name = job_name
         else:
-            # honor supplied base_job_name or generate it
-            if self.base_job_name:
-                base_name = self.base_job_name
-            elif isinstance(self, sagemaker.algorithm.AlgorithmEstimator):
-                base_name = self.algorithm_arn.split("/")[-1]  # pylint: disable=no-member
-            else:
-                base_name = base_name_from_image(self.train_image())
-
-            self._current_job_name = name_from_base(base_name)
+            self._ensure_base_job_name()
+            self._current_job_name = name_from_base(self.base_job_name)
 
         # if output_path was specified we use it otherwise initialize here.
         # For Local Mode with local_code=True we don't need an explicit output_path
@@ -483,7 +485,7 @@ def fit(self, inputs=None, wait=True, logs="All", job_name=None, experiment_conf
                 compatibility, boolean values are also accepted and converted to strings.
                 Only meaningful when wait is True.
             job_name (str): Training job name. If not specified, the estimator generates
-                a default job name, based on the training image name and current timestamp.
+                a default job name based on the training image name and current timestamp.
             experiment_config (dict[str, str]): Experiment management configuration.
                 Dictionary contains three optional keys,
                 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
@@ -667,7 +669,8 @@ def deploy(
             wait (bool): Whether the call should wait until the deployment of
                 model completes (default: True).
             model_name (str): Name to use for creating an Amazon SageMaker
-                model. If not specified, the name of the training job is used.
+                model. If not specified, the estimator generates a default job name
+                based on the training image name and current timestamp.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 data on the storage volume attached to the instance hosting the
                 endpoint.
@@ -691,8 +694,11 @@ def deploy(
                 endpoint and obtain inferences.
         """
         self._ensure_latest_training_job()
-        endpoint_name = endpoint_name or self.latest_training_job.name
-        model_name = model_name or self.latest_training_job.name
+        self._ensure_base_job_name()
+        default_name = name_from_base(self.base_job_name)
+        endpoint_name = endpoint_name or default_name
+        model_name = model_name or default_name
+
         self.deploy_instance_type = instance_type
         if use_compiled_model:
             family = "_".join(instance_type.split(".")[:-1])
@@ -898,18 +904,20 @@ def transformer(
                 If not specified, this setting is taken from the estimator's
                 current configuration.
             model_name (str): Name to use for creating an Amazon SageMaker
-                model. If not specified, the name of the training job is used.
+                model. If not specified, the estimator generates a default job name
+                based on the training image name and current timestamp.
         """
         tags = tags or self.tags
 
+        self._ensure_base_job_name()
+        model_name = model_name or name_from_base(self.base_job_name)
+
         if self.latest_training_job is None:
             logging.warning(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )
-            model_name = model_name or self._current_job_name
         else:
-            model_name = model_name or self.latest_training_job.name
             if enable_network_isolation is None:
                 enable_network_isolation = self.enable_network_isolation()
 
@@ -1993,7 +2001,8 @@ def transformer(
                 If not specified, this setting is taken from the estimator's
                 current configuration.
             model_name (str): Name to use for creating an Amazon SageMaker
-                model. If not specified, the name of the training job is used.
+                model. If not specified, the estimator generates a default job name
+                based on the training image name and current timestamp.
 
         Returns:
             sagemaker.transformer.Transformer: a ``Transformer`` object that can be used to start a
@@ -2002,6 +2011,9 @@ def transformer(
         role = role or self.role
         tags = tags or self.tags
 
+        self._ensure_base_job_name()
+        model_name = model_name or name_from_base(self.base_job_name)
+
         if self.latest_training_job is not None:
             if enable_network_isolation is None:
                 enable_network_isolation = self.enable_network_isolation()
@@ -2017,7 +2029,6 @@ def transformer(
             )
             model._create_sagemaker_model(instance_type, tags=tags)
 
-            model_name = model.name
             transform_env = model.env.copy()
             if env is not None:
                 transform_env.update(env)
@@ -2026,7 +2037,6 @@ def transformer(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )
-            model_name = model_name or self._current_job_name
             transform_env = env or {}
 
         return Transformer(

src/sagemaker/mxnet/estimator.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from sagemaker import utils
 from sagemaker.estimator import Framework
 from sagemaker.fw_utils import (
     framework_name_from_image,
@@ -219,7 +220,8 @@ def create_model(
             kwargs["image"] = image_name or self.image_name
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         return MXNetModel(
             self.model_data,

src/sagemaker/pytorch/estimator.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from sagemaker import utils
 from sagemaker.estimator import Framework
 from sagemaker.fw_utils import (
     framework_name_from_image,
@@ -171,7 +172,8 @@ def create_model(
             kwargs["image"] = self.image_name
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         return PyTorchModel(
             self.model_data,

src/sagemaker/rl/estimator.py

@@ -17,8 +17,8 @@
 import logging
 import re
 
+from sagemaker import fw_utils, utils
 from sagemaker.estimator import Framework
-import sagemaker.fw_utils as fw_utils
 from sagemaker.model import FrameworkModel, SAGEMAKER_OUTPUT_LOCATION
 from sagemaker.mxnet.model import MXNetModel
 from sagemaker.tensorflow.model import TensorFlowModel
@@ -218,11 +218,13 @@ def create_model(
         Raises:
             ValueError: If image_name is not specified and framework enum is not valid.
         """
+        self._ensure_base_job_name()
+
         base_args = dict(
             model_data=self.model_data,
             role=role or self.role,
             image=kwargs.get("image", self.image_name),
-            name=kwargs.get("name", self._current_job_name),
+            name=kwargs.get("name", utils.name_from_base(self.base_job_name)),
             container_log_level=self.container_log_level,
             sagemaker_session=self.sagemaker_session,
             vpc_config=self.get_vpc_config(vpc_config_override),

src/sagemaker/sklearn/estimator.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from sagemaker import utils
 from sagemaker.estimator import Framework
 from sagemaker.fw_registry import default_framework_uri
 from sagemaker.fw_utils import (
@@ -194,7 +195,8 @@ def create_model(
             kwargs["enable_network_isolation"] = self.enable_network_isolation()
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         return SKLearnModel(
             self.model_data,

src/sagemaker/tensorflow/estimator.py

@@ -274,7 +274,8 @@ def create_model(
             kwargs["image"] = self.image_name
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         if "enable_network_isolation" not in kwargs:
             kwargs["enable_network_isolation"] = self.enable_network_isolation()
@@ -441,17 +442,21 @@ def transformer(
                 If not specified, this setting is taken from the estimator's
                 current configuration.
             model_name (str): Name to use for creating an Amazon SageMaker
-                model. If not specified, the name of the training job is used.
+                model. If not specified, the estimator generates a default job name
+                based on the training image name and current timestamp.
         """
         role = role or self.role
 
+        self._ensure_base_job_name()
+        model_name = model_name or utils.name_from_base(self.base_job_name)
+
         if self.latest_training_job is None:
             logging.warning(
                 "No finished training job found associated with this estimator. Please make sure "
                 "this estimator is only used for building workflow config"
             )
             return Transformer(
-                model_name or self._current_job_name,
+                model_name,
                 instance_count,
                 instance_type,
                 strategy=strategy,

src/sagemaker/xgboost/estimator.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from sagemaker import utils
 from sagemaker.estimator import Framework, _TrainingJob
 from sagemaker.fw_registry import default_framework_uri
 from sagemaker.fw_utils import (
@@ -169,7 +170,8 @@ def create_model(
             kwargs["image"] = self.image_name
 
         if "name" not in kwargs:
-            kwargs["name"] = self._current_job_name
+            self._ensure_base_job_name()
+            kwargs["name"] = utils.name_from_base(self.base_job_name)
 
         return XGBoostModel(
             self.model_data,

tests/integ/test_mxnet_train.py

@@ -70,6 +70,49 @@ def test_attach_deploy(mxnet_training_job, sagemaker_session, cpu_instance_type)
         assert result is not None
 
 
+def test_deploy_estimator_with_different_instance_types(
+    mxnet_training_job,
+    sagemaker_session,
+    cpu_instance_type,
+    alternative_cpu_instance_type,
+):
+    def _deploy_estimator_and_assert_instance_type(estimator, instance_type):
+        # don't use timeout_and_delete_endpoint_by_name because this tests if
+        # deploy() creates a new endpoint config/endpoint each time
+        with timeout(minutes=45):
+            try:
+                predictor = estimator.deploy(1, instance_type)
+
+                model_name = predictor._model_names[0]
+                endpoint_name = predictor.endpoint
+                config_name = sagemaker_session.sagemaker_client.describe_endpoint(
+                    EndpointName=endpoint_name
+                )["EndpointConfigName"]
+                config = sagemaker_session.sagemaker_client.describe_endpoint_config(
+                    EndpointConfigName=config_name
+                )
+            finally:
+                predictor.delete_model()
+                predictor.delete_endpoint()
+
+        assert config["ProductionVariants"][0]["InstanceType"] == instance_type
+
+        return (model_name, endpoint_name, config_name)
+
+    estimator = MXNet.attach(mxnet_training_job, sagemaker_session)
+
+    old_model_name, old_endpoint_name, old_config_name = _deploy_estimator_and_assert_instance_type(
+        estimator, cpu_instance_type
+    )
+    new_model_name, new_endpoint_name, new_config_name = _deploy_estimator_and_assert_instance_type(
+        estimator, alternative_cpu_instance_type
+    )
+
+    assert old_model_name != new_model_name
+    assert old_endpoint_name != new_endpoint_name
+    assert old_config_name != new_config_name
+
+
 def test_deploy_model(
     mxnet_training_job,
     sagemaker_session,