Merge branch 'zwei' into rc0-changes

Author: laurenyu

doc/frameworks/chainer/using_chainer.rst

@@ -141,6 +141,7 @@ directories ('train' and 'test').
                                 train_instance_type='ml.p3.2xlarge',
                                 train_instance_count=1,
                                 framework_version='5.0.0',
+                                py_version='py3',
                                 hyperparameters = {'epochs': 20, 'batch-size': 64, 'learning-rate': 0.1})
     chainer_estimator.fit({'train': 's3://my-data-bucket/path/to/my/training/data',
                            'test': 's3://my-data-bucket/path/to/my/test/data'})
@@ -222,7 +223,8 @@ operation.
     chainer_estimator = Chainer(entry_point='train_and_deploy.py',
                                 train_instance_type='ml.p3.2xlarge',
                                 train_instance_count=1,
-                                framework_version='5.0.0')
+                                framework_version='5.0.0',
+                                py_version='py3')
     chainer_estimator.fit('s3://my_bucket/my_training_data/')
 
     # Deploy my estimator to a SageMaker Endpoint and get a Predictor

doc/frameworks/tensorflow/upgrade_from_legacy.rst

@@ -104,6 +104,7 @@ the difference in code would be as follows:
         ...
         source_dir="code",
         framework_version="1.10.0",
+        py_version="py2",
         train_instance_type="ml.m4.xlarge",
         image_name="520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.10.0-cpu-py2",
         hyperparameters={

doc/frameworks/tensorflow/using_tf.rst

@@ -319,7 +319,7 @@ To run training job with Pipe input mode, pass in ``input_mode='Pipe'`` to your
     tf_estimator = TensorFlow(entry_point='tf-train-with-pipemodedataset.py', role='SageMakerRole',
                               training_steps=10000, evaluation_steps=100,
                               train_instance_count=1, train_instance_type='ml.p2.xlarge',
-                              framework_version='1.10.0', input_mode='Pipe')
+                              framework_version='1.10.0', py_version='py3', input_mode='Pipe')
 
     tf_estimator.fit('s3://bucket/path/to/training/data')
 
@@ -383,7 +383,8 @@ estimator object to create a SageMaker Endpoint:
   from sagemaker.tensorflow import TensorFlow
 
   estimator = TensorFlow(entry_point='tf-train.py', ..., train_instance_count=1,
-                         train_instance_type='ml.c4.xlarge', framework_version='1.11')
+                         train_instance_type='ml.c4.xlarge', framework_version='1.11',
+                         py_version='py3')
 
   estimator.fit(inputs)
 

src/sagemaker/chainer/defaults.py

@@ -13,12 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-CHAINER_VERSION = "4.1.0"
-"""Default Chainer version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "5.0.0"
-"""The latest version of Chainer included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "5.0.0"

src/sagemaker/chainer/estimator.py

@@ -19,8 +19,8 @@
 from sagemaker.fw_utils import (
     framework_name_from_image,
     framework_version_from_tag,
-    empty_framework_version_warning,
     python_deprecation_warning,
+    validate_version_or_image_args,
 )
 from sagemaker.chainer import defaults
 from sagemaker.chainer.model import ChainerModel
@@ -40,8 +40,6 @@ class Chainer(Framework):
     _process_slots_per_host = "sagemaker_process_slots_per_host"
     _additional_mpi_options = "sagemaker_additional_mpi_options"
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,
@@ -51,8 +49,8 @@ def __init__(
         additional_mpi_options=None,
         source_dir=None,
         hyperparameters=None,
-        py_version="py3",
         framework_version=None,
+        py_version=None,
         image_name=None,
         **kwargs
     ):
@@ -103,11 +101,12 @@ def __init__(
                 and values, but ``str()`` will be called to convert them before
                 training.
             py_version (str): Python version you want to use for executing your
-                model training code (default: 'py2'). One of 'py2' or 'py3'.
+                model training code. Defaults to ``None``. Required unless ``image_name``
+                is provided.
             framework_version (str): Chainer version you want to use for
-                executing your model training code. List of supported versions
+                executing your model training code. Defaults to ``None``. Required unless
+                ``image_name`` is provided. List of supported versions:
                 https://github.com/aws/sagemaker-python-sdk#chainer-sagemaker-estimators.
-                If not specified, this will default to 4.1.
             image_name (str): If specified, the estimator will use this image
                 for training and hosting, instead of selecting the appropriate
                 SageMaker official image based on framework_version and
@@ -117,6 +116,9 @@ def __init__(
                     * ``123412341234.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0``
                     * ``custom-image:latest``
 
+                If ``framework_version`` or ``py_version`` are ``None``, then
+                ``image_name`` is required. If also ``None``, then a ``ValueError``
+                will be raised.
             **kwargs: Additional kwargs passed to the
                 :class:`~sagemaker.estimator.Framework` constructor.
 
@@ -126,22 +128,18 @@ def __init__(
             :class:`~sagemaker.estimator.Framework` and
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
-        if framework_version is None:
+        validate_version_or_image_args(framework_version, py_version, image_name)
+        if py_version == "py2":
             logger.warning(
-                empty_framework_version_warning(defaults.CHAINER_VERSION, self.LATEST_VERSION)
+                python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )
-        self.framework_version = framework_version or defaults.CHAINER_VERSION
+        self.framework_version = framework_version
+        self.py_version = py_version
 
         super(Chainer, self).__init__(
             entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs
         )
 
-        if py_version == "py2":
-            logger.warning(
-                python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
-            )
-
-        self.py_version = py_version
         self.use_mpi = use_mpi
         self.num_processes = num_processes
         self.process_slots_per_host = process_slots_per_host
@@ -262,15 +260,19 @@ class constructor
         image_name = init_params.pop("image")
         framework, py_version, tag, _ = framework_name_from_image(image_name)
 
+        if tag is None:
+            framework_version = None
+        else:
+            framework_version = framework_version_from_tag(tag)
+        init_params["framework_version"] = framework_version
+        init_params["py_version"] = py_version
+
         if not framework:
             # If we were unable to parse the framework name from the image it is not one of our
             # officially supported images, in this case just add the image to the init params.
             init_params["image_name"] = image_name
             return init_params
 
-        init_params["py_version"] = py_version
-        init_params["framework_version"] = framework_version_from_tag(tag)
-
         training_job_name = init_params["base_job_name"]
 
         if framework != cls.__framework_name__:

src/sagemaker/chainer/model.py

@@ -22,7 +22,7 @@
     create_image_uri,
     model_code_key_prefix,
     python_deprecation_warning,
-    empty_framework_version_warning,
+    validate_version_or_image_args,
 )
 from sagemaker.model import FrameworkModel, MODEL_SERVER_WORKERS_PARAM_NAME
 from sagemaker.chainer import defaults
@@ -67,8 +67,8 @@ def __init__(
         role,
         entry_point,
         image=None,
-        py_version="py3",
         framework_version=None,
+        py_version=None,
         predictor_cls=ChainerPredictor,
         model_server_workers=None,
         **kwargs
@@ -88,11 +88,15 @@ def __init__(
                 hosting. If ``source_dir`` is specified, then ``entry_point``
                 must point to a file located at the root of ``source_dir``.
             image (str): A Docker image URI (default: None). If not specified, a
-                default image for Chainer will be used.
-            py_version (str): Python version you want to use for executing your
-                model training code (default: 'py2').
+                default image for Chainer will be used. If ``framework_version``
+                or ``py_version`` are ``None``, then ``image`` is required. If
+                also ``None``, then a ``ValueError`` will be raised.
             framework_version (str): Chainer version you want to use for
-                executing your model training code.
+                executing your model training code. Defaults to ``None``. Required
+                unless ``image`` is provided.
+            py_version (str): Python version you want to use for executing your
+                model training code. Defaults to ``None``. Required unless
+                ``image`` is provided.
             predictor_cls (callable[str, sagemaker.session.Session]): A function
                 to call to create a predictor with an endpoint name and
                 SageMaker ``Session``. If specified, ``deploy()`` returns the
@@ -109,21 +113,18 @@ def __init__(
             :class:`~sagemaker.model.FrameworkModel` and
             :class:`~sagemaker.model.Model`.
         """
-        super(ChainerModel, self).__init__(
-            model_data, image, role, entry_point, predictor_cls=predictor_cls, **kwargs
-        )
+        validate_version_or_image_args(framework_version, py_version, image)
         if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )
+        self.framework_version = framework_version
+        self.py_version = py_version
 
-        if framework_version is None:
-            logger.warning(
-                empty_framework_version_warning(defaults.CHAINER_VERSION, defaults.LATEST_VERSION)
-            )
+        super(ChainerModel, self).__init__(
+            model_data, image, role, entry_point, predictor_cls=predictor_cls, **kwargs
+        )
 
-        self.py_version = py_version
-        self.framework_version = framework_version or defaults.CHAINER_VERSION
         self.model_server_workers = model_server_workers
 
     def prepare_container_def(self, instance_type, accelerator_type=None):

src/sagemaker/cli/mxnet.py

@@ -14,7 +14,8 @@
 from __future__ import absolute_import
 
 from sagemaker.cli.common import HostCommand, TrainCommand
-from sagemaker.mxnet import defaults
+
+MXNET_VERSION = "1.2"
 
 
 def train(args):
@@ -42,7 +43,7 @@ def create_estimator(self):
 
         return MXNet(
             entry_point=self.script,
-            framework_version=defaults.MXNET_VERSION,
+            framework_version=MXNET_VERSION,
             py_version=self.python,
             role=self.role_name,
             base_job_name=self.job_name,
@@ -66,7 +67,7 @@ def create_model(self, model_url):
             model_data=model_url,
             role=self.role_name,
             entry_point=self.script,
-            framework_version=defaults.MXNET_VERSION,
+            framework_version=MXNET_VERSION,
             py_version=self.python,
             name=self.endpoint_name,
             env=self.environment,

src/sagemaker/mxnet/defaults.py

@@ -13,12 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-MXNET_VERSION = "1.2"
-"""Default MXNet version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "1.6.0"
-"""The latest version of MXNet included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "1.6.0"

src/sagemaker/mxnet/estimator.py

@@ -38,8 +38,6 @@ class MXNet(Framework):
     __framework_name__ = "mxnet"
     _LOWEST_SCRIPT_MODE_VERSION = ["1", "3"]
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,
@@ -115,7 +113,7 @@ def __init__(
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
         validate_version_or_image_args(framework_version, py_version, image_name)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/mxnet/model.py

@@ -115,7 +115,7 @@ def __init__(
             :class:`~sagemaker.model.Model`.
         """
         validate_version_or_image_args(framework_version, py_version, image)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/pytorch/defaults.py

@@ -13,14 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-PYTORCH_VERSION = "0.4"
-"""Default PyTorch version for when the framework version is not specified.
-The default version is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "1.5.0"
-"""The latest version of PyTorch included in the SageMaker pre-built Docker images."""
-
-PYTHON_VERSION = "py3"
-
 LATEST_PY2_VERSION = "1.3.1"

src/sagemaker/pytorch/estimator.py

@@ -35,8 +35,6 @@ class PyTorch(Framework):
 
     __framework_name__ = "pytorch"
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,

src/sagemaker/pytorch/model.py

@@ -114,7 +114,7 @@ def __init__(
             :class:`~sagemaker.model.Model`.
         """
         validate_version_or_image_args(framework_version, py_version, image)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/sklearn/defaults.py

@@ -14,7 +14,3 @@
 from __future__ import absolute_import
 
 SKLEARN_NAME = "scikit-learn"
-
-SKLEARN_VERSION = "0.20.0"
-
-LATEST_PY2_VERSION = "0.20.0"

src/sagemaker/tensorflow/defaults.py

@@ -13,15 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-TF_VERSION = "1.11"
-"""Default TF version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "2.2.0"
-"""The latest version of TensorFlow included in the SageMaker pre-built Docker images."""
-
-LATEST_SERVING_VERSION = "2.1.0"
-"""The latest version of TensorFlow Serving included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "2.1.0"