Merge branch 'master' into requests

Author: jesterhazy

CHANGELOG.rst

@@ -6,6 +6,10 @@ CHANGELOG
 ==========
 
 * bug-fix: add version bound for requests module to avoid version conflicts between docker-compose and docker-py
+* bug-fix: Remove unnecessary dependency tensorflow
+* doc-fix: Change ``distribution`` to ``distributions``
+* bug-fix: Increase docker-compose http timeout and health check timeout to 120.
+* feature: Local Mode: Add support for intermediate output to a local directory.
 
 1.16.1.post1
 ============
@@ -31,6 +35,7 @@ CHANGELOG
 * feature: Add support for SageMaker Inference Pipelines
 * feature: Add support for SparkML serving container
 
+
 1.15.2
 ======
 

src/sagemaker/fw_utils.py

@@ -219,9 +219,11 @@ def framework_name_from_image(image_name):
     else:
         # extract framework, python version and image tag
         # We must support both the legacy and current image name format.
-        name_pattern = \
-            re.compile('^sagemaker(?:-rl)?-(tensorflow|mxnet|chainer|pytorch|scikit-learn):(.*)-(.*?)-(py2|py3)$')
-        legacy_name_pattern = re.compile('^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$')
+        name_pattern = re.compile(
+            r'^sagemaker(?:-rl)?-(tensorflow|mxnet|chainer|pytorch|scikit-learn):(.*)-(.*?)-(py2|py3)$')
+        legacy_name_pattern = re.compile(
+            r'^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$')
+
         name_match = name_pattern.match(sagemaker_match.group(8))
         legacy_match = legacy_name_pattern.match(sagemaker_match.group(8))
 

src/sagemaker/local/entities.py

@@ -29,7 +29,7 @@
 logger.setLevel(logging.WARNING)
 
 _UNUSED_ARN = 'local:arn-does-not-matter'
-HEALTH_CHECK_TIMEOUT_LIMIT = 30
+HEALTH_CHECK_TIMEOUT_LIMIT = 120
 
 
 class _LocalTrainingJob(object):
@@ -405,7 +405,7 @@ def _wait_for_serving_container(serving_port):
 
     endpoint_url = 'http://localhost:%s/ping' % serving_port
     while True:
-        i += 1
+        i += 5
         if i >= HEALTH_CHECK_TIMEOUT_LIMIT:
             raise RuntimeError('Giving up, endpoint didn\'t launch correctly')
 
@@ -416,7 +416,7 @@ def _wait_for_serving_container(serving_port):
         else:
             return
 
-        time.sleep(1)
+        time.sleep(5)
 
 
 def _perform_request(endpoint_url, pool_manager=None):

src/sagemaker/local/image.py

@@ -39,6 +39,9 @@
 
 CONTAINER_PREFIX = 'algo'
 DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml'
+DOCKER_COMPOSE_HTTP_TIMEOUT_ENV = 'COMPOSE_HTTP_TIMEOUT'
+DOCKER_COMPOSE_HTTP_TIMEOUT = '120'
+
 
 # Environment variables to be set during training
 REGION_ENV_NAME = 'AWS_REGION'
@@ -101,7 +104,8 @@ def train(self, input_data_config, output_data_config, hyperparameters, job_name
         os.mkdir(shared_dir)
 
         data_dir = self._create_tmp_folder()
-        volumes = self._prepare_training_volumes(data_dir, input_data_config, hyperparameters)
+        volumes = self._prepare_training_volumes(data_dir, input_data_config, output_data_config,
+                                                 hyperparameters)
 
         # Create the configuration files for each container that we will create
         # Each container will map the additional local volumes (if any).
@@ -278,7 +282,8 @@ def write_config_files(self, host, hyperparameters, input_data_config):
         _write_json_file(os.path.join(config_path, 'resourceconfig.json'), resource_config)
         _write_json_file(os.path.join(config_path, 'inputdataconfig.json'), json_input_data_config)
 
-    def _prepare_training_volumes(self, data_dir, input_data_config, hyperparameters):
+    def _prepare_training_volumes(self, data_dir, input_data_config, output_data_config,
+                                  hyperparameters):
         shared_dir = os.path.join(self.container_root, 'shared')
         model_dir = os.path.join(self.container_root, 'model')
         volumes = []
@@ -306,6 +311,14 @@ def _prepare_training_volumes(self, data_dir, input_data_config, hyperparameters
                 # Also mount a directory that all the containers can access.
                 volumes.append(_Volume(shared_dir, '/opt/ml/shared'))
 
+        parsed_uri = urlparse(output_data_config['S3OutputPath'])
+        if parsed_uri.scheme == 'file' \
+                and sagemaker.rl.estimator.SAGEMAKER_OUTPUT_LOCATION in hyperparameters:
+            intermediate_dir = os.path.join(parsed_uri.path, 'output', 'intermediate')
+            if not os.path.exists(intermediate_dir):
+                os.makedirs(intermediate_dir)
+            volumes.append(_Volume(intermediate_dir, '/opt/ml/output/intermediate'))
+
         return volumes
 
     def _prepare_serving_volumes(self, model_location):
@@ -359,6 +372,9 @@ def _generate_compose_file(self, command, additional_volumes=None, additional_en
         additional_env_var_list = ['{}={}'.format(k, v) for k, v in additional_env_vars.items()]
         environment.extend(additional_env_var_list)
 
+        if os.environ.get(DOCKER_COMPOSE_HTTP_TIMEOUT_ENV) is None:
+            os.environ[DOCKER_COMPOSE_HTTP_TIMEOUT_ENV] = DOCKER_COMPOSE_HTTP_TIMEOUT
+
         if command == 'train':
             optml_dirs = {'output', 'output/data', 'input'}
 

src/sagemaker/mxnet/README.rst

@@ -209,15 +209,15 @@ If you were previously relying on the default save method, you can now import on
 
         save(args.model_dir, model)
 
-Lastly, if you were relying on the container launching a parameter server for use with distributed training, you must now set ``distribution`` to the following dictionary when creating an MXNet estimator:
+Lastly, if you were relying on the container launching a parameter server for use with distributed training, you must now set ``distributions`` to the following dictionary when creating an MXNet estimator:
 
 .. code:: python
 
     from sagemaker.mxnet import MXNet
 
     estimator = MXNet('path-to-distributed-training-script.py',
                       ...,
-                      distribution={'parameter_server': {'enabled': True}})
+                      distributions={'parameter_server': {'enabled': True}})
 
 
 Using third-party libraries
@@ -323,7 +323,7 @@ The following are optional arguments. When you create an ``MXNet`` object, you c
    framework_version and py_version. Refer to: `SageMaker MXNet Docker Containers
    <#sagemaker-mxnet-docker-containers>`_ for details on what the Official images support
    and where to find the source code to build your custom image.
--  ``distribution`` For versions 1.3 and above only.
+-  ``distributions`` For versions 1.3 and above only.
    Specifies information for how to run distributed training.
    To launch a parameter server during training, set this argument to:
 

src/sagemaker/mxnet/estimator.py

@@ -67,7 +67,7 @@ def __init__(self, entry_point, source_dir=None, hyperparameters=None, py_versio
                     Examples:
                         123.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0
                         custom-image:latest.
-             distribution (dict): A dictionary with information on how to run distributed training
+             distributions (dict): A dictionary with information on how to run distributed training
                 (default: None).
             **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor.
         """

src/sagemaker/tensorflow/__init__.py

@@ -19,13 +19,5 @@
 # classes for tensorflow serving. Currently tensorflow_serving_api can only be pip-installed for python 2.
 sys.path.append(os.path.dirname(__file__))
 
-from distutils.version import LooseVersion  # noqa: E402, F401 pylint: disable=no-name-in-module
-import tensorflow  # noqa: E402, F401
-
-if LooseVersion(tensorflow.__version__) < LooseVersion("1.3.0"):  # pylint: disable=no-member
-    message = 'Tensorflow version must be >= 1.3.0. Current version: {}'.format(
-        tensorflow.__version__)  # pylint: disable=no-member
-    raise AssertionError(message)
-
 from sagemaker.tensorflow.estimator import TensorFlow  # noqa: E402, F401
 from sagemaker.tensorflow.model import TensorFlowModel, TensorFlowPredictor  # noqa: E402, F401

src/sagemaker/tensorflow/estimator.py

@@ -199,7 +199,7 @@ def __init__(self, training_steps=None, evaluation_steps=None, checkpoint_path=N
                         custom-image:latest.
             script_mode (bool): If set to True will the estimator will use the Script Mode containers (default: False).
                 This will be ignored if py_version is set to 'py3'.
-            distribution (dict): A dictionary with information on how to run distributed training
+            distributions (dict): A dictionary with information on how to run distributed training
                 (default: None). Currently we only support distributed training with parameter servers. To enable it
                 use the following setup:
                     {

src/sagemaker/tensorflow/serving.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import logging
-
 import sagemaker
 from sagemaker.content_types import CONTENT_TYPE_JSON
 from sagemaker.fw_utils import create_image_uri
@@ -144,7 +143,6 @@ def _get_image_uri(self, instance_type):
         if self.image:
             return self.image
 
-        # reuse standard image uri function, then strip unwanted python component
         region_name = self.sagemaker_session.boto_region_name
         return create_image_uri(region_name, Model.FRAMEWORK_NAME, instance_type,
                                 self._framework_version)

src/sagemaker/utils.py

@@ -14,6 +14,7 @@
 
 import errno
 import os
+import random
 import re
 import sys
 import tarfile
@@ -64,6 +65,14 @@ def name_from_base(base, max_length=63, short=False):
     return '{}-{}'.format(trimmed_base, timestamp)
 
 
+def unique_name_from_base(base, max_length=63):
+    unique = '%04x' % random.randrange(16**4)  # 4-digit hex
+    ts = str(int(time.time()))
+    available_length = max_length - 2 - len(ts) - len(unique)
+    trimmed = base[:available_length]
+    return '{}-{}-{}'.format(trimmed, ts, unique)
+
+
 def airflow_name_from_base(base, max_length=63, short=False):
     """Append airflow execution_date macro (https://airflow.apache.org/code.html?#macros)
     to the provided string. The macro will beevaluated in Airflow operator runtime.

tests/conftest.py

@@ -95,6 +95,11 @@ def mxnet_version(request):
     return request.param
 
 
+@pytest.fixture(scope='module', params=['1.3', '1.3.0'])
+def ei_mxnet_version(request):
+    return request.param
+
+
 @pytest.fixture(scope='module', params=['0.4', '0.4.0'])
 def pytorch_version(request):
     return request.param
@@ -112,6 +117,11 @@ def tf_version(request):
     return request.param
 
 
+@pytest.fixture(scope='module', params=['1.11', '1.11.0'])
+def ei_tf_version(request):
+    return request.param
+
+
 @pytest.fixture(scope='module', params=['0.10.1', '0.10.1', '0.11', '0.11.0'])
 def rl_coach_tf_version(request):
     return request.param

tests/data/marketplace/training/iris.csv

@@ -0,0 +1,150 @@
+setosa,5.1,3.5,1.4,0.2
+setosa,4.9,3,1.4,0.2
+setosa,4.7,3.2,1.3,0.2
+setosa,4.6,3.1,1.5,0.2
+setosa,5,3.6,1.4,0.2
+setosa,5.4,3.9,1.7,0.4
+setosa,4.6,3.4,1.4,0.3
+setosa,5,3.4,1.5,0.2
+setosa,4.4,2.9,1.4,0.2
+setosa,4.9,3.1,1.5,0.1
+setosa,5.4,3.7,1.5,0.2
+setosa,4.8,3.4,1.6,0.2
+setosa,4.8,3,1.4,0.1
+setosa,4.3,3,1.1,0.1
+setosa,5.8,4,1.2,0.2
+setosa,5.7,4.4,1.5,0.4
+setosa,5.4,3.9,1.3,0.4
+setosa,5.1,3.5,1.4,0.3
+setosa,5.7,3.8,1.7,0.3
+setosa,5.1,3.8,1.5,0.3
+setosa,5.4,3.4,1.7,0.2
+setosa,5.1,3.7,1.5,0.4
+setosa,4.6,3.6,1,0.2
+setosa,5.1,3.3,1.7,0.5
+setosa,4.8,3.4,1.9,0.2
+setosa,5,3,1.6,0.2
+setosa,5,3.4,1.6,0.4
+setosa,5.2,3.5,1.5,0.2
+setosa,5.2,3.4,1.4,0.2
+setosa,4.7,3.2,1.6,0.2
+setosa,4.8,3.1,1.6,0.2
+setosa,5.4,3.4,1.5,0.4
+setosa,5.2,4.1,1.5,0.1
+setosa,5.5,4.2,1.4,0.2
+setosa,4.9,3.1,1.5,0.2
+setosa,5,3.2,1.2,0.2
+setosa,5.5,3.5,1.3,0.2
+setosa,4.9,3.6,1.4,0.1
+setosa,4.4,3,1.3,0.2
+setosa,5.1,3.4,1.5,0.2
+setosa,5,3.5,1.3,0.3
+setosa,4.5,2.3,1.3,0.3
+setosa,4.4,3.2,1.3,0.2
+setosa,5,3.5,1.6,0.6
+setosa,5.1,3.8,1.9,0.4
+setosa,4.8,3,1.4,0.3
+setosa,5.1,3.8,1.6,0.2
+setosa,4.6,3.2,1.4,0.2
+setosa,5.3,3.7,1.5,0.2
+setosa,5,3.3,1.4,0.2
+versicolor,7,3.2,4.7,1.4
+versicolor,6.4,3.2,4.5,1.5
+versicolor,6.9,3.1,4.9,1.5
+versicolor,5.5,2.3,4,1.3
+versicolor,6.5,2.8,4.6,1.5
+versicolor,5.7,2.8,4.5,1.3
+versicolor,6.3,3.3,4.7,1.6
+versicolor,4.9,2.4,3.3,1
+versicolor,6.6,2.9,4.6,1.3
+versicolor,5.2,2.7,3.9,1.4
+versicolor,5,2,3.5,1
+versicolor,5.9,3,4.2,1.5
+versicolor,6,2.2,4,1
+versicolor,6.1,2.9,4.7,1.4
+versicolor,5.6,2.9,3.6,1.3
+versicolor,6.7,3.1,4.4,1.4
+versicolor,5.6,3,4.5,1.5
+versicolor,5.8,2.7,4.1,1
+versicolor,6.2,2.2,4.5,1.5
+versicolor,5.6,2.5,3.9,1.1
+versicolor,5.9,3.2,4.8,1.8
+versicolor,6.1,2.8,4,1.3
+versicolor,6.3,2.5,4.9,1.5
+versicolor,6.1,2.8,4.7,1.2
+versicolor,6.4,2.9,4.3,1.3
+versicolor,6.6,3,4.4,1.4
+versicolor,6.8,2.8,4.8,1.4
+versicolor,6.7,3,5,1.7
+versicolor,6,2.9,4.5,1.5
+versicolor,5.7,2.6,3.5,1
+versicolor,5.5,2.4,3.8,1.1
+versicolor,5.5,2.4,3.7,1
+versicolor,5.8,2.7,3.9,1.2
+versicolor,6,2.7,5.1,1.6
+versicolor,5.4,3,4.5,1.5
+versicolor,6,3.4,4.5,1.6
+versicolor,6.7,3.1,4.7,1.5
+versicolor,6.3,2.3,4.4,1.3
+versicolor,5.6,3,4.1,1.3
+versicolor,5.5,2.5,4,1.3
+versicolor,5.5,2.6,4.4,1.2
+versicolor,6.1,3,4.6,1.4
+versicolor,5.8,2.6,4,1.2
+versicolor,5,2.3,3.3,1
+versicolor,5.6,2.7,4.2,1.3
+versicolor,5.7,3,4.2,1.2
+versicolor,5.7,2.9,4.2,1.3
+versicolor,6.2,2.9,4.3,1.3
+versicolor,5.1,2.5,3,1.1
+versicolor,5.7,2.8,4.1,1.3
+virginica,6.3,3.3,6,2.5
+virginica,5.8,2.7,5.1,1.9
+virginica,7.1,3,5.9,2.1
+virginica,6.3,2.9,5.6,1.8
+virginica,6.5,3,5.8,2.2
+virginica,7.6,3,6.6,2.1
+virginica,4.9,2.5,4.5,1.7
+virginica,7.3,2.9,6.3,1.8
+virginica,6.7,2.5,5.8,1.8
+virginica,7.2,3.6,6.1,2.5
+virginica,6.5,3.2,5.1,2
+virginica,6.4,2.7,5.3,1.9
+virginica,6.8,3,5.5,2.1
+virginica,5.7,2.5,5,2
+virginica,5.8,2.8,5.1,2.4
+virginica,6.4,3.2,5.3,2.3
+virginica,6.5,3,5.5,1.8
+virginica,7.7,3.8,6.7,2.2
+virginica,7.7,2.6,6.9,2.3
+virginica,6,2.2,5,1.5
+virginica,6.9,3.2,5.7,2.3
+virginica,5.6,2.8,4.9,2
+virginica,7.7,2.8,6.7,2
+virginica,6.3,2.7,4.9,1.8
+virginica,6.7,3.3,5.7,2.1
+virginica,7.2,3.2,6,1.8
+virginica,6.2,2.8,4.8,1.8
+virginica,6.1,3,4.9,1.8
+virginica,6.4,2.8,5.6,2.1
+virginica,7.2,3,5.8,1.6
+virginica,7.4,2.8,6.1,1.9
+virginica,7.9,3.8,6.4,2
+virginica,6.4,2.8,5.6,2.2
+virginica,6.3,2.8,5.1,1.5
+virginica,6.1,2.6,5.6,1.4
+virginica,7.7,3,6.1,2.3
+virginica,6.3,3.4,5.6,2.4
+virginica,6.4,3.1,5.5,1.8
+virginica,6,3,4.8,1.8
+virginica,6.9,3.1,5.4,2.1
+virginica,6.7,3.1,5.6,2.4
+virginica,6.9,3.1,5.1,2.3
+virginica,5.8,2.7,5.1,1.9
+virginica,6.8,3.2,5.9,2.3
+virginica,6.7,3.3,5.7,2.5
+virginica,6.7,3,5.2,2.3
+virginica,6.3,2.5,5,1.9
+virginica,6.5,3,5.2,2
+virginica,6.2,3.4,5.4,2.3
+virginica,5.9,3,5.1,1.8