Merge branch 'master' into augmented-manifest-file

Author: jesterhazy

CHANGELOG.rst

@@ -2,9 +2,15 @@
 CHANGELOG
 =========
 
-1.16.2
-======
+1.16.2.dev
+==========
+
 * feature: Add support for AugmentedManifestFile and ShuffleConfig
+* bug-fix: add version bound for requests module to avoid version conflicts between docker-compose and docker-py
+* bug-fix: Remove unnecessary dependency tensorflow
+* doc-fix: Change ``distribution`` to ``distributions``
+* bug-fix: Increase docker-compose http timeout and health check timeout to 120.
+* feature: Local Mode: Add support for intermediate output to a local directory.
 
 1.16.1.post1
 ============
@@ -30,6 +36,7 @@ CHANGELOG
 * feature: Add support for SageMaker Inference Pipelines
 * feature: Add support for SparkML serving container
 
+
 1.15.2
 ======
 

setup.py

@@ -35,7 +35,7 @@ def read(fname):
 # Declare minimal set for installation
 required_packages = ['boto3>=1.9.55', 'numpy>=1.9.0', 'protobuf>=3.1', 'scipy>=0.19.0',
                      'urllib3>=1.21', 'PyYAML>=3.2, <4', 'protobuf3-to-dict>=0.1.5',
-                     'docker-compose>=1.23.0']
+                     'docker-compose>=1.23.0', 'requests>=2.14.2, !=2.18.0, <2.21']
 
 # enum is introduced in Python 3.4. Installing enum back port
 if sys.version_info < (3, 4):

src/sagemaker/fw_utils.py

@@ -219,9 +219,11 @@ def framework_name_from_image(image_name):
     else:
         # extract framework, python version and image tag
         # We must support both the legacy and current image name format.
-        name_pattern = \
-            re.compile('^sagemaker(?:-rl)?-(tensorflow|mxnet|chainer|pytorch|scikit-learn):(.*)-(.*?)-(py2|py3)$')
-        legacy_name_pattern = re.compile('^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$')
+        name_pattern = re.compile(
+            r'^sagemaker(?:-rl)?-(tensorflow|mxnet|chainer|pytorch|scikit-learn):(.*)-(.*?)-(py2|py3)$')
+        legacy_name_pattern = re.compile(
+            r'^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$')
+
         name_match = name_pattern.match(sagemaker_match.group(8))
         legacy_match = legacy_name_pattern.match(sagemaker_match.group(8))
 

src/sagemaker/local/entities.py

@@ -29,7 +29,7 @@
 logger.setLevel(logging.WARNING)
 
 _UNUSED_ARN = 'local:arn-does-not-matter'
-HEALTH_CHECK_TIMEOUT_LIMIT = 30
+HEALTH_CHECK_TIMEOUT_LIMIT = 120
 
 
 class _LocalTrainingJob(object):
@@ -405,7 +405,7 @@ def _wait_for_serving_container(serving_port):
 
     endpoint_url = 'http://localhost:%s/ping' % serving_port
     while True:
-        i += 1
+        i += 5
         if i >= HEALTH_CHECK_TIMEOUT_LIMIT:
             raise RuntimeError('Giving up, endpoint didn\'t launch correctly')
 
@@ -416,7 +416,7 @@ def _wait_for_serving_container(serving_port):
         else:
             return
 
-        time.sleep(1)
+        time.sleep(5)
 
 
 def _perform_request(endpoint_url, pool_manager=None):

src/sagemaker/local/image.py

@@ -39,6 +39,9 @@
 
 CONTAINER_PREFIX = 'algo'
 DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml'
+DOCKER_COMPOSE_HTTP_TIMEOUT_ENV = 'COMPOSE_HTTP_TIMEOUT'
+DOCKER_COMPOSE_HTTP_TIMEOUT = '120'
+
 
 # Environment variables to be set during training
 REGION_ENV_NAME = 'AWS_REGION'
@@ -101,7 +104,8 @@ def train(self, input_data_config, output_data_config, hyperparameters, job_name
         os.mkdir(shared_dir)
 
         data_dir = self._create_tmp_folder()
-        volumes = self._prepare_training_volumes(data_dir, input_data_config, hyperparameters)
+        volumes = self._prepare_training_volumes(data_dir, input_data_config, output_data_config,
+                                                 hyperparameters)
 
         # Create the configuration files for each container that we will create
         # Each container will map the additional local volumes (if any).
@@ -278,7 +282,8 @@ def write_config_files(self, host, hyperparameters, input_data_config):
         _write_json_file(os.path.join(config_path, 'resourceconfig.json'), resource_config)
         _write_json_file(os.path.join(config_path, 'inputdataconfig.json'), json_input_data_config)
 
-    def _prepare_training_volumes(self, data_dir, input_data_config, hyperparameters):
+    def _prepare_training_volumes(self, data_dir, input_data_config, output_data_config,
+                                  hyperparameters):
         shared_dir = os.path.join(self.container_root, 'shared')
         model_dir = os.path.join(self.container_root, 'model')
         volumes = []
@@ -306,6 +311,14 @@ def _prepare_training_volumes(self, data_dir, input_data_config, hyperparameters
                 # Also mount a directory that all the containers can access.
                 volumes.append(_Volume(shared_dir, '/opt/ml/shared'))
 
+        parsed_uri = urlparse(output_data_config['S3OutputPath'])
+        if parsed_uri.scheme == 'file' \
+                and sagemaker.rl.estimator.SAGEMAKER_OUTPUT_LOCATION in hyperparameters:
+            intermediate_dir = os.path.join(parsed_uri.path, 'output', 'intermediate')
+            if not os.path.exists(intermediate_dir):
+                os.makedirs(intermediate_dir)
+            volumes.append(_Volume(intermediate_dir, '/opt/ml/output/intermediate'))
+
         return volumes
 
     def _prepare_serving_volumes(self, model_location):
@@ -359,6 +372,9 @@ def _generate_compose_file(self, command, additional_volumes=None, additional_en
         additional_env_var_list = ['{}={}'.format(k, v) for k, v in additional_env_vars.items()]
         environment.extend(additional_env_var_list)
 
+        if os.environ.get(DOCKER_COMPOSE_HTTP_TIMEOUT_ENV) is None:
+            os.environ[DOCKER_COMPOSE_HTTP_TIMEOUT_ENV] = DOCKER_COMPOSE_HTTP_TIMEOUT
+
         if command == 'train':
             optml_dirs = {'output', 'output/data', 'input'}
 

src/sagemaker/mxnet/README.rst

@@ -209,15 +209,15 @@ If you were previously relying on the default save method, you can now import on
 
         save(args.model_dir, model)
 
-Lastly, if you were relying on the container launching a parameter server for use with distributed training, you must now set ``distribution`` to the following dictionary when creating an MXNet estimator:
+Lastly, if you were relying on the container launching a parameter server for use with distributed training, you must now set ``distributions`` to the following dictionary when creating an MXNet estimator:
 
 .. code:: python
 
     from sagemaker.mxnet import MXNet
 
     estimator = MXNet('path-to-distributed-training-script.py',
                       ...,
-                      distribution={'parameter_server': {'enabled': True}})
+                      distributions={'parameter_server': {'enabled': True}})
 
 
 Using third-party libraries
@@ -323,7 +323,7 @@ The following are optional arguments. When you create an ``MXNet`` object, you c
    framework_version and py_version. Refer to: `SageMaker MXNet Docker Containers
    <#sagemaker-mxnet-docker-containers>`_ for details on what the Official images support
    and where to find the source code to build your custom image.
--  ``distribution`` For versions 1.3 and above only.
+-  ``distributions`` For versions 1.3 and above only.
    Specifies information for how to run distributed training.
    To launch a parameter server during training, set this argument to:
 

src/sagemaker/mxnet/estimator.py

@@ -67,7 +67,7 @@ def __init__(self, entry_point, source_dir=None, hyperparameters=None, py_versio
                     Examples:
                         123.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0
                         custom-image:latest.
-             distribution (dict): A dictionary with information on how to run distributed training
+             distributions (dict): A dictionary with information on how to run distributed training
                 (default: None).
             **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor.
         """

src/sagemaker/tensorflow/__init__.py

@@ -19,13 +19,5 @@
 # classes for tensorflow serving. Currently tensorflow_serving_api can only be pip-installed for python 2.
 sys.path.append(os.path.dirname(__file__))
 
-from distutils.version import LooseVersion  # noqa: E402, F401 pylint: disable=no-name-in-module
-import tensorflow  # noqa: E402, F401
-
-if LooseVersion(tensorflow.__version__) < LooseVersion("1.3.0"):  # pylint: disable=no-member
-    message = 'Tensorflow version must be >= 1.3.0. Current version: {}'.format(
-        tensorflow.__version__)  # pylint: disable=no-member
-    raise AssertionError(message)
-
 from sagemaker.tensorflow.estimator import TensorFlow  # noqa: E402, F401
 from sagemaker.tensorflow.model import TensorFlowModel, TensorFlowPredictor  # noqa: E402, F401

src/sagemaker/tensorflow/estimator.py

@@ -199,7 +199,7 @@ def __init__(self, training_steps=None, evaluation_steps=None, checkpoint_path=N
                         custom-image:latest.
             script_mode (bool): If set to True will the estimator will use the Script Mode containers (default: False).
                 This will be ignored if py_version is set to 'py3'.
-            distribution (dict): A dictionary with information on how to run distributed training
+            distributions (dict): A dictionary with information on how to run distributed training
                 (default: None). Currently we only support distributed training with parameter servers. To enable it
                 use the following setup:
                     {

src/sagemaker/tensorflow/serving.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import logging
-
 import sagemaker
 from sagemaker.content_types import CONTENT_TYPE_JSON
 from sagemaker.fw_utils import create_image_uri
@@ -144,7 +143,6 @@ def _get_image_uri(self, instance_type):
         if self.image:
             return self.image
 
-        # reuse standard image uri function, then strip unwanted python component
         region_name = self.sagemaker_session.boto_region_name
         return create_image_uri(region_name, Model.FRAMEWORK_NAME, instance_type,
                                 self._framework_version)

src/sagemaker/utils.py

@@ -14,6 +14,7 @@
 
 import errno
 import os
+import random
 import re
 import sys
 import tarfile
@@ -64,6 +65,14 @@ def name_from_base(base, max_length=63, short=False):
     return '{}-{}'.format(trimmed_base, timestamp)
 
 
+def unique_name_from_base(base, max_length=63):
+    unique = '%04x' % random.randrange(16**4)  # 4-digit hex
+    ts = str(int(time.time()))
+    available_length = max_length - 2 - len(ts) - len(unique)
+    trimmed = base[:available_length]
+    return '{}-{}-{}'.format(trimmed, ts, unique)
+
+
 def airflow_name_from_base(base, max_length=63, short=False):
     """Append airflow execution_date macro (https://airflow.apache.org/code.html?#macros)
     to the provided string. The macro will beevaluated in Airflow operator runtime.

tests/conftest.py

@@ -95,6 +95,11 @@ def mxnet_version(request):
     return request.param
 
 
+@pytest.fixture(scope='module', params=['1.3', '1.3.0'])
+def ei_mxnet_version(request):
+    return request.param
+
+
 @pytest.fixture(scope='module', params=['0.4', '0.4.0'])
 def pytorch_version(request):
     return request.param
@@ -112,6 +117,11 @@ def tf_version(request):
     return request.param
 
 
+@pytest.fixture(scope='module', params=['1.11', '1.11.0'])
+def ei_tf_version(request):
+    return request.param
+
+
 @pytest.fixture(scope='module', params=['0.10.1', '0.10.1', '0.11', '0.11.0'])
 def rl_coach_tf_version(request):
     return request.param