Merge branch 'master' into master

Author: caxiaohu

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## v1.30.0 (2019-06-25)
+
+### Features
+
+ * add TensorFlow 1.13 support
+ * add git_config and git_clone, validate method
+
+### Bug fixes and other changes
+
+ * add pytest.mark.local_mode annotation to broken tests
+
+## v1.29.0 (2019-06-24)
+
+### Features
+
+ * network isolation mode in training
+
+### Bug fixes and other changes
+
+ * Integrate black into development process
+ * moving not canary TFS tests to local mode
+
 ## v1.28.3 (2019-06-20)
 
 ### Bug fixes and other changes

README.rst

@@ -173,7 +173,7 @@ TensorFlow SageMaker Estimators
 
 By using TensorFlow SageMaker Estimators, you can train and host TensorFlow models on Amazon SageMaker.
 
-Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``.
+Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``, ``1.13.1``.
 
 Supported versions of TensorFlow for Elastic Inference: ``1.11.0``, ``1.12.0``.
 

VERSION

@@ -1 +1 @@
-1.28.4.dev0
+1.30.1.dev0

buildspec-release.yml

@@ -9,6 +9,9 @@ phases:
       # run linters
       - tox -e flake8,pylint
 
+      # run format verification
+      - tox -e black-check
+
       # run package and docbuild checks
       - tox -e twine
       - tox -e sphinx

buildspec.yml

@@ -14,6 +14,9 @@ phases:
       - tox -e twine
       - tox -e sphinx
 
+      # run format verification
+      - tox -e black-check
+
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=

doc/conf.py

@@ -28,51 +28,65 @@ def __getattr__(cls, name):
             return MagicMock()
 
 
-MOCK_MODULES = ['tensorflow', 'tensorflow.core', 'tensorflow.core.framework', 'tensorflow.python',
-                'tensorflow.python.framework', 'tensorflow_serving', 'tensorflow_serving.apis',
-                'numpy', 'scipy', 'scipy.sparse']
+MOCK_MODULES = [
+    "tensorflow",
+    "tensorflow.core",
+    "tensorflow.core.framework",
+    "tensorflow.python",
+    "tensorflow.python.framework",
+    "tensorflow_serving",
+    "tensorflow_serving.apis",
+    "numpy",
+    "scipy",
+    "scipy.sparse",
+]
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
-project = u'sagemaker'
+project = u"sagemaker"
 version = pkg_resources.require(project)[0].version
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
-              'sphinx.ext.intersphinx', 'sphinx.ext.todo',
-              'sphinx.ext.coverage', 'sphinx.ext.autosummary',
-              'sphinx.ext.napoleon']
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
-source_suffix = '.rst'  # The suffix of source filenames.
-master_doc = 'index'  # The master toctree document.
+source_suffix = ".rst"  # The suffix of source filenames.
+master_doc = "index"  # The master toctree document.
 
-copyright = u'%s, Amazon' % datetime.now().year
+copyright = u"%s, Amazon" % datetime.now().year
 
 # The full version, including alpha/beta/rc tags.
 release = version
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['_build']
+exclude_trees = ["_build"]
 
-pygments_style = 'default'
+pygments_style = "default"
 
 autoclass_content = "both"
-autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
-autodoc_member_order = 'bysource'
+autodoc_default_flags = ["show-inheritance", "members", "undoc-members"]
+autodoc_member_order = "bysource"
 
-if 'READTHEDOCS' in os.environ:
-    html_theme = 'default'
+if "READTHEDOCS" in os.environ:
+    html_theme = "default"
 else:
-    html_theme = 'haiku'
+    html_theme = "haiku"
 html_static_path = []
-htmlhelp_basename = '%sdoc' % project
+htmlhelp_basename = "%sdoc" % project
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {"http://docs.python.org/": None}
 
 # autosummary
 autosummary_generate = True

doc/overview.rst

@@ -84,6 +84,65 @@ For more `information <https://boto3.amazonaws.com/v1/documentation/api/latest/r
     # Deletes the SageMaker model
     mxnet_predictor.delete_model()
 
+Git Support
+~~~~~~~~~~~
+If you have your training scripts in your GitHub repository, you can use them directly without the trouble to download
+them to local machine. Git support can be enabled simply by providing ``git_config`` parameter when initializing an
+estimator. If Git support is enabled, then ``entry_point``, ``source_dir`` and  ``dependencies`` should all be relative
+paths in the Git repo. Note that if you decided to use Git support, then everything you need for ``entry_point``,
+``source_dir`` and ``dependencies`` should be in a single Git repo.
+
+Here are ways to specify ``git_config``:
+
+.. code:: python
+
+        # Specifies the git_config parameter
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git',
+                      'branch': 'branch1',
+                      'commit': '4893e528afa4a790331e1b5286954f073b0f14a2'}
+
+        # Alternatively, you can also specify git_config by providing only 'repo' and 'branch'.
+        # If this is the case, the latest commit in the branch will be used.
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git',
+                      'branch': 'branch1'}
+
+        # Only providing 'repo' is also allowed. If this is the case, latest commit in
+        # 'master' branch will be used.
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git'
+
+The following are some examples to define estimators with Git support:
+
+.. code:: python
+
+        # In this example, the source directory 'pytorch' contains the entry point 'mnist.py' and other source code.
+        # and it is  relative path inside the Git repo.
+        pytorch_estimator = PyTorch(entry_point='mnist.py',
+                                    role='SageMakerRole',
+                                    source_dir='pytorch',
+                                    git_config=git_config,
+                                    train_instance_count=1,
+                                    train_instance_type='ml.c4.xlarge')
+
+        # In this example, the entry point 'mnist.py' is all we need for source code.
+        # We need to specify the path to it in the Git repo.
+        mx_estimator = MXNet(entry_point='mxnet/mnist.py',
+                             role='SageMakerRole',
+                             git_config=git_config,
+                             train_instance_count=1,
+                             train_instance_type='ml.c4.xlarge')
+
+        # In this example, besides entry point and other source code in source directory, we still need some
+        # dependencies for the training job. Dependencies should also be paths inside the Git repo.
+        pytorch_estimator = PyTorch(entry_point='mnist.py',
+                                    role='SageMakerRole',
+                                    source_dir='pytorch',
+                                    dependencies=['dep.py', 'foo/bar.py'],
+                                    git_config=git_config,
+                                    train_instance_count=1,
+                                    train_instance_type='ml.c4.xlarge')
+
+When Git support is enabled, users can still use local mode in the same way.
+
 Training Metrics
 ~~~~~~~~~~~~~~~~
 The SageMaker Python SDK allows you to specify a name and a regular expression for metrics you want to track for training.
@@ -268,6 +327,7 @@ Currently, the following algorithms support incremental training:
 - Object Detection
 - Semantic Segmentation
 
+
 Using SageMaker AlgorithmEstimators
 -----------------------------------
 

doc/using_tf.rst

@@ -443,20 +443,10 @@ After a TensorFlow estimator has been fit, it saves a TensorFlow SavedModel in
 the S3 location defined by ``output_path``. You can call ``deploy`` on a TensorFlow
 estimator to create a SageMaker Endpoint.
 
-SageMaker provides two different options for deploying TensorFlow models to a SageMaker
-Endpoint:
+Your model will be deployed to a TensorFlow Serving-based server. The server provides a super-set of the
+`TensorFlow Serving REST API <https://www.tensorflow.org/serving/api_rest>`_.
 
-- The first option uses a Python-based server that allows you to specify your own custom
-  input and output handling functions in a Python script. This is the default option.
-
-  See `Deploying to Python-based Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_python.rst>`_ to learn how to use this option.
-
-
-- The second option uses a TensorFlow Serving-based server to provide a super-set of the
-  `TensorFlow Serving REST API <https://www.tensorflow.org/serving/api_rest>`_. This option
-  does not require (or allow) a custom python script.
-
-  See `Deploying to TensorFlow Serving Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_tensorflow_serving.rst>`_ to learn how to use this option.
+See `Deploying to TensorFlow Serving Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_tensorflow_serving.rst>`_ to learn how to deploy your model and make inference requests.
 
 
 SageMaker TensorFlow Docker containers

examples/cli/host/script.py

@@ -12,12 +12,12 @@ def model_fn(model_dir):
     :param: model_dir The directory where model files are stored.
     :return: a model (in this case a Gluon network)
     """
-    symbol = mx.sym.load('%s/model.json' % model_dir)
-    outputs = mx.symbol.softmax(data=symbol, name='softmax_label')
-    inputs = mx.sym.var('data')
-    param_dict = gluon.ParameterDict('model_')
+    symbol = mx.sym.load("%s/model.json" % model_dir)
+    outputs = mx.symbol.softmax(data=symbol, name="softmax_label")
+    inputs = mx.sym.var("data")
+    param_dict = gluon.ParameterDict("model_")
     net = gluon.SymbolBlock(outputs, inputs, param_dict)
-    net.load_params('%s/model.params' % model_dir, ctx=mx.cpu())
+    net.load_params("%s/model.params" % model_dir, ctx=mx.cpu())
     return net
 
 

examples/cli/train/download_training_data.py

@@ -2,8 +2,8 @@
 
 
 def download_training_data():
-    gluon.data.vision.MNIST('./data/training', train=True)
-    gluon.data.vision.MNIST('./data/training', train=False)
+    gluon.data.vision.MNIST("./data/training", train=True)
+    gluon.data.vision.MNIST("./data/training", train=False)
 
 
 if __name__ == "__main__":

examples/cli/train/script.py

@@ -15,13 +15,13 @@ def train(channel_input_dirs, hyperparameters, **kwargs):
     ctx = mx.cpu()
 
     # retrieve the hyperparameters we set in notebook (with some defaults)
-    batch_size = hyperparameters.get('batch_size', 100)
-    epochs = hyperparameters.get('epochs', 10)
-    learning_rate = hyperparameters.get('learning_rate', 0.1)
-    momentum = hyperparameters.get('momentum', 0.9)
-    log_interval = hyperparameters.get('log_interval', 100)
+    batch_size = hyperparameters.get("batch_size", 100)
+    epochs = hyperparameters.get("epochs", 10)
+    learning_rate = hyperparameters.get("learning_rate", 0.1)
+    momentum = hyperparameters.get("momentum", 0.9)
+    log_interval = hyperparameters.get("log_interval", 100)
 
-    training_data = channel_input_dirs['training']
+    training_data = channel_input_dirs["training"]
 
     # load training and validation data
     # we use the gluon.data.vision.MNIST class because of its built in mnist pre-processing logic,
@@ -35,8 +35,9 @@ def train(channel_input_dirs, hyperparameters, **kwargs):
     # Collect all parameters from net and its children, then initialize them.
     net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)
     # Trainer is for updating parameters with gradient.
-    trainer = gluon.Trainer(net.collect_params(), 'sgd',
-                            {'learning_rate': learning_rate, 'momentum': momentum})
+    trainer = gluon.Trainer(
+        net.collect_params(), "sgd", {"learning_rate": learning_rate, "momentum": momentum}
+    )
     metric = mx.metric.Accuracy()
     loss = gluon.loss.SoftmaxCrossEntropyLoss()
 
@@ -61,32 +62,34 @@ def train(channel_input_dirs, hyperparameters, **kwargs):
 
             if i % log_interval == 0 and i > 0:
                 name, acc = metric.get()
-                logger.info('[Epoch %d Batch %d] Training: %s=%f, %f samples/s' %
-                            (epoch, i, name, acc, batch_size / (time.time() - btic)))
+                logger.info(
+                    "[Epoch %d Batch %d] Training: %s=%f, %f samples/s"
+                    % (epoch, i, name, acc, batch_size / (time.time() - btic))
+                )
 
             btic = time.time()
 
         name, acc = metric.get()
-        logger.info('[Epoch %d] Training: %s=%f' % (epoch, name, acc))
+        logger.info("[Epoch %d] Training: %s=%f" % (epoch, name, acc))
 
         name, val_acc = test(ctx, net, val_data)
-        logger.info('[Epoch %d] Validation: %s=%f' % (epoch, name, val_acc))
+        logger.info("[Epoch %d] Validation: %s=%f" % (epoch, name, val_acc))
 
     return net
 
 
 def save(net, model_dir):
     # save the model
-    y = net(mx.sym.var('data'))
-    y.save('%s/model.json' % model_dir)
-    net.collect_params().save('%s/model.params' % model_dir)
+    y = net(mx.sym.var("data"))
+    y.save("%s/model.json" % model_dir)
+    net.collect_params().save("%s/model.params" % model_dir)
 
 
 def define_network():
     net = nn.Sequential()
     with net.name_scope():
-        net.add(nn.Dense(128, activation='relu'))
-        net.add(nn.Dense(64, activation='relu'))
+        net.add(nn.Dense(128, activation="relu"))
+        net.add(nn.Dense(64, activation="relu"))
         net.add(nn.Dense(10))
     return net
 
@@ -99,13 +102,18 @@ def input_transformer(data, label):
 def get_train_data(data_dir, batch_size):
     return gluon.data.DataLoader(
         gluon.data.vision.MNIST(data_dir, train=True, transform=input_transformer),
-        batch_size=batch_size, shuffle=True, last_batch='discard')
+        batch_size=batch_size,
+        shuffle=True,
+        last_batch="discard",
+    )
 
 
 def get_val_data(data_dir, batch_size):
     return gluon.data.DataLoader(
         gluon.data.vision.MNIST(data_dir, train=False, transform=input_transformer),
-        batch_size=batch_size, shuffle=False)
+        batch_size=batch_size,
+        shuffle=False,
+    )
 
 
 def test(ctx, net, val_data):