doc: address merge conflicts

Author: eslesar-aws

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v1.35.0 (2019-07-31)
+
+### Features
+
+ * allow serving image to be specified when calling MXNet.deploy
+
+## v1.34.3 (2019-07-30)
+
+### Bug fixes and other changes
+
+ * waiting for training tags to propagate in the test
+
+## v1.34.2 (2019-07-29)
+
+### Bug fixes and other changes
+
+ * removing unnecessary tests cases
+ * Replaced generic ValueError with custom subclass when reporting unexpected resource status
+
+### Documentation changes
+
+ * correct wording for Cloud9 environment setup instructions
+
 ## v1.34.1 (2019-07-23)
 
 ### Bug fixes and other changes

VERSION

@@ -1 +1 @@
-1.34.2.dev0
+1.35.1.dev0

doc/using_mxnet.rst

@@ -801,7 +801,101 @@ The following are optional arguments. When you create an ``MXNet`` object, you c
 SageMaker MXNet Containers
 **************************
 
+=======
+
+Required arguments
+==================
+
+The following are required arguments to the ``MXNet`` constructor. When you create an MXNet object, you must include these in the constructor, either positionally or as keyword arguments.
+
+-  ``entry_point`` Path (absolute or relative) to the Python file which
+   should be executed as the entry point to training.
+-  ``role`` An AWS IAM role (either name or full ARN). The Amazon
+   SageMaker training jobs and APIs that create Amazon SageMaker
+   endpoints use this role to access training data and model artifacts.
+   After the endpoint is created, the inference code might use the IAM
+   role, if accessing AWS resource.
+-  ``train_instance_count`` Number of Amazon EC2 instances to use for
+   training.
+-  ``train_instance_type`` Type of EC2 instance to use for training, for
+   example, 'ml.c4.xlarge'.
+
+Optional arguments
+==================
+
+The following are optional arguments. When you create an ``MXNet`` object, you can specify these as keyword arguments.
+
+-  ``source_dir`` Path (absolute or relative) to a directory with any
+   other training source code dependencies including the entry point
+   file. Structure within this directory will be preserved when training
+   on SageMaker.
+-  ``dependencies (list[str])`` A list of paths to directories (absolute or relative) with
+   any additional libraries that will be exported to the container (default: ``[]``).
+   The library folders will be copied to SageMaker in the same folder where the entrypoint is copied.
+   If the ``source_dir`` points to S3, code will be uploaded and the S3 location will be used
+   instead. For example, the following call
+
+   >>> MXNet(entry_point='train.py', dependencies=['my/libs/common', 'virtual-env'])
+
+   results in the following inside the container:
+
+   .. code::
+
+       opt/ml/code
+         ├── train.py
+         ├── common
+         └── virtual-env
+
+-  ``hyperparameters`` Hyperparameters that will be used for training.
+   Will be made accessible as a dict[str, str] to the training code on
+   SageMaker. For convenience, accepts other types besides str, but
+   str() will be called on keys and values to convert them before
+   training.
+-  ``py_version`` Python version you want to use for executing your
+   model training code. Valid values: 'py2' and 'py3'.
+-  ``train_volume_size`` Size in GB of the EBS volume to use for storing
+   input data during training. Must be large enough to store training
+   data if input_mode='File' is used (which is the default).
+-  ``train_max_run`` Timeout in seconds for training, after which Amazon
+   SageMaker terminates the job regardless of its current status.
+-  ``input_mode`` The input mode that the algorithm supports. Valid
+   modes: 'File' - Amazon SageMaker copies the training dataset from the
+   S3 location to a directory in the Docker container. 'Pipe' - Amazon
+   SageMaker streams data directly from S3 to the container via a Unix
+   named pipe.
+-  ``output_path`` Location where you want the training result (model artifacts and optional output files) saved.
+   This should be an S3 location unless you're using Local Mode, which also supports local output paths.
+   If not specified, results are stored to a default S3 bucket.
+-  ``output_kms_key`` Optional KMS key ID to optionally encrypt training
+   output with.
+-  ``job_name`` Name to assign for the training job that the fit()
+   method launches. If not specified, the estimator generates a default
+   job name, based on the training image name and current timestamp
+-  ``image_name`` An alternative docker image to use for training and
+   serving.  If specified, the estimator will use this image for training and
+   hosting, instead of selecting the appropriate SageMaker official image based on
+   framework_version and py_version. Refer to: `SageMaker MXNet Docker Containers
+   <#sagemaker-mxnet-docker-containers>`_ for details on what the Official images support
+   and where to find the source code to build your custom image.
+-  ``distributions`` For versions 1.3 and above only.
+   Specifies information for how to run distributed training.
+   To launch a parameter server during training, set this argument to:
+
+.. code::
+
+    {
+      'parameter_server': {
+        'enabled': True
+      }
+    }
+
+**************************
+SageMaker MXNet Containers
+**************************
+
 For information about SageMaker MXNet containers, see the following topics:
 
 - training: https://github.com/aws/sagemaker-mxnet-container
 - serving: https://github.com/aws/sagemaker-mxnet-serving-container
+
+For information about the dependencies installed in SageMaker MXNet containers, see https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/mxnet/README.rst#sagemaker-mxnet-containers.

doc/using_sklearn.rst

@@ -97,6 +97,16 @@ inadvertently run your training code at the wrong point in execution.
 
 For more on training environment variables, please visit https://github.com/aws/sagemaker-containers.
 
+Using third-party libraries
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When running your training script on SageMaker, it will have access to some pre-installed third-party libraries including ``scikit-learn``, ``numpy``, and ``pandas``.
+For more information on the runtime environment, including specific package versions, see `SageMaker Scikit-learn Docker containers <https://github.com/aws/sagemaker-scikit-learn-container>`__.
+
+If there are other packages you want to use with your script, you can include a ``requirements.txte` file in the same directory as your training script to install other dependencies at runtime.
+A ``requirements.txt`` file is a text file that contains a list of items that are installed by using ``pip install``. You can also specify the version of an item to install.
+For information about the format of a ``requirements.txt`` file, see `Requirements Files <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`__ in the pip documentation.
+
 Running a Scikit-learn training script in SageMaker
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/sagemaker/mxnet/estimator.py

@@ -140,6 +140,7 @@ def create_model(
         entry_point=None,
         source_dir=None,
         dependencies=None,
+        image_name=None,
     ):
         """Create a SageMaker ``MXNetModel`` object that can be deployed to an
         ``Endpoint``.
@@ -164,6 +165,12 @@ def create_model(
             dependencies (list[str]): A list of paths to directories (absolute or relative) with
                 any additional libraries that will be exported to the container.
                 If not specified, the dependencies from training are used.
+            image_name (str): If specified, the estimator will use this image for hosting, instead
+                of selecting the appropriate SageMaker official image based on framework_version
+                and py_version. It can be an ECR url or dockerhub image and tag.
+                Examples:
+                    123.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0
+                    custom-image:latest.
 
         Returns:
             sagemaker.mxnet.model.MXNetModel: A SageMaker ``MXNetModel`` object.
@@ -180,7 +187,7 @@ def create_model(
             code_location=self.code_location,
             py_version=self.py_version,
             framework_version=self.framework_version,
-            image=self.image_name,
+            image=(image_name or self.image_name),
             model_server_workers=model_server_workers,
             sagemaker_session=self.sagemaker_session,
             vpc_config=self.get_vpc_config(vpc_config_override),

tests/integ/test_tf_script_mode.py

@@ -146,14 +146,14 @@ def test_mnist_async(sagemaker_session):
     training_job_name = estimator.latest_training_job.name
     time.sleep(20)
     endpoint_name = training_job_name
-    model_name = "model-name-1"
     _assert_training_job_tags_match(
         sagemaker_session.sagemaker_client, estimator.latest_training_job.name, TAGS
     )
     with tests.integ.timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
         estimator = TensorFlow.attach(
             training_job_name=training_job_name, sagemaker_session=sagemaker_session
         )
+        model_name = "model-mnist-async"
         predictor = estimator.deploy(
             initial_instance_count=1,
             instance_type="ml.c4.xlarge",
@@ -215,14 +215,14 @@ def _assert_s3_files_exist(s3_url, files):
             raise ValueError("File {} is not found under {}".format(f, s3_url))
 
 
-def _assert_tags_match(sagemaker_client, resource_arn, tags, retries=1):
+def _assert_tags_match(sagemaker_client, resource_arn, tags, retries=15):
     actual_tags = None
     for _ in range(retries):
         actual_tags = sagemaker_client.list_tags(ResourceArn=resource_arn)["Tags"]
         if actual_tags:
             break
         else:
-            # endpoint tags might take minutes to propagate. Sleeping.
+            # endpoint and training tags might take minutes to propagate. Sleeping.
             time.sleep(30)
     assert actual_tags == tags
 
@@ -235,7 +235,7 @@ def _assert_model_tags_match(sagemaker_client, model_name, tags):
 def _assert_endpoint_tags_match(sagemaker_client, endpoint_name, tags):
     endpoint_description = sagemaker_client.describe_endpoint(EndpointName=endpoint_name)
 
-    _assert_tags_match(sagemaker_client, endpoint_description["EndpointArn"], tags, retries=10)
+    _assert_tags_match(sagemaker_client, endpoint_description["EndpointArn"], tags)
 
 
 def _assert_training_job_tags_match(sagemaker_client, training_job_name, tags):

tests/unit/test_mxnet.py

@@ -704,3 +704,26 @@ def test_empty_framework_version(warning, sagemaker_session):
 
     assert mx.framework_version == defaults.MXNET_VERSION
     warning.assert_called_with(defaults.MXNET_VERSION, mx.LATEST_VERSION)
+
+
+def test_create_model_with_custom_hosting_image(sagemaker_session):
+    container_log_level = '"logging.INFO"'
+    source_dir = "s3://mybucket/source"
+    custom_image = "mxnet:2.0"
+    custom_hosting_image = "mxnet_hosting:2.0"
+    mx = MXNet(
+        entry_point=SCRIPT_PATH,
+        role=ROLE,
+        sagemaker_session=sagemaker_session,
+        train_instance_count=INSTANCE_COUNT,
+        train_instance_type=INSTANCE_TYPE,
+        image_name=custom_image,
+        container_log_level=container_log_level,
+        base_job_name="job",
+        source_dir=source_dir,
+    )
+
+    mx.fit(inputs="s3://mybucket/train", job_name="new_name")
+    model = mx.create_model(image_name=custom_hosting_image)
+
+    assert model.image == custom_hosting_image