Merge branch 'master' into remove_cw_metrics_arg

Author: iquintero

CHANGELOG.rst

@@ -2,7 +2,19 @@
 CHANGELOG
 =========
 
-1.7.1dev
+1.8.0
+=====
+
+* bug-fix: removing PCA from tuner
+* feature: Estimators: add support for Amazon k-nearest neighbors(KNN) algorithm
+
+1.7.2
+=====
+
+* bug-fix: Prediction output for the TF_JSON_SERIALIZER
+* enhancement: Add better training job status report
+
+1.7.1
 =====
 
 * bug-fix: get_execution_role no longer fails if user can't call get_role

README.rst

@@ -51,7 +51,12 @@ You can install from source by cloning this repository and issuing a pip install
 
     git clone https://github.com/aws/sagemaker-python-sdk.git
     python setup.py sdist
-    pip install dist/sagemaker-1.7.0.tar.gz
+    pip install dist/sagemaker-1.8.0.tar.gz
+
+Supported Operating Systems
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SageMaker Python SDK supports Unix/Linux and Mac.
 
 Supported Python versions
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -290,7 +295,7 @@ Amazon SageMaker provides several built-in machine learning algorithms that you
 
 The full list of algorithms is available on the AWS website: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
 
-SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM) and Random Cut Forest algorithms.
+SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM) Random Cut Forest and k-nearest neighbors (k-NN) algorithms.
 
 More details at `AWS SageMaker Estimators and Models`_.
 

setup.py

@@ -23,7 +23,7 @@ def read(fname):
 
 
 setup(name="sagemaker",
-      version="1.7.0",
+      version="1.8.0",
       description="Open source library for training and deploying models on Amazon SageMaker.",
       packages=find_packages('src'),
       package_dir={'': 'src'},
@@ -45,7 +45,7 @@ def read(fname):
 
       # Declare minimal set for installation
       install_requires=['boto3>=1.4.8', 'numpy>=1.9.0', 'protobuf>=3.1', 'scipy>=0.19.0', 'urllib3>=1.2',
-                        'PyYAML>=3.2'],
+                        'PyYAML>=3.2', 'protobuf3-to-dict>=0.1.5'],
 
       extras_require={
           'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist',

src/sagemaker/__init__.py

@@ -22,6 +22,7 @@
 from sagemaker.amazon.ntm import NTM, NTMModel, NTMPredictor  # noqa: F401
 from sagemaker.amazon.randomcutforest import (RandomCutForest, RandomCutForestModel,  # noqa: F401
                                               RandomCutForestPredictor)
+from sagemaker.amazon.knn import KNN, KNNModel, KNNPredictor  # noqa: F401
 
 from sagemaker.analytics import TrainingJobAnalytics, HyperparameterTuningJobAnalytics  # noqa: F401
 from sagemaker.local.local_session import LocalSession  # noqa: F401

src/sagemaker/amazon/README.rst

@@ -7,7 +7,7 @@ Amazon SageMaker provides several built-in machine learning algorithms that you
 
 The full list of algorithms is available on the AWS website: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
 
-SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM) and Random Cut Forest algorithms.
+SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM), Random Cut Forest algorithms and k-nearest neighbors (k-NN).
 
 Definition and usage
 ~~~~~~~~~~~~~~~~~~~~

src/sagemaker/amazon/amazon_estimator.py

@@ -278,7 +278,7 @@ def registry(region_name, algorithm=None):
     https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon
     """
     if algorithm in [None, "pca", "kmeans", "linear-learner", "factorization-machines", "ntm",
-                     "randomcutforest"]:
+                     "randomcutforest", "knn"]:
         account_id = {
             "us-east-1": "382416733822",
             "us-east-2": "404615174143",

src/sagemaker/amazon/knn.py

@@ -0,0 +1,142 @@
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase, registry
+from sagemaker.amazon.common import numpy_to_record_serializer, record_deserializer
+from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
+from sagemaker.amazon.validation import ge, isin
+from sagemaker.predictor import RealTimePredictor
+from sagemaker.model import Model
+from sagemaker.session import Session
+
+
+class KNN(AmazonAlgorithmEstimatorBase):
+    repo_name = 'knn'
+    repo_version = 1
+
+    k = hp('k', (ge(1)), 'An integer greater than 0', int)
+    sample_size = hp('sample_size', (ge(1)), 'An integer greater than 0', int)
+    predictor_type = hp('predictor_type', isin('classifier', 'regressor'),
+                        'One of "classifier" or "regressor"', str)
+    dimension_reduction_target = hp('dimension_reduction_target', (ge(1)),
+                                    'An integer greater than 0 and less than feature_dim', int)
+    dimension_reduction_type = hp('dimension_reduction_type', isin('sign', 'fjlt'), 'One of "sign" or "fjlt"', str)
+    index_metric = hp('index_metric', isin('COSINE', 'INNER_PRODUCT', 'L2'),
+                      'One of "COSINE", "INNER_PRODUCT", "L2"', str)
+    index_type = hp('index_type', isin('faiss.Flat', 'faiss.IVFFlat', 'faiss.IVFPQ'),
+                    'One of "faiss.Flat", "faiss.IVFFlat", "faiss.IVFPQ"', str)
+    faiss_index_ivf_nlists = hp('faiss_index_ivf_nlists', (), '"auto" or an integer greater than 0', str)
+    faiss_index_pq_m = hp('faiss_index_pq_m', (ge(1)), 'An integer greater than 0', int)
+
+    def __init__(self, role, train_instance_count, train_instance_type, k, sample_size, predictor_type,
+                 dimension_reduction_type=None, dimension_reduction_target=None, index_type=None,
+                 index_metric=None, faiss_index_ivf_nlists=None, faiss_index_pq_m=None, **kwargs):
+        """k-nearest neighbors (KNN) is :class:`Estimator` used for classification and regression.
+
+        This Estimator may be fit via calls to
+        :meth:`~sagemaker.amazon.amazon_estimator.AmazonAlgorithmEstimatorBase.fit`. It requires Amazon
+        :class:`~sagemaker.amazon.record_pb2.Record` protobuf serialized data to be stored in S3.
+        There is an utility :meth:`~sagemaker.amazon.amazon_estimator.AmazonAlgorithmEstimatorBase.record_set` that
+        can be used to upload data to S3 and creates :class:`~sagemaker.amazon.amazon_estimator.RecordSet` to be passed
+        to the `fit` call.
+
+        To learn more about the Amazon protobuf Record class and how to prepare bulk data in this format, please
+        consult AWS technical documentation: https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html
+
+        After this Estimator is fit, model data is stored in S3. The model may be deployed to an Amazon SageMaker
+        Endpoint by invoking :meth:`~sagemaker.amazon.estimator.EstimatorBase.deploy`. As well as deploying an Endpoint,
+        deploy returns a :class:`~sagemaker.amazon.knn.KNNPredictor` object that can be used
+        for inference calls using the trained model hosted in the SageMaker Endpoint.
+
+        KNN Estimators can be configured by setting hyperparameters. The available hyperparameters for
+        KNN are documented below.
+
+        For further information on the AWS KNN algorithm,
+        please consult AWS technical documentation: https://docs.aws.amazon.com/sagemaker/latest/dg/knn.html
+
+        Args:
+            role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training jobs and
+                APIs that create Amazon SageMaker endpoints use this role to access
+                training data and model artifacts. After the endpoint is created,
+                the inference code might use the IAM role, if accessing AWS resource.
+            train_instance_type (str): Type of EC2 instance to use for training, for example, 'ml.c4.xlarge'.
+            k (int): Required. Number of nearest neighbors.
+            sample_size(int): Required. Number of data points to be sampled from the training data set.
+            predictor_type (str): Required. Type of inference to use on the data's labels,
+                allowed values are 'classifier' and 'regressor'.
+            dimension_reduction_type (str): Optional. Type of dimension reduction technique to use.
+                Valid values: “sign”, “fjlt”
+            dimension_reduction_target (int): Optional. Target dimension to reduce to. Required when
+                dimension_reduction_type is specified.
+            index_type (str): Optional. Type of index to use. Valid values are
+                “faiss.Flat”, “faiss.IVFFlat”, “faiss.IVFPQ”.
+            index_metric(str): Optional. Distance metric to measure between points when finding nearest neighbors.
+                Valid values are "COSINE", "INNER_PRODUCT", "L2"
+            faiss_index_ivf_nlists(str): Optional. Number of centroids to construct in the index if
+                index_type is “faiss.IVFFlat” or “faiss.IVFPQ”.
+            faiss_index_pq_m(int): Optional. Number of vector sub-components to construct in the index,
+                if index_type is “faiss.IVFPQ”.
+            **kwargs: base class keyword argument values.
+        """
+
+        super(KNN, self).__init__(role, train_instance_count, train_instance_type, **kwargs)
+        self.k = k
+        self.sample_size = sample_size
+        self.predictor_type = predictor_type
+        self.dimension_reduction_type = dimension_reduction_type
+        self.dimension_reduction_target = dimension_reduction_target
+        self.index_type = index_type
+        self.index_metric = index_metric
+        self.faiss_index_ivf_nlists = faiss_index_ivf_nlists
+        self.faiss_index_pq_m = faiss_index_pq_m
+        if dimension_reduction_type and not dimension_reduction_target:
+            raise ValueError('"dimension_reduction_target" is required when "dimension_reduction_type" is set.')
+
+    def create_model(self):
+        """Return a :class:`~sagemaker.amazon.KNNModel` referencing the latest
+        s3 model data produced by this Estimator."""
+
+        return KNNModel(self.model_data, self.role, sagemaker_session=self.sagemaker_session)
+
+    def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
+        super(KNN, self)._prepare_for_training(records, mini_batch_size=mini_batch_size, job_name=job_name)
+
+
+class KNNPredictor(RealTimePredictor):
+    """Performs classification or regression prediction from input vectors.
+
+    The implementation of :meth:`~sagemaker.predictor.RealTimePredictor.predict` in this
+    `RealTimePredictor` requires a numpy ``ndarray`` as input. The array should contain the
+    same number of columns as the feature-dimension of the data used to fit the model this
+    Predictor performs inference on.
+
+    :func:`predict` returns a list of :class:`~sagemaker.amazon.record_pb2.Record` objects, one
+    for each row in the input ``ndarray``. The prediction is stored in the ``"predicted_label"``
+    key of the ``Record.label`` field."""
+
+    def __init__(self, endpoint, sagemaker_session=None):
+        super(KNNPredictor, self).__init__(endpoint, sagemaker_session, serializer=numpy_to_record_serializer(),
+                                           deserializer=record_deserializer())
+
+
+class KNNModel(Model):
+    """Reference S3 model data created by KNN estimator. Calling :meth:`~sagemaker.model.Model.deploy`
+    creates an Endpoint and returns :class:`KNNPredictor`."""
+
+    def __init__(self, model_data, role, sagemaker_session=None):
+        sagemaker_session = sagemaker_session or Session()
+        repo = '{}:{}'.format(KNN.repo_name, KNN.repo_version)
+        image = '{}/{}'.format(registry(sagemaker_session.boto_session.region_name, KNN.repo_name), repo)
+        super(KNNModel, self).__init__(model_data, image, role, predictor_cls=KNNPredictor,
+                                       sagemaker_session=sagemaker_session)

src/sagemaker/amazon/linear_learner.py

@@ -247,7 +247,7 @@ def __init__(self, role, train_instance_count, train_instance_type, predictor_ty
                 "For predictor_type 'multiclass_classifier', 'num_classes' should be set to a value greater than 2.")
 
     def create_model(self):
-        """Return a :class:`~sagemaker.amazon.kmeans.LinearLearnerModel` referencing the latest
+        """Return a :class:`~sagemaker.amazon.LinearLearnerModel` referencing the latest
         s3 model data produced by this Estimator."""
 
         return LinearLearnerModel(self.model_data, self.role, self.sagemaker_session)

src/sagemaker/chainer/README.rst

@@ -145,7 +145,7 @@ Optional arguments
 The following are optional arguments. When you create a ``Chainer`` object, you can specify these as keyword arguments.
 
 -  ``source_dir`` Path (absolute or relative) to a directory with any
-   other training source code dependencies aside from the entry point
+   other training source code dependencies including the entry point
    file. Structure within this directory will be preserved when training
    on SageMaker.
 -  ``hyperparameters`` Hyperparameters that will be used for training.
@@ -574,7 +574,7 @@ The ChainerModel constructor takes the following arguments:
 -  ``entry_point (str):`` Path (absolute or relative) to the Python file
    which should be executed as the entry point to model hosting.
 -  ``source_dir (str):`` Optional. Path (absolute or relative) to a
-   directory with any other training source code dependencies aside from
+   directory with any other training source code dependencies including
    tne entry point file. Structure within this directory will be
    preserved when training on SageMaker.
 -  ``enable_cloudwatch_metrics (boolean):`` Optional. If true, training

src/sagemaker/mxnet/README.rst

@@ -123,7 +123,7 @@ Optional arguments
 The following are optional arguments. When you create an ``MXNet`` object, you can specify these as keyword arguments.
 
 -  ``source_dir`` Path (absolute or relative) to a directory with any
-   other training source code dependencies aside from the entry point
+   other training source code dependencies including the entry point
    file. Structure within this directory will be preserved when training
    on SageMaker.
 -  ``hyperparameters`` Hyperparameters that will be used for training.
@@ -171,7 +171,7 @@ Required argument
 -  ``inputs``: This can take one of the following forms: A string
    s3 URI, for example ``s3://my-bucket/my-training-data``. In this
    case, the s3 objects rooted at the ``my-training-data`` prefix will
-   be available in the default ``train`` channel. A dict from
+   be available in the default ``training`` channel. A dict from
    string channel names to s3 URIs. In this case, the objects rooted at
    each s3 prefix will available as files in each channel directory.
 
@@ -540,7 +540,7 @@ The MXNetModel constructor takes the following arguments:
 -  ``entry_point (str):`` Path (absolute or relative) to the Python file
    which should be executed as the entry point to model hosting.
 -  ``source_dir (str):`` Optional. Path (absolute or relative) to a
-   directory with any other training source code dependencies aside from
+   directory with any other training source code dependencies including
    tne entry point file. Structure within this directory will be
    preserved when training on SageMaker.
 -  ``container_log_level (int):`` Log level to use within the container.

src/sagemaker/pytorch/README.rst

@@ -171,7 +171,7 @@ Optional arguments
 The following are optional arguments. When you create a ``PyTorch`` object, you can specify these as keyword arguments.
 
 -  ``source_dir`` Path (absolute or relative) to a directory with any
-   other training source code dependencies aside from the entry point
+   other training source code dependencies including the entry point
    file. Structure within this directory will be preserved when training
    on SageMaker.
 -  ``hyperparameters`` Hyperparameters that will be used for training.
@@ -606,7 +606,7 @@ The PyTorchModel constructor takes the following arguments:
 -  ``entry_point:`` Path (absolute or relative) to the Python file
    which should be executed as the entry point to model hosting.
 -  ``source_dir:`` Optional. Path (absolute or relative) to a
-   directory with any other training source code dependencies aside from
+   directory with any other training source code dependencies including
    tne entry point file. Structure within this directory will be
    preserved when training on SageMaker.
 -  ``enable_cloudwatch_metrics:`` Optional. If true, training