Merge branch 'master' into fix-integ-test-region

Author: iquintero

CHANGELOG.rst

@@ -6,6 +6,7 @@ CHANGELOG
 ========
 
 * feature: Tests: create configurable ``sagemaker_session`` pytest fixture for all integration tests
+* bug-fix: AmazonEstimators: fix inaccurate hyper-parameters in kmeans, pca and linear learner
 
 1.1.2
 =====

src/sagemaker/amazon/kmeans.py

@@ -13,7 +13,7 @@
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase, registry
 from sagemaker.amazon.common import numpy_to_record_serializer, record_deserializer
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
-from sagemaker.amazon.validation import gt, isin, ge
+from sagemaker.amazon.validation import gt, isin, ge, le
 from sagemaker.predictor import RealTimePredictor
 from sagemaker.model import Model
 from sagemaker.session import Session
@@ -27,16 +27,18 @@ class KMeans(AmazonAlgorithmEstimatorBase):
     k = hp('k', gt(1), 'An integer greater-than 1', int)
     init_method = hp('init_method', isin('random', 'kmeans++'), 'One of "random", "kmeans++"', str)
     max_iterations = hp('local_lloyd_max_iterations', gt(0), 'An integer greater-than 0', int)
-    tol = hp('local_lloyd_tol', gt(0), 'An integer greater-than 0', int)
+    tol = hp('local_lloyd_tol', (ge(0), le(1)), 'An float in [0, 1]', float)
     num_trials = hp('local_lloyd_num_trials', gt(0), 'An integer greater-than 0', int)
     local_init_method = hp('local_lloyd_init_method', isin('random', 'kmeans++'), 'One of "random", "kmeans++"', str)
     half_life_time_size = hp('half_life_time_size', ge(0), 'An integer greater-than-or-equal-to 0', int)
     epochs = hp('epochs', gt(0), 'An integer greater-than 0', int)
     center_factor = hp('extra_center_factor', gt(0), 'An integer greater-than 0', int)
+    eval_metrics = hp(name='eval_metrics', validation_message='A comma separated list of "msd" or "ssd"',
+                      data_type=list)
 
     def __init__(self, role, train_instance_count, train_instance_type, k, init_method=None,
                  max_iterations=None, tol=None, num_trials=None, local_init_method=None,
-                 half_life_time_size=None, epochs=None, center_factor=None, **kwargs):
+                 half_life_time_size=None, epochs=None, center_factor=None, eval_metrics=None, **kwargs):
         """
         A k-means clustering :class:`~sagemaker.amazon.AmazonAlgorithmEstimatorBase`. Finds k clusters of data in an
         unlabeled dataset.
@@ -70,7 +72,7 @@ def __init__(self, role, train_instance_count, train_instance_type, k, init_meth
             k (int): The number of clusters to produce.
             init_method (str): How to initialize cluster locations. One of 'random' or 'kmeans++'.
             max_iterations (int): Maximum iterations for Lloyds EM procedure in the local kmeans used in finalize stage.
-            tol (int): Tolerance for change in ssd for early stopping in local kmeans.
+            tol (float): Tolerance for change in ssd for early stopping in local kmeans.
             num_trials (int): Local version is run multiple times and the one with the best loss is chosen. This
                               determines how many times.
             local_init_method (str): Initialization method for local version. One of 'random', 'kmeans++'
@@ -82,6 +84,9 @@ def __init__(self, role, train_instance_count, train_instance_type, k, init_meth
             epochs (int): Number of passes done over the training data.
             center_factor(int): The algorithm will create ``num_clusters * extra_center_factor`` as it runs and
                 reduce the number of centers to ``k`` when finalizing
+            eval_metrics(list): JSON list of metrics types to be used for reporting the score for the model.
+                Allowed values are "msd" Means Square Error, "ssd": Sum of square distance. If test data is provided,
+                the score shall be reported in terms of all requested metrics.
             **kwargs: base class keyword argument values.
         """
         super(KMeans, self).__init__(role, train_instance_count, train_instance_type, **kwargs)
@@ -94,6 +99,7 @@ def __init__(self, role, train_instance_count, train_instance_type, k, init_meth
         self.half_life_time_size = half_life_time_size
         self.epochs = epochs
         self.center_factor = center_factor
+        self.eval_metrics = eval_metrics
 
     def create_model(self):
         """Return a :class:`~sagemaker.amazon.kmeans.KMeansModel` referencing the latest

src/sagemaker/amazon/linear_learner.py

@@ -32,7 +32,6 @@ class LinearLearner(AmazonAlgorithmEstimatorBase):
                                                     data_type=str)
     target_recall = hp('target_recall', (gt(0), lt(1)), "A float in (0,1)", float)
     target_precision = hp('target_precision', (gt(0), lt(1)), "A float in (0,1)", float)
-    positive_example_weight_mult = hp('positive_example_weight_mult', gt(0), "A float greater than 0", float)
     epochs = hp('epochs', gt(0), "An integer greater-than 0", int)
     predictor_type = hp('predictor_type', isin('binary_classifier', 'regressor'),
                         'One of "binary_classifier" or "regressor"', str)
@@ -64,9 +63,9 @@ class LinearLearner(AmazonAlgorithmEstimatorBase):
     unbias_label = hp('unbias_label', (), 'A boolean', bool)
     num_point_for_scaler = hp('num_point_for_scaler', gt(0), 'An integer greater-than 0', int)
 
-    def __init__(self, role, train_instance_count, train_instance_type, predictor_type='binary_classifier',
+    def __init__(self, role, train_instance_count, train_instance_type, predictor_type,
                  binary_classifier_model_selection_criteria=None, target_recall=None, target_precision=None,
-                 positive_example_weight_mult=None, epochs=None, use_bias=None, num_models=None,
+                 epochs=None, use_bias=None, num_models=None,
                  num_calibration_samples=None, init_method=None, init_scale=None, init_sigma=None, init_bias=None,
                  optimizer=None, loss=None, wd=None, l1=None, momentum=None, learning_rate=None, beta_1=None,
                  beta_2=None, bias_lr_mult=None, bias_wd_mult=None, use_lr_scheduler=None, lr_scheduler_step=None,
@@ -114,8 +113,6 @@ def __init__(self, role, train_instance_count, train_instance_type, predictor_ty
                 precision_at_target_recall.
             target_precision (float): Target precision. Only applicable if binary_classifier_model_selection_criteria
                 is recall_at_target_precision.
-            positive_example_weight_mult (float): The importance weight of positive examples is multiplied by this
-                constant. Useful for skewed datasets. Only applies for classification tasks.
             epochs (int): The maximum number of passes to make over the training data.
             use_bias (bool): Whether to include a bias field
             num_models (int): Number of models to train in parallel. If not set, the number of parallel models to
@@ -160,7 +157,6 @@ def __init__(self, role, train_instance_count, train_instance_type, predictor_ty
         self.binary_classifier_model_selection_criteria = binary_classifier_model_selection_criteria
         self.target_recall = target_recall
         self.target_precision = target_precision
-        self.positive_example_weight_mult = positive_example_weight_mult
         self.epochs = epochs
         self.use_bias = use_bias
         self.num_models = num_models

src/sagemaker/amazon/pca.py

@@ -13,6 +13,7 @@
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase, registry
 from sagemaker.amazon.common import numpy_to_record_serializer, record_deserializer
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
+from sagemaker.amazon.validation import gt, isin
 from sagemaker.predictor import RealTimePredictor
 from sagemaker.model import Model
 from sagemaker.session import Session
@@ -25,13 +26,13 @@ class PCA(AmazonAlgorithmEstimatorBase):
 
     DEFAULT_MINI_BATCH_SIZE = 500
 
-    num_components = hp(name='num_components', validate=lambda x: x > 0,
-                        validation_message='Value must be an integer greater than zero', data_type=int)
-    algorithm_mode = hp(name='algorithm_mode', validate=lambda x: x in ['regular', 'stable', 'randomized'],
-                        validation_message='Value must be one of "regular", "stable", "randomized"', data_type=str)
+    num_components = hp('num_components', gt(0), 'Value must be an integer greater than zero', int)
+    algorithm_mode = hp('algorithm_mode', isin('regular', 'randomized'),
+                        'Value must be one of "regular" and "randomized"', str)
     subtract_mean = hp(name='subtract_mean', validation_message='Value must be a boolean', data_type=bool)
-    extra_components = hp(name='extra_components', validate=lambda x: x >= 0,
-                          validation_message="Value must be an integer greater than or equal to 0", data_type=int)
+    extra_components = hp(name='extra_components',
+                          validation_message="Value must be an integer greater than or equal to 0, or -1.",
+                          data_type=int)
 
     def __init__(self, role, train_instance_count, train_instance_type, num_components,
                  algorithm_mode=None, subtract_mean=None, extra_components=None, **kwargs):
@@ -68,12 +69,13 @@ def __init__(self, role, train_instance_count, train_instance_type, num_componen
             train_instance_count (int): Number of Amazon EC2 instances to use for training.
             train_instance_type (str): Type of EC2 instance to use for training, for example, 'ml.c4.xlarge'.
             num_components(int): The number of principal components. Must be greater than zero.
-            algorithm_mode (str): Mode for computing the principal components. One of 'regular', 'stable' or
+            algorithm_mode (str): Mode for computing the principal components. One of 'regular' or
                 'randomized'.
             subtract_mean (bool): Whether the data should be unbiased both during train and at inference.
             extra_components (int): As the value grows larger, the solution becomes more accurate but the
-                runtime and memory consumption increase linearly. If this value is unset, then a default value equal
-                to the maximum of 10 and num_components will be used. Valid for randomized mode only.
+                runtime and memory consumption increase linearly. If this value is unset or set to -1,
+                then a default value equal to the maximum of 10 and num_components will be used.
+                Valid for randomized mode only.
             **kwargs: base class keyword argument values.
         """
         super(PCA, self).__init__(role, train_instance_count, train_instance_type, **kwargs)

src/sagemaker/tensorflow/estimator.py

@@ -10,8 +10,10 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import contextlib
 import logging
 import os
+import shutil
 import subprocess
 import tempfile
 import threading
@@ -46,6 +48,47 @@ def _cmd_exists(cmd):
             for path in os.environ["PATH"].split(os.pathsep)
         )
 
+    @staticmethod
+    def _sync_directories(from_directory, to_directory):
+        """Sync to_directory with from_directory by copying each file in
+        to_directory with new contents. Files in to_directory will be
+        overwritten by files of the same name in from_directory. We need to
+        keep two copies of the log directory because otherwise TensorBoard
+        picks up temp files from `aws s3 sync` and then stops reading the
+        correct tfevent files. We walk the directory and copy each file
+        individually because the directory that TensorBoard watches needs to
+        always exist.
+
+        Args:
+            from_directory (str): The directory with updated files.
+            to_directory (str): The directory to be synced.
+        """
+        if not os.path.exists(to_directory):
+            os.mkdir(to_directory)
+        for root, dirs, files in os.walk(from_directory):
+            to_root = root.replace(from_directory, to_directory)
+            for directory in dirs:
+                to_child_dir = os.path.join(to_root, directory)
+                if not os.path.exists(to_child_dir):
+                    os.mkdir(to_child_dir)
+            for fname in files:
+                from_file = os.path.join(root, fname)
+                to_file = os.path.join(to_root, fname)
+                with open(from_file, 'rb') as a, open(to_file, 'wb') as b:
+                    b.write(a.read())
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _temporary_directory():
+        """Context manager for a temporary directory. This is similar to
+        tempfile.TemporaryDirectory in python>=3.2.
+        """
+        name = tempfile.mkdtemp()
+        try:
+            yield name
+        finally:
+            shutil.rmtree(name)
+
     def validate_requirements(self):
         """Ensure that TensorBoard and the AWS CLI are installed.
 
@@ -96,10 +139,12 @@ def run(self):
         LOGGER.info('TensorBoard 0.1.7 at http://localhost:{}'.format(port))
         while not self.estimator.checkpoint_path:
             self.event.wait(1)
-        while not self.event.is_set():
-            args = ['aws', 's3', 'sync', self.estimator.checkpoint_path, self.logdir]
-            subprocess.call(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            self.event.wait(10)
+        with self._temporary_directory() as aws_sync_dir:
+            while not self.event.is_set():
+                args = ['aws', 's3', 'sync', self.estimator.checkpoint_path, aws_sync_dir]
+                subprocess.call(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                self._sync_directories(aws_sync_dir, self.logdir)
+                self.event.wait(10)
         tensorboard_process.terminate()
 
 

tests/integ/test_linear_learner.py

@@ -38,13 +38,11 @@ def test_linear_learner(sagemaker_session):
         train_set = train_set[0], train_set[1].astype(np.dtype('float32'))
 
         ll = LinearLearner('SageMakerRole', 1, 'ml.c4.2xlarge', base_job_name='test-linear-learner',
-                           sagemaker_session=sagemaker_session)
+                           predictor_type='binary_classifier', sagemaker_session=sagemaker_session)
         ll.binary_classifier_model_selection_criteria = 'accuracy'
         ll.target_recall = 0.5
         ll.target_precision = 0.5
-        ll.positive_example_weight_mult = 0.1
         ll.epochs = 1
-        ll.predictor_type = 'binary_classifier'
         ll.use_bias = True
         ll.num_models = 1
         ll.num_calibration_samples = 1
@@ -100,13 +98,11 @@ def test_async_linear_learner(sagemaker_session):
         train_set = train_set[0], train_set[1].astype(np.dtype('float32'))
 
         ll = LinearLearner('SageMakerRole', 1, 'ml.c4.2xlarge', base_job_name='test-linear-learner',
-                           sagemaker_session=sagemaker_session)
+                           predictor_type='binary_classifier', sagemaker_session=sagemaker_session)
         ll.binary_classifier_model_selection_criteria = 'accuracy'
         ll.target_recall = 0.5
         ll.target_precision = 0.5
-        ll.positive_example_weight_mult = 0.1
         ll.epochs = 1
-        ll.predictor_type = 'binary_classifier'
         ll.use_bias = True
         ll.num_models = 1
         ll.num_calibration_samples = 1

tests/unit/test_amazon_estimator.py

@@ -95,22 +95,6 @@ def test_data_location_does_not_call_default_bucket(sagemaker_session):
     assert not sagemaker_session.default_bucket.called
 
 
-def test_pca_hyperparameters(sagemaker_session):
-    pca = PCA(num_components=55, algorithm_mode='randomized',
-              subtract_mean=True, extra_components=33, sagemaker_session=sagemaker_session,
-              **COMMON_ARGS)
-    assert pca.hyperparameters() == dict(
-        num_components='55',
-        extra_components='33',
-        subtract_mean='True',
-        algorithm_mode='randomized')
-
-
-def test_image(sagemaker_session):
-    pca = PCA(num_components=55, sagemaker_session=sagemaker_session, **COMMON_ARGS)
-    assert pca.train_image() == registry('us-west-2') + '/pca:1'
-
-
 @patch('time.strftime', return_value=TIMESTAMP)
 def test_fit_ndarray(time, sagemaker_session):
     mock_s3 = Mock()