Add hyperparameter range validation (aws#31)

Author: ChoiByungWook

src/sagemaker/tuner.py

@@ -12,6 +12,8 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
+import inspect
+
 from sagemaker.job import _Job
 from sagemaker.utils import base_name_from_image, name_from_base
 
@@ -64,16 +66,18 @@ def __init__(self, estimator, objective_metric_name, hyperparameter_ranges, metr
                  objective_type='Maximize', max_jobs=1, max_parallel_jobs=1, base_tuning_job_name=None):
         if objective_type not in HyperparameterTuner.__objectives__:
             raise ValueError("Unsupported 'objective' values")
+
         self.estimator = estimator
-        self.metric_name = objective_metric_name
+        self.objective_metric_name = objective_metric_name
         self._hyperparameter_ranges = hyperparameter_ranges
         self.strategy = strategy
-        self.objective = objective_type
+        self.objective_type = objective_type
         self.max_jobs = max_jobs
         self.max_parallel_jobs = max_parallel_jobs
         self.tuning_job_name = base_tuning_job_name
         self.metric_definitions = metric_definitions
         self.latest_tuning_job = None
+        self._validate_parameter_ranges()
 
     def fit(self, inputs):
         """Create tuning job
@@ -108,6 +112,30 @@ def hyperparameter_ranges(self):
             hyperparameter_ranges[range_type + 'ParameterRanges'] = parameter_range
         return hyperparameter_ranges
 
+    def _validate_parameter_ranges(self):
+        from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
+
+        for kls in inspect.getmro(self.estimator.__class__)[::-1]:
+            for attribute, value in kls.__dict__.items():
+                if isinstance(value, hp):
+                    try:
+                        # The hyperparam names may not be the same as the class attribute that holds them,
+                        # for instance: local_lloyd_init_method is called local_init_method. We need to map these
+                        # and pass the correct name to the constructor.
+                        parameter_range = self._hyperparameter_ranges[value.name]
+
+                        if isinstance(parameter_range, _ParameterRange):
+                            for parameter_range_attribute, parameter_range_value in parameter_range.__dict__.items():
+                                # Categorical ranges
+                                if isinstance(parameter_range_value, list):
+                                    for categorical_value in parameter_range_value:
+                                        value.validate(categorical_value)
+                                # Continuous, Integer ranges
+                                else:
+                                    value.validate(parameter_range_value)
+                    except KeyError:
+                        pass
+
 
 class _TuningJob(_Job):
     SAGEMAKER_ESTIMATOR_CLASS_NAME = 'sagemaker_estimator_class_name'
@@ -140,7 +168,7 @@ def start_new(cls, tuner, inputs):
         tuning_job_name = name_from_base(base_name)
 
         tuner.estimator.sagemaker_session.tune(job_name=tuning_job_name, strategy=tuner.strategy,
-                                               objective=tuner.objective, metric_name=tuner.metric_name,
+                                               objective=tuner.objective_type, metric_name=tuner.objective_metric_name,
                                                max_jobs=tuner.max_jobs, max_parallel_jobs=tuner.max_parallel_jobs,
                                                parameter_ranges=tuner.hyperparameter_ranges(),
                                                static_hp=static_hyperparameters,

tests/unit/test_tuner.py

@@ -59,6 +59,32 @@ def tuner(estimator):
                                hyperparameter_ranges=HYPERPARAMETER_RANGES, metric_definitions=METRIC_DEFINTIONS)
 
 
+def test_validate_parameter_ranges_number_validation_error(sagemaker_session):
+    pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
+              base_job_name='pca', sagemaker_session=sagemaker_session)
+
+    invalid_hyperparameter_ranges = {'num_components': IntegerParameter(-1, 2)}
+
+    with pytest.raises(ValueError) as e:
+        HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME,
+                            hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS)
+
+    assert 'Value must be an integer greater than zero' in str(e)
+
+
+def test_validate_parameter_ranges_string_value_validation_error(sagemaker_session):
+    pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
+              base_job_name='pca', sagemaker_session=sagemaker_session)
+
+    invalid_hyperparameter_ranges = {'algorithm_mode': CategoricalParameter([0, 5])}
+
+    with pytest.raises(ValueError) as e:
+        HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME,
+                            hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS)
+
+    assert 'Value must be one of "regular" and "randomized"' in str(e)
+
+
 def test_tune_1p(sagemaker_session, tuner):
     pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
               base_job_name='pca', sagemaker_session=sagemaker_session)
@@ -67,18 +93,18 @@ def test_tune_1p(sagemaker_session, tuner):
     pca.subtract_mean = True
     pca.extra_components = 5
 
-    hyperparameter_ranges = {'num_components': IntegerParameter(2, 4)}
+    hyperparameter_ranges = {'num_components': IntegerParameter(2, 4),
+                             'algorithm_mode': CategoricalParameter(['regular', 'randomized'])}
     tuner.estimator = pca
     tuner._hyperparameter_ranges = hyperparameter_ranges
 
     tuner.fit(INPUTS)
 
     _, _, tune_kwargs = sagemaker_session.tune.mock_calls[0]
 
-    assert len(tune_kwargs['static_hp']) == 5
+    assert len(tune_kwargs['static_hp']) == 4
     assert tune_kwargs['static_hp']['sagemaker_estimator_class_name'] == pca.__class__.__name__
-    assert tune_kwargs['static_hp']['sagemaker_estimator_module'] == pca.__module__
-    assert tune_kwargs['static_hp']['algorithm_mode'] == 'randomized'
+    assert tune_kwargs['static_hp']['extra_components'] == '5'
     assert len(tune_kwargs['parameter_ranges']['IntegerParameterRanges']) == 1
     assert tune_kwargs['job_name'].startswith('pca')