Define and use HpoParameter for hyperparameters in 1P algorithms. (aws#5)

Author: lukmis

src/sagemaker/amazon/hyperparameter.py

@@ -10,6 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from sagemaker.hpo import _HpoParameter
 
 
 class Hyperparameter(object):
@@ -45,13 +46,35 @@ def validate(self, value):
                 raise ValueError(error_message)
 
     def __get__(self, obj, objtype):
-        """Return the value of this hyperparameter"""
+        """Return the value of this hyperparameter, whether it be a range or one discrete value"""
+        if '_hpo_parameters' in dir(obj) and self.name in obj._hpo_parameters:
+            return obj._hpo_parameters[self.name]
         if '_hyperparameters' not in dir(obj) or self.name not in obj._hyperparameters:
             raise AttributeError()
         return obj._hyperparameters[self.name]
 
     def __set__(self, obj, value):
+        """Assign values for hyperparameters"""
+        if isinstance(value, _HpoParameter):
+            self._set_hpo_parameter(obj, value)
+        else:
+            self._set_hyperparameter(obj, value)
+
+    def _set_hpo_parameter(self, obj, value):
+        # remove from the hyperparameters if it's there
+        if '_hyperparameters' in dir(obj) and self.name in obj._hyperparameters:
+            del obj._hyperparameters[self.name]
+
+        if '_hpo_parameters' not in dir(obj):
+            obj._hpo_parameters = dict()
+        obj._hpo_parameters[self.name] = value
+
+    def _set_hyperparameter(self, obj, value):
         """Validate the supplied value and set this hyperparameter to value"""
+        # remove from the hpo_parameters if it's there
+        if '_hpo_parameters' in dir(obj) and self.name in obj._hpo_parameters:
+            del obj._hpo_parameters[self.name]
+
         value = None if value is None else self.data_type(value)
         self.validate(value)
         if '_hyperparameters' not in dir(obj):
@@ -60,11 +83,34 @@ def __set__(self, obj, value):
 
     def __delete__(self, obj):
         """Delete this hyperparameter"""
-        del obj._hyperparameters[self.name]
+        if '_hyperparameters' in dir(obj) and self.name in obj._hyperparameters:
+            del obj._hyperparameters[self.name]
+        if '_hpo_parameters' in dir(obj) and self.name in obj._hpo_parameters:
+            del obj._hpo_parameters[self.name]
 
     @staticmethod
     def serialize_all(obj):
         """Return all non-None ``hyperparameter`` values on ``obj`` as a ``dict[str,str].``"""
         if '_hyperparameters' not in dir(obj):
             return {}
         return {k: str(v) for k, v in obj._hyperparameters.items() if v is not None}
+
+    @staticmethod
+    def serialize_all_hpo(obj):
+        """Return collections of ``ParameterRanges``
+
+        Args:
+            obj (object): 1P estimator that has hyperparameters defined
+
+        Returns:
+            Dictionary of ParameterRanges suitable for HPO tuning job.
+        """
+        if '_hpo_parameters' not in dir(obj):
+            obj._hpo_parameters = dict()
+
+        parameter_ranges = dict()
+        for range_type in _HpoParameter.__all_types__:
+            parameter_range = [param.as_hpo_range(p_name)
+                               for p_name, param in obj._hpo_parameters.items() if param.__name__ == range_type]
+            parameter_ranges[range_type+'ParameterRange'] = parameter_range
+        return parameter_ranges

src/sagemaker/hpo.py

@@ -0,0 +1,55 @@
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+class _HpoParameter(object):
+    __all_types__ = ['Continuous', 'Categorical', 'Integer']
+
+    def __init__(self, min_value, max_value):
+        self.min_value = min_value
+        self.max_value = max_value
+
+    def as_hpo_range(self, name):
+        return {'Name': name,
+                'Type': self.__name__,
+                'MinValue': str(self.min_value),
+                'MaxValue': str(self.max_value)}
+
+
+class ContinuousParameter(_HpoParameter):
+    __name__ = 'Continuous'
+
+    def __init__(self, min_value, max_value):
+        super(ContinuousParameter, self).__init__(min_value, max_value)
+
+
+class CategoricalParameter(_HpoParameter):
+    __name__ = 'Categorical'
+
+    def __init__(self, values):
+        if isinstance(values, list):
+            self.values = [str(v) for v in values]
+        else:
+            self.values = [str(values)]
+
+    def as_hpo_range(self, name):
+        return {'Name': name,
+                'Type': self.__name__,
+                'Values': self.values}
+
+
+class IntegerParameter(_HpoParameter):
+    __name__ = 'Integer'
+
+    def __init__(self, min_value, max_value):
+        super(IntegerParameter, self).__init__(min_value, max_value)

tests/unit/test_hpo.py

@@ -0,0 +1,75 @@
+# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from sagemaker.hpo import _HpoParameter, ContinuousParameter, IntegerParameter, CategoricalParameter
+
+
+def test_continuous_parameter():
+    cont_param = ContinuousParameter(0.1, 1e-2)
+    assert isinstance(cont_param, _HpoParameter)
+    assert cont_param.__name__ is 'Continuous'
+
+
+def test_continuous_parameter_hpo():
+    cont_param = ContinuousParameter(0.1, 1e-2)
+    hpo = cont_param.as_hpo_range('some')
+    assert len(hpo.keys()) == 4
+    assert hpo['Name'] == 'some'
+    assert hpo['Type'] == cont_param.__name__
+    assert hpo['MinValue'] == '0.1'
+    assert hpo['MaxValue'] == '0.01'  # min < max verification is performed by HPO
+
+
+def test_integer_parameter():
+    int_param = IntegerParameter(1, 2)
+    assert isinstance(int_param, _HpoParameter)
+    assert int_param.__name__ is 'Integer'
+
+
+def test_integer_parameter_hpo():
+    int_param = IntegerParameter(1, 2)
+    hpo = int_param.as_hpo_range('some')
+    assert len(hpo.keys()) == 4
+    assert hpo['Name'] == 'some'
+    assert hpo['Type'] == int_param.__name__
+    assert hpo['MinValue'] == '1'
+    assert hpo['MaxValue'] == '2'
+
+
+def test_categorical_parameter_list():
+    cat_param = CategoricalParameter(['a', 'z'])
+    assert isinstance(cat_param, _HpoParameter)
+    assert cat_param.__name__ is 'Categorical'
+
+
+def test_categorical_parameter__list_hpo():
+    cat_param = CategoricalParameter([1, 10])
+    hpo = cat_param.as_hpo_range('some')
+    assert len(hpo.keys()) == 3
+    assert hpo['Name'] == 'some'
+    assert hpo['Type'] == cat_param.__name__
+    assert hpo['Values'] == ['1', '10']
+
+
+def test_categorical_parameter_value():
+    cat_param = CategoricalParameter('a')
+    assert isinstance(cat_param, _HpoParameter)
+    assert cat_param.__name__ is 'Categorical'
+
+
+def test_categorical_parameter_value_hpo():
+    cat_param = CategoricalParameter('a')
+    hpo = cat_param.as_hpo_range('some')
+    assert len(hpo.keys()) == 3
+    assert hpo['Name'] == 'some'
+    assert hpo['Type'] == cat_param.__name__
+    assert hpo['Values'] == ['a']

tests/unit/test_hyperparameter.py

@@ -12,13 +12,14 @@
 # language governing permissions and limitations under the License.
 import pytest
 from sagemaker.amazon.hyperparameter import Hyperparameter
+from sagemaker.hpo import ContinuousParameter, IntegerParameter, CategoricalParameter
 
 
 class Test(object):
 
-    blank = Hyperparameter(name="some-name", data_type=int)
+    blank = Hyperparameter(name='some-name', data_type=int)
     elizabeth = Hyperparameter(name='elizabeth')
-    validated = Hyperparameter(name="validated", validate=lambda value: value > 55, data_type=int)
+    validated = Hyperparameter(name='validated', validate=lambda value: value > 55, data_type=int)
 
 
 def test_blank_access():
@@ -60,7 +61,7 @@ def test_validated():
 def test_data_type():
     x = Test()
     x.validated = 66
-    assert type(x.validated) == Test.__dict__["validated"].data_type
+    assert type(x.validated) == Test.__dict__['validated'].data_type
 
 
 def test_from_string():
@@ -72,3 +73,77 @@ def test_from_string():
 
     x.validated = from_api
     assert x.validated == value
+
+
+def test_serialize_hpo():
+    x = Test()
+
+    x.validated = ContinuousParameter(0, 5)
+    x.elizabeth = IntegerParameter(0, 5)
+    x.blank = CategoricalParameter([0, 5])
+
+    assert isinstance(x.validated, ContinuousParameter)
+    assert isinstance(x.elizabeth, IntegerParameter)
+    assert isinstance(x.blank, CategoricalParameter)
+
+    hpo_ranges = Hyperparameter.serialize_all_hpo(x)
+    assert hpo_ranges[x.validated.__name__+'ParameterRange'][0]['Name'] == 'validated'
+    assert hpo_ranges[x.elizabeth.__name__+'ParameterRange'][0]['Name'] == 'elizabeth'
+    assert hpo_ranges[x.blank.__name__+'ParameterRange'][0]['Name'] == 'some-name'
+
+
+def test_assign_to_hpo():
+    x = Test()
+    x.validated = 66
+
+    assert Hyperparameter.serialize_all(x)['validated']
+    hpo_ranges = Hyperparameter.serialize_all_hpo(x)
+    assert len(hpo_ranges.keys()) == 3
+    assert [len(hpo_ranges[r]) for r in hpo_ranges.keys()] == [0, 0, 0]
+
+    x.validated = ContinuousParameter(0, 5)
+    assert isinstance(x.validated, ContinuousParameter)
+    assert Hyperparameter.serialize_all(x) == {}
+    hpo_ranges = Hyperparameter.serialize_all_hpo(x)
+    assert len(hpo_ranges) == 3
+    assert hpo_ranges[x.validated.__name__+'ParameterRange'][0]['Name'] == 'validated'
+
+    # hp is preserved, necessary when attaching to a training job
+    assert int == Test.__dict__['validated'].data_type
+
+
+def test_assign_from_hpo():
+    x = Test()
+
+    x.validated = IntegerParameter(0, 5)
+    x.validated = 67
+
+    assert Hyperparameter.serialize_all(x)['validated']
+    hpo_ranges = Hyperparameter.serialize_all_hpo(x)
+    assert len(hpo_ranges.keys()) == 3
+    assert [len(hpo_ranges[r]) for r in hpo_ranges.keys()] == [0, 0, 0]
+
+    assert type(x.validated) == Test.__dict__['validated'].data_type
+
+
+def test_hpo_assign_none():
+    x = Test()
+
+    x.validated = ContinuousParameter(0, 5)
+    x.validated = None
+
+    assert Hyperparameter.serialize_all(x) == {}
+    hpo_ranges = Hyperparameter.serialize_all_hpo(x)
+    assert len(hpo_ranges.keys()) == 3
+    assert [len(hpo_ranges[r]) for r in hpo_ranges.keys()] == [0, 0, 0]
+
+    assert int == Test.__dict__['validated'].data_type
+
+
+def test_delete_in_hpo():
+    x = Test()
+    x.validated = ContinuousParameter(0, 5)
+    del(x.validated)
+
+    with pytest.raises(AttributeError):
+        x.validated