Merge branch 'master' into user-context-fix

Author: icywang86rui

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v2.32.0 (2021-03-26)
+
+### Features
+
+ * upgrade neo mxnet to 1.8
+ * Enable Profiler in China Regions
+
+### Bug Fixes and Other Changes
+
+ * use workflow parameters in training hyperparameters (#2114) (#2115)
+ * skip HuggingFace tests in regions without p2 instances
+
+### Documentation Changes
+
+ * add Feature Store methods docs
+
 ## v2.31.1 (2021-03-23)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.31.2.dev0
+2.32.1.dev0

doc/api/prep_data/feature_store.rst

@@ -5,39 +5,70 @@ Feature group
 *************
 
 .. autoclass:: sagemaker.feature_store.feature_group.FeatureGroup
+    :members:
+    :exclude-members: load_feature_definitions
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_group.AthenaQuery
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_group.IngestionManagerPandas
+    :members:
+    :show-inheritance:
 
 
 Feature definition
 ******************
 
 .. autoclass:: sagemaker.feature_store.feature_definition.FeatureDefinition
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_definition.FractionalFeatureDefinition
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_definition.IntegralFeatureDefinition
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_definition.StringFeatureDefinition
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_definition.FeatureTypeEnum
+    :members:
+    :show-inheritance:
 
 
 Inputs
 ******
 
 .. autoclass:: sagemaker.feature_store.inputs.Config
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.DataCatalogConfig
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.OfflineStoreConfig
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.OnlineStoreConfig
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.OnlineStoreSecurityConfig
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.S3StorageConfig
+    :members:
+    :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.inputs.FeatureValue
+    :members:
+    :show-inheritance:

doc/frameworks/pytorch/using_pytorch.rst

@@ -154,7 +154,7 @@ directories ('train' and 'test').
     pytorch_estimator = PyTorch('pytorch-train.py',
                                 instance_type='ml.p3.2xlarge',
                                 instance_count=1,
-                                framework_version='1.5.0',
+                                framework_version='1.8.0',
                                 py_version='py3',
                                 hyperparameters = {'epochs': 20, 'batch-size': 64, 'learning-rate': 0.1})
     pytorch_estimator.fit({'train': 's3://my-data-bucket/path/to/my/training/data',
@@ -248,7 +248,7 @@ operation.
     pytorch_estimator = PyTorch(entry_point='train_and_deploy.py',
                                 instance_type='ml.p3.2xlarge',
                                 instance_count=1,
-                                framework_version='1.5.0',
+                                framework_version='1.8.0',
                                 py_version='py3')
     pytorch_estimator.fit('s3://my_bucket/my_training_data/')
 

src/sagemaker/estimator.py

@@ -52,6 +52,9 @@
     _region_supports_profiler,
     get_mp_parameters,
 )
+from sagemaker.workflow.properties import Properties
+from sagemaker.workflow.parameters import Parameter
+from sagemaker.workflow.entities import Expression
 from sagemaker.inputs import TrainingInput
 from sagemaker.job import _Job
 from sagemaker.local import LocalSession
@@ -1456,7 +1459,10 @@ def _get_train_args(cls, estimator, inputs, experiment_config):
 
         current_hyperparameters = estimator.hyperparameters()
         if current_hyperparameters is not None:
-            hyperparameters = {str(k): str(v) for (k, v) in current_hyperparameters.items()}
+            hyperparameters = {
+                str(k): (v if isinstance(v, (Parameter, Expression, Properties)) else str(v))
+                for (k, v) in current_hyperparameters.items()
+            }
 
         train_args = config.copy()
         train_args["input_mode"] = estimator.input_mode

src/sagemaker/fw_utils.py

@@ -59,8 +59,8 @@
     "local_gpu",
 )
 SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSIONS = {
-    "tensorflow": ["2.3.1", "2.3.2", "2.4.1"],
-    "pytorch": ["1.6.0", "1.7.1", "1.8.0"],
+    "tensorflow": ["2.3", "2.3.1", "2.3.2", "2.4", "2.4.1"],
+    "pytorch": ["1.6", "1.6.0", "1.7", "1.7.1", "1.8", "1.8.0"],
 }
 SMDISTRIBUTED_SUPPORTED_STRATEGIES = ["dataparallel", "modelparallel"]
 

src/sagemaker/image_uri_config/neo-mxnet.json

@@ -2,20 +2,21 @@
     "processors": ["cpu", "gpu"],
     "scope": ["inference"],
     "version_aliases": {
-        "0.12.1": "1.7",
-        "1.0.0": "1.7",
-        "1.1.0": "1.7",
-        "1.2": "1.7",
-        "1.2.0": "1.7",
-        "1.2.1": "1.7",
-        "1.3": "1.7",
-        "1.3.0": "1.7",
-        "1.4": "1.7",
-        "1.4.0": "1.7",
-        "1.4.1": "1.7"
+        "0.12.1": "1.8",
+        "1.0.0": "1.8",
+        "1.1.0": "1.8",
+        "1.2": "1.8",
+        "1.2.0": "1.8",
+        "1.2.1": "1.8",
+        "1.3": "1.8",
+        "1.3.0": "1.8",
+        "1.4": "1.8",
+        "1.4.0": "1.8",
+        "1.4.1": "1.8",
+        "1.7": "1.8"
     },
     "versions": {
-        "1.7": {
+        "1.8": {
             "py_versions": ["py3"],
             "registries": {
                 "af-south-1": "774647643957",

src/sagemaker/processing.py

@@ -32,6 +32,7 @@
 from sagemaker.session import Session
 from sagemaker.network import NetworkConfig  # noqa: F401 # pylint: disable=unused-import
 from sagemaker.workflow.properties import Properties
+from sagemaker.workflow.parameters import Parameter
 from sagemaker.workflow.entities import Expression
 from sagemaker.dataset_definition.inputs import S3Input, DatasetDefinition
 from sagemaker.apiutils._base_types import ApiObject
@@ -292,7 +293,9 @@ def _normalize_inputs(self, inputs=None, kms_key=None):
                 if isinstance(file_input.source, Properties) or file_input.dataset_definition:
                     normalized_inputs.append(file_input)
                     continue
-
+                if isinstance(file_input.s3_input.s3_uri, (Parameter, Expression, Properties)):
+                    normalized_inputs.append(file_input)
+                    continue
                 # If the source is a local path, upload it to S3
                 # and save the S3 uri in the ProcessingInput source.
                 parse_result = urlparse(file_input.s3_input.s3_uri)
@@ -340,8 +343,7 @@ def _normalize_outputs(self, outputs=None):
                 # Generate a name for the ProcessingOutput if it doesn't have one.
                 if output.output_name is None:
                     output.output_name = "output-{}".format(count)
-                # if the output's destination is a workflow expression, do no normalization
-                if isinstance(output.destination, Expression):
+                if isinstance(output.destination, (Parameter, Expression, Properties)):
                     normalized_outputs.append(output)
                     continue
                 # If the output's destination is not an s3_uri, create one.
@@ -1099,7 +1101,7 @@ def _create_s3_input(self):
             self.s3_data_type = self.s3_input.s3_data_type
             self.s3_input_mode = self.s3_input.s3_input_mode
             self.s3_data_distribution_type = self.s3_input.s3_data_distribution_type
-        elif self.source and self.destination:
+        elif self.source is not None and self.destination is not None:
             self.s3_input = S3Input(
                 s3_uri=self.source,
                 local_path=self.destination,

src/sagemaker/workflow/pipeline.py

@@ -83,7 +83,7 @@ def create(
 
         Args:
             role_arn (str): The role arn that is assumed by the pipeline to create step artifacts.
-            pipeline_description (str): A description of the pipeline.
+            description (str): A description of the pipeline.
             experiment_name (str): The name of the experiment.
             tags (List[Dict[str, str]]): A list of {"Key": "string", "Value": "string"} dicts as
                 tags.

tests/data/mxnet_mnist/mnist_neo.py

@@ -105,7 +105,7 @@ def train(
 
 
 def model_fn(path_to_model_files):
-    import neomxnet  # noqa: F401
+    import neomx  # noqa: F401
 
     ctx = mx.cpu()
     sym, arg_params, aux_params = mx.model.load_checkpoint(
@@ -120,7 +120,7 @@ def model_fn(path_to_model_files):
 
 
 def transform_fn(mod, payload, input_content_type, requested_output_content_type):
-    import neomxnet  # noqa: F401
+    import neomx  # noqa: F401
 
     if input_content_type != "application/vnd+python.numpy+binary":
         raise RuntimeError("Input content type must be application/vnd+python.numpy+binary")

tests/integ/test_huggingface.py

@@ -17,11 +17,15 @@
 import pytest
 
 from sagemaker.huggingface import HuggingFace
+from tests import integ
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout
 
 
 @pytest.mark.release
+@pytest.mark.skipif(
+    integ.test_region() in integ.TRAINING_NO_P2_REGIONS, reason="no ml.p2 instances in this region"
+)
 def test_huggingface_training(
     sagemaker_session,
     gpu_instance_type,

tests/integ/test_neo_mxnet.py

@@ -71,7 +71,7 @@ def test_attach_deploy(
 
         estimator.compile_model(
             target_instance_family=cpu_instance_family,
-            input_shape={"data": [1, 1, 28, 28], "softmax_label": [1]},
+            input_shape={"data": [1, 1, 28, 28]},
             output_path=estimator.output_path,
         )
 
@@ -121,7 +121,7 @@ def test_deploy_model(
 
         model.compile(
             target_instance_family=cpu_instance_family,
-            input_shape={"data": [1, 1, 28, 28], "softmax_label": [1]},
+            input_shape={"data": [1, 1, 28, 28]},
             role=role,
             job_name=unique_name_from_base("test-deploy-model-compilation-job"),
             output_path="/".join(model_data.split("/")[:-1]),
@@ -163,7 +163,7 @@ def test_inferentia_deploy_model(
 
         model.compile(
             target_instance_family=inf_instance_family,
-            input_shape={"data": [1, 1, 28, 28], "softmax_label": [1]},
+            input_shape={"data": [1, 1, 28, 28]},
             role=role,
             job_name=unique_name_from_base("test-deploy-model-compilation-job"),
             output_path="/".join(model_data.split("/")[:-1]),

tests/integ/test_tfs.py

@@ -161,6 +161,9 @@ def test_predict_with_accelerator(tfs_predictor_with_accelerator):
 
 
 @pytest.mark.local_mode
+@pytest.mark.skip(
+    reason="This test is broken due to a regression." "This test should be reenabled later."
+)
 def test_predict_with_entry_point(tfs_predictor_with_model_and_entry_point_same_tar):
     input_data = {"instances": [1.0, 2.0, 5.0]}
     expected_result = {"predictions": [4.0, 4.5, 6.0]}
@@ -170,6 +173,9 @@ def test_predict_with_entry_point(tfs_predictor_with_model_and_entry_point_same_
 
 
 @pytest.mark.local_mode
+@pytest.mark.skip(
+    reason="This test is broken due to a regression." "This test should be reenabled later."
+)
 def test_predict_with_model_and_entry_point_and_dependencies_separated(
     tfs_predictor_with_model_and_entry_point_and_dependencies,
 ):