Merge branch 'master' into master

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## v2.39.0.post0 (2021-05-04)
+
+### Testing and Release Infrastructure
+
+ * disable smdataparallel tests
+
+## v2.39.0 (2021-04-28)
+
+### Features
+
+ * Add HF transformer version 4.5.0
+
+### Bug Fixes and Other Changes
+
+ * Allow hyperparameters in Tensorflow estimator to be parameterized
+
+### Testing and Release Infrastructure
+
+ * black format unit tests
+
+## v2.38.0 (2021-04-21)
+
+### Features
+
+ * support multiprocess feature group ingest (#2111)
+
+## v2.37.0 (2021-04-20)
+
+### Features
+
+ * add experiment_config for clarify processing job
+
+### Documentation Changes
+
+ * release notes for smdistributed.dataparallel v1.1.2
+
+## v2.36.0 (2021-04-19)
+
+### Features
+
+ * enable smdataparallel custom mpi options support
+
 ## v2.35.0 (2021-04-14)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.35.1.dev0
+2.39.1.dev0

doc/api/training/sdp_versions/latest.rst

@@ -1,5 +1,5 @@
 
-Version 1.1.1 (Latest)
+Version 1.1.2 (Latest)
 ======================
 
 .. toctree::

doc/api/training/smd_data_parallel_release_notes/smd_data_parallel_change_log.md

@@ -1,3 +1,29 @@
+# Sagemaker Distributed Data Parallel 1.2.0 Release Notes
+
+* New features
+* Bug Fixes
+
+*New features:*
+
+* Support of [EFA network interface](https://aws.amazon.com/hpc/efa/) for distributed AllReduce. For best performance, it is recommended you use an instance type that supports Amazon Elastic Fabric Adapter (ml.p3dn.24xlarge and ml.p4d.24xlarge) when you train a model using SageMaker distributed data parallel.
+
+*Bug Fixes:*
+
+* Improved performance on single node and small clusters.
+
+# Sagemaker Distributed Data Parallel 1.1.2 Release Notes
+
+* Bug Fixes
+* Known Issues
+
+*Bug Fixes:*
+
+* Fixed a bug that caused some TensorFlow operations to not work with certain data types. Operations forwarded from C++ have been extended to support every dtype supported by NCCL.
+
+*Known Issues:*
+
+* SageMaker distributed data parallel has slower throughput than NCCL when run using a single node. For the best performance, use multi-node distributed training with smdistributed.dataparallel. Use a single node only for experimental runs while preparing your training pipeline.
+
 # Sagemaker Distributed Data Parallel 1.1.1 Release Notes
 
 * New Features

setup.py

@@ -43,6 +43,7 @@ def read_version():
     "importlib-metadata>=1.4.0",
     "packaging>=20.0",
     "pandas",
+    "pathos",
 ]
 
 # Specific use case dependencies

src/sagemaker/clarify.py

@@ -403,6 +403,7 @@ def _run(
         logs,
         job_name,
         kms_key,
+        experiment_config,
     ):
         """Runs a ProcessingJob with the Sagemaker Clarify container and an analysis config.
 
@@ -415,6 +416,9 @@ def _run(
             job_name (str): Processing job name.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys:
+                'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
         """
         analysis_config["methods"]["report"] = {"name": "report", "title": "Analysis Report"}
         with tempfile.TemporaryDirectory() as tmpdirname:
@@ -457,6 +461,7 @@ def _run(
                 logs=logs,
                 job_name=job_name,
                 kms_key=kms_key,
+                experiment_config=experiment_config,
             )
 
     def run_pre_training_bias(
@@ -468,6 +473,7 @@ def run_pre_training_bias(
         logs=True,
         job_name=None,
         kms_key=None,
+        experiment_config=None,
     ):
         """Runs a ProcessingJob to compute the requested bias 'methods' of the input data.
 
@@ -487,13 +493,16 @@ def run_pre_training_bias(
                 "Clarify-Pretraining-Bias" and current timestamp.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys:
+                'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
         """
         analysis_config = data_config.get_config()
         analysis_config.update(data_bias_config.get_config())
         analysis_config["methods"] = {"pre_training_bias": {"methods": methods}}
         if job_name is None:
             job_name = utils.name_from_base("Clarify-Pretraining-Bias")
-        self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
+        self._run(data_config, analysis_config, wait, logs, job_name, kms_key, experiment_config)
 
     def run_post_training_bias(
         self,
@@ -506,6 +515,7 @@ def run_post_training_bias(
         logs=True,
         job_name=None,
         kms_key=None,
+        experiment_config=None,
     ):
         """Runs a ProcessingJob to compute the requested bias 'methods' of the model predictions.
 
@@ -532,6 +542,9 @@ def run_post_training_bias(
                 "Clarify-Posttraining-Bias" and current timestamp.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys:
+                'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
         """
         analysis_config = data_config.get_config()
         analysis_config.update(data_bias_config.get_config())
@@ -545,7 +558,7 @@ def run_post_training_bias(
         _set(probability_threshold, "probability_threshold", analysis_config)
         if job_name is None:
             job_name = utils.name_from_base("Clarify-Posttraining-Bias")
-        self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
+        self._run(data_config, analysis_config, wait, logs, job_name, kms_key, experiment_config)
 
     def run_bias(
         self,
@@ -559,6 +572,7 @@ def run_bias(
         logs=True,
         job_name=None,
         kms_key=None,
+        experiment_config=None,
     ):
         """Runs a ProcessingJob to compute the requested bias 'methods' of the model predictions.
 
@@ -589,6 +603,9 @@ def run_bias(
                 "Clarify-Bias" and current timestamp.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys:
+                'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
         """
         analysis_config = data_config.get_config()
         analysis_config.update(bias_config.get_config())
@@ -609,7 +626,7 @@ def run_bias(
         }
         if job_name is None:
             job_name = utils.name_from_base("Clarify-Bias")
-        self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
+        self._run(data_config, analysis_config, wait, logs, job_name, kms_key, experiment_config)
 
     def run_explainability(
         self,
@@ -621,6 +638,7 @@ def run_explainability(
         logs=True,
         job_name=None,
         kms_key=None,
+        experiment_config=None,
     ):
         """Runs a ProcessingJob computing for each example in the input the feature importance.
 
@@ -649,6 +667,9 @@ def run_explainability(
                 "Clarify-Explainability" and current timestamp.
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys:
+                'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
         """
         analysis_config = data_config.get_config()
         predictor_config = model_config.get_predictor_config()
@@ -657,7 +678,7 @@ def run_explainability(
         analysis_config["predictor"] = predictor_config
         if job_name is None:
             job_name = utils.name_from_base("Clarify-Explainability")
-        self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
+        self._run(data_config, analysis_config, wait, logs, job_name, kms_key, experiment_config)
 
 
 def _upload_analysis_config(analysis_config_file, s3_output_path, sagemaker_session, kms_key):

src/sagemaker/estimator.py

@@ -2443,7 +2443,13 @@ def attach(cls, training_job_name, sagemaker_session=None, model_channel_name="m
     @staticmethod
     def _json_encode_hyperparameters(hyperparameters):
         """Placeholder docstring"""
-        return {str(k): json.dumps(v) for (k, v) in hyperparameters.items()}
+        current_hyperparameters = hyperparameters
+        if current_hyperparameters is not None:
+            hyperparameters = {
+                str(k): (v if isinstance(v, (Parameter, Expression, Properties)) else json.dumps(v))
+                for (k, v) in current_hyperparameters.items()
+            }
+        return hyperparameters
 
     @classmethod
     def _update_init_params(cls, hp, tf_arguments):