Merge branch 'master' into clarify-acc

Author: milah

CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## v2.33.0 (2021-04-05)
+
+### Features
+
+ * Add environment variable support for SageMaker training job
+
+### Bug Fixes and Other Changes
+
+ * add version length mismatch validation for HuggingFace
+ * Disable debugger when checkpointing is enabled with distributed training
+ * map user context is list associations response
+
+### Testing and Release Infrastructure
+
+ * disable_profiler on mx-horovod test
+
+## v2.32.1 (2021-04-01)
+
+### Bug Fixes and Other Changes
+
+ * disable profiler in some release tests
+ * remove outdated notebook from test
+ * add compilation option for ml_eia2
+ * add short version to smdataparallel supported list
+
+### Documentation Changes
+
+ * creating a "latest" version sm distributed docs
+ * add docs for Sagemaker Model Parallel 1.3, released with PT 1.8
+ * update PyTorch version in doc
+
 ## v2.32.0 (2021-03-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.32.1.dev0
+2.33.1.dev0

src/sagemaker/estimator.py

@@ -123,6 +123,7 @@ def __init__(
         enable_network_isolation=False,
         profiler_config=None,
         disable_profiler=False,
+        environment=None,
         **kwargs,
     ):
         """Initialize an ``EstimatorBase`` instance.
@@ -266,6 +267,8 @@ def __init__(
                 ``disable_profiler`` parameter to ``True``.
             disable_profiler (bool): Specifies whether Debugger monitoring and profiling
                 will be disabled (default: ``False``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         """
         instance_count = renamed_kwargs(
@@ -352,6 +355,8 @@ def __init__(
         self.profiler_config = profiler_config
         self.disable_profiler = disable_profiler
 
+        self.environment = environment
+
         if not _region_supports_profiler(self.sagemaker_session.boto_region_name):
             self.disable_profiler = True
 
@@ -1471,6 +1476,7 @@ def _get_train_args(cls, estimator, inputs, experiment_config):
         train_args["tags"] = estimator.tags
         train_args["metric_definitions"] = estimator.metric_definitions
         train_args["experiment_config"] = experiment_config
+        train_args["environment"] = estimator.environment
 
         if isinstance(inputs, TrainingInput):
             if "InputMode" in inputs.config:
@@ -1659,6 +1665,7 @@ def __init__(
         enable_sagemaker_metrics=None,
         profiler_config=None,
         disable_profiler=False,
+        environment=None,
         **kwargs,
     ):
         """Initialize an ``Estimator`` instance.
@@ -1807,6 +1814,8 @@ def __init__(
                 ``disable_profiler`` parameter to ``True``.
             disable_profiler (bool): Specifies whether Debugger monitoring and profiling
                 will be disabled (default: ``False``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
         """
         self.image_uri = image_uri
         self.hyperparam_dict = hyperparameters.copy() if hyperparameters else {}
@@ -1840,6 +1849,7 @@ def __init__(
             enable_network_isolation=enable_network_isolation,
             profiler_config=profiler_config,
             disable_profiler=disable_profiler,
+            environment=environment,
             **kwargs,
         )
 
@@ -2209,7 +2219,21 @@ def _validate_and_set_debugger_configs(self):
         ):
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
         elif not self.debugger_hook_config:
-            self.debugger_hook_config = None
+            # set hook config to False if _region_supports_debugger is False
+            self.debugger_hook_config = False
+
+        # Disable debugger if checkpointing is enabled by the customer
+        if self.checkpoint_s3_uri and self.checkpoint_local_path and self.debugger_hook_config:
+            if self._framework_name in {"mxnet", "pytorch", "tensorflow"}:
+                if self.instance_count > 1 or (
+                    hasattr(self, "distribution")
+                    and self.distribution is not None  # pylint: disable=no-member
+                ):
+                    logger.info(
+                        "SMDebug Does Not Currently Support \
+                        Distributed Training Jobs With Checkpointing Enabled"
+                    )
+                    self.debugger_hook_config = False
 
     def _stage_user_code_in_s3(self):
         """Upload the user training script to s3 and return the location.

src/sagemaker/huggingface/estimator.py

@@ -189,6 +189,7 @@ def _validate_args(self, image_uri):
         """Placeholder docstring"""
         if image_uri is not None:
             return
+
         if self.framework_version is None and image_uri is None:
             raise ValueError(
                 "transformers_version, and image_uri are both None. "
@@ -204,6 +205,17 @@ def _validate_args(self, image_uri):
                 "tensorflow_version and pytorch_version are both None. "
                 "Specify either tensorflow_version or pytorch_version."
             )
+        base_framework_version_len = (
+            len(self.tensorflow_version.split("."))
+            if self.tensorflow_version is not None
+            else len(self.pytorch_version.split("."))
+        )
+        transformers_version_len = len(self.framework_version.split("."))
+        if transformers_version_len != base_framework_version_len:
+            raise ValueError(
+                "Please use either full version or shortened version for both "
+                "transformers_version, tensorflow_version and pytorch_version."
+            )
 
     def hyperparameters(self):
         """Return hyperparameters used by your custom PyTorch code during model training."""

src/sagemaker/lineage/_api_types.py

@@ -181,6 +181,36 @@ class ContextSummary(_base_types.ApiObject):
     last_modified_time = None
 
 
+class UserContext(_base_types.ApiObject):
+    """Summary model of a user context.
+
+    Attributes:
+        user_profile_arn (str): User profile ARN.
+        user_profile_name (str): User profile name.
+        domain_id (str): DomainId.
+    """
+
+    user_profile_arn = None
+    user_profile_name = None
+    domain_id = None
+
+    def __init__(self, user_profile_arn=None, user_profile_name=None, domain_id=None, **kwargs):
+        """Initialize UserContext.
+
+        Args:
+            user_profile_arn (str): User profile ARN.
+            user_profile_name (str): User profile name.
+            domain_id (str): DomainId.
+            **kwargs: Arbitrary keyword arguments.
+        """
+        super(UserContext, self).__init__(
+            user_profile_arn=user_profile_arn,
+            user_profile_name=user_profile_name,
+            domain_id=domain_id,
+            **kwargs
+        )
+
+
 class AssociationSummary(_base_types.ApiObject):
     """Summary model of an association.
 
@@ -196,6 +226,9 @@ class AssociationSummary(_base_types.ApiObject):
         created_by (obj): Context on creator.
     """
 
+    _custom_boto_types = {
+        "created_by": (UserContext, False),
+    }
     source_arn = None
     source_name = None
     destination_arn = None
@@ -204,4 +237,3 @@ class AssociationSummary(_base_types.ApiObject):
     destination_type = None
     association_type = None
     creation_time = None
-    created_by = None

src/sagemaker/session.py

@@ -456,6 +456,7 @@ def train(  # noqa: C901
         enable_sagemaker_metrics=None,
         profiler_rule_configs=None,
         profiler_config=None,
+        environment=None,
     ):
         """Create an Amazon SageMaker training job.
 
@@ -522,9 +523,12 @@ def train(  # noqa: C901
                 Series. For more information see:
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries
                 (default: ``None``).
-            profiler_rule_configs (list[dict]): A list of profiler rule configurations.
+            profiler_rule_configs (list[dict]): A list of profiler rule
+                configurations.src/sagemaker/lineage/artifact.py:285
             profiler_config (dict): Configuration for how profiling information is emitted
                 with SageMaker Profiler. (default: ``None``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         Returns:
             str: ARN of the training job, if it is created.
@@ -556,6 +560,7 @@ def train(  # noqa: C901
             enable_sagemaker_metrics=enable_sagemaker_metrics,
             profiler_rule_configs=profiler_rule_configs,
             profiler_config=profiler_config,
+            environment=environment,
         )
         LOGGER.info("Creating training-job with name: %s", job_name)
         LOGGER.debug("train request: %s", json.dumps(train_request, indent=4))
@@ -588,6 +593,7 @@ def _get_train_request(  # noqa: C901
         enable_sagemaker_metrics=None,
         profiler_rule_configs=None,
         profiler_config=None,
+        environment=None,
     ):
         """Constructs a request compatible for creating an Amazon SageMaker training job.
 
@@ -657,6 +663,8 @@ def _get_train_request(  # noqa: C901
             profiler_rule_configs (list[dict]): A list of profiler rule configurations.
             profiler_config(dict): Configuration for how profiling information is emitted with
                 SageMaker Profiler. (default: ``None``).
+            environment (dict[str, str]) : Environment variables to be set for
+                use during training job (default: ``None``)
 
         Returns:
             Dict: a training request dict
@@ -699,6 +707,9 @@ def _get_train_request(  # noqa: C901
         if hyperparameters and len(hyperparameters) > 0:
             train_request["HyperParameters"] = hyperparameters
 
+        if environment is not None:
+            train_request["Environment"] = environment
+
         if tags is not None:
             train_request["Tags"] = tags
 

src/sagemaker/tensorflow/estimator.py

@@ -18,7 +18,6 @@
 from packaging import version
 
 from sagemaker import image_uris, s3, utils
-from sagemaker.debugger import DebuggerHookConfig
 from sagemaker.deprecations import renamed_kwargs
 from sagemaker.estimator import Framework
 import sagemaker.fw_utils as fw
@@ -347,6 +346,7 @@ def _validate_and_set_debugger_configs(self):
 
         Else, set default HookConfig
         """
+        super(TensorFlow, self)._validate_and_set_debugger_configs()
         ps_enabled = "parameter_server" in self.distribution and self.distribution[
             "parameter_server"
         ].get("enabled", False)
@@ -358,11 +358,6 @@ def _validate_and_set_debugger_configs(self):
                 )
             self.debugger_hook_config = None
             self.debugger_rule_configs = None
-        elif self.debugger_hook_config is None and fw._region_supports_debugger(
-            self.sagemaker_session.boto_session.region_name
-        ):
-            # Set defaults for debugging.
-            self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
 
     def transformer(
         self,

tests/conftest.py

@@ -190,12 +190,31 @@ def pytorch_inference_py_version(pytorch_inference_version, request):
         return "py3"
 
 
+def _huggingface_pytorch_version(huggingface_vesion):
+    config = image_uris.config_for_framework("huggingface")
+    training_config = config.get("training")
+    original_version = huggingface_vesion
+    if "version_aliases" in training_config:
+        huggingface_vesion = training_config.get("version_aliases").get(
+            huggingface_vesion, huggingface_vesion
+        )
+    version_config = training_config.get("versions").get(huggingface_vesion)
+    for key in list(version_config.keys()):
+        if key.startswith("pytorch"):
+            pt_version = key[7:]
+            if len(original_version.split(".")) == 2:
+                pt_version = ".".join(pt_version.split(".")[:-1])
+            return pt_version
+
+
 @pytest.fixture(scope="module")
 def huggingface_pytorch_version(huggingface_training_version):
-    if Version(huggingface_training_version) <= Version("4.4.2"):
-        return "1.6.0"
-    else:
-        pytest.skip("Skipping Huggingface version.")
+    return _huggingface_pytorch_version(huggingface_training_version)
+
+
+@pytest.fixture(scope="module")
+def huggingface_pytorch_latest_version(huggingface_training_latest_version):
+    return _huggingface_pytorch_version(huggingface_training_latest_version)
 
 
 @pytest.fixture(scope="module")

tests/integ/sagemaker/lineage/test_association.py

@@ -56,6 +56,9 @@ def test_list(association_objs, sagemaker_session):
     # sanity check
     assert association_keys_listed
 
+    for listed_asscn in listed:
+        assert listed_asscn.created_by is None
+
 
 @pytest.mark.timeout(30)
 def test_set_tag(association_obj, sagemaker_session):