Merge branch 'master' into fix-issue-2426

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## v2.49.1 (2021-07-19)
+
+### Bug Fixes and Other Changes
+
+ * Set flag when debugger is disabled
+ * KMS Key fix for kwargs
+ * Update BiasConfig to accept multiple facet params
+
+### Documentation Changes
+
+ * Update huggingface estimator documentation
+
+## v2.49.0 (2021-07-15)
+
+### Features
+
+ * Adding serial inference pipeline support to RegisterModel Step
+
+### Documentation Changes
+
+ * add tuning step get_top_model_s3_uri and callback step to doc
+ * links for HF in sdk
+ * Add Clarify module to Model Monitoring API docs
+
 ## v2.48.2 (2021-07-12)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.48.3.dev0
+2.49.2.dev0

src/sagemaker/__init__.py

@@ -55,6 +55,7 @@
 from sagemaker.processing import Processor, ScriptProcessor  # noqa: F401
 from sagemaker.session import Session  # noqa: F401
 from sagemaker.session import container_def, pipeline_container_def  # noqa: F401
+from sagemaker.session import get_model_package_args  # noqa: F401
 from sagemaker.session import production_variant  # noqa: F401
 from sagemaker.session import get_execution_role  # noqa: F401
 

src/sagemaker/clarify.py

@@ -88,21 +88,34 @@ def __init__(
         Args:
             label_values_or_threshold (Any): List of label values or threshold to indicate positive
                 outcome used for bias metrics.
-            facet_name (str): Sensitive attribute in the input data for which we like to compare
-                metrics.
+            facet_name (str or [str]): String or List of strings of sensitive attribute(s) in the
+            input data for which we like to compare metrics.
             facet_values_or_threshold (list): Optional list of values to form a sensitive group or
                 threshold for a numeric facet column that defines the lower bound of a sensitive
                 group. Defaults to considering each possible value as sensitive group and
                 computing metrics vs all the other examples.
+                If facet_name is a list, this needs to be None or a List consisting of lists or None
+                with the same length as facet_name list.
             group_name (str): Optional column name or index to indicate a group column to be used
                 for the bias metric 'Conditional Demographic Disparity in Labels - CDDL' or
                 'Conditional Demographic Disparity in Predicted Labels - CDDPL'.
         """
-        facet = {"name_or_index": facet_name}
-        _set(facet_values_or_threshold, "value_or_threshold", facet)
+        if isinstance(facet_name, str):
+            facet = {"name_or_index": facet_name}
+            _set(facet_values_or_threshold, "value_or_threshold", facet)
+            facet_list = [facet]
+        elif facet_values_or_threshold is None or len(facet_name) == len(facet_values_or_threshold):
+            facet_list = []
+            for i, single_facet_name in enumerate(facet_name):
+                facet = {"name_or_index": single_facet_name}
+                if facet_values_or_threshold is not None:
+                    _set(facet_values_or_threshold[i], "value_or_threshold", facet)
+                facet_list.append(facet)
+        else:
+            raise ValueError("Wrong combination of argument values passed")
         self.analysis_config = {
             "label_values_or_threshold": label_values_or_threshold,
-            "facet": [facet],
+            "facet": facet_list,
         }
         _set(group_name, "group_variable", self.analysis_config)
 

src/sagemaker/debugger/__init__.py

@@ -15,6 +15,7 @@
 
 from sagemaker.debugger.debugger import (  # noqa: F401
     CollectionConfig,
+    DEBUGGER_FLAG,
     DebuggerHookConfig,
     framework_name,
     get_default_profiler_rule,

src/sagemaker/debugger/debugger.py

@@ -32,6 +32,7 @@
 from sagemaker.utils import build_dict
 
 framework_name = "debugger"
+DEBUGGER_FLAG = "USE_SMDEBUG"
 
 
 def get_rule_container_image_uri(region):

src/sagemaker/estimator.py

@@ -29,6 +29,7 @@
 from sagemaker.analytics import TrainingJobAnalytics
 from sagemaker.debugger import TensorBoardOutputConfig  # noqa: F401 # pylint: disable=unused-import
 from sagemaker.debugger import (
+    DEBUGGER_FLAG,
     DebuggerHookConfig,
     FrameworkProfile,
     get_default_profiler_rule,
@@ -2269,6 +2270,11 @@ def _validate_and_set_debugger_configs(self):
                     )
                     self.debugger_hook_config = False
 
+        if self.debugger_hook_config is False:
+            if self.environment is None:
+                self.environment = {}
+            self.environment[DEBUGGER_FLAG] = "0"
+
     def _stage_user_code_in_s3(self):
         """Upload the user training script to s3 and return the location.
 

src/sagemaker/huggingface/estimator.py

@@ -70,10 +70,10 @@ def __init__(
                 ``image_uri`` is provided. The current supported version is ``4.6.1``.
             tensorflow_version (str): TensorFlow version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
-                ``pytorch_version`` is provided. The current supported version is ``1.6.0``.
+                ``pytorch_version`` is provided. The current supported version is ``2.4.1``.
             pytorch_version (str): PyTorch version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
-                ``tensorflow_version`` is provided. The current supported version is ``2.4.1``.
+                ``tensorflow_version`` is provided. The current supported versions are ``1.7.1`` and ``1.6.0``.
             source_dir (str): Path (absolute, relative or an S3 URI) to a directory
                 with any other training source code dependencies aside from the entry
                 point file (default: None). If ``source_dir`` is an S3 URI, it must

src/sagemaker/model.py

@@ -158,14 +158,15 @@ def register(
         if self.model_data is None:
             raise ValueError("SageMaker Model Package cannot be created without model data.")
 
-        model_pkg_args = self._get_model_package_args(
+        model_pkg_args = sagemaker.get_model_package_args(
             content_types,
             response_types,
             inference_instances,
             transform_instances,
             model_package_name,
             model_package_group_name,
-            image_uri,
+            self.model_data,
+            image_uri or self.image_uri,
             model_metrics,
             metadata_properties,
             marketplace_cert,
@@ -181,80 +182,6 @@ def register(
             model_package_arn=model_package.get("ModelPackageArn"),
         )
 
-    def _get_model_package_args(
-        self,
-        content_types,
-        response_types,
-        inference_instances,
-        transform_instances,
-        model_package_name=None,
-        model_package_group_name=None,
-        image_uri=None,
-        model_metrics=None,
-        metadata_properties=None,
-        marketplace_cert=False,
-        approval_status=None,
-        description=None,
-        tags=None,
-    ):
-        """Get arguments for session.create_model_package method.
-
-        Args:
-            content_types (list): The supported MIME types for the input data.
-            response_types (list): The supported MIME types for the output data.
-            inference_instances (list): A list of the instance types that are used to
-                generate inferences in real-time.
-            transform_instances (list): A list of the instance types on which a transformation
-                job can be run or on which an endpoint can be deployed.
-            model_package_name (str): Model Package name, exclusive to `model_package_group_name`,
-                using `model_package_name` makes the Model Package un-versioned (default: None).
-            model_package_group_name (str): Model Package Group name, exclusive to
-                `model_package_name`, using `model_package_group_name` makes the Model Package
-                versioned (default: None).
-            image_uri (str): Inference image uri for the container. Model class' self.image will
-                be used if it is None (default: None).
-            model_metrics (ModelMetrics): ModelMetrics object (default: None).
-            metadata_properties (MetadataProperties): MetadataProperties object (default: None).
-            marketplace_cert (bool): A boolean value indicating if the Model Package is certified
-                for AWS Marketplace (default: False).
-            approval_status (str): Model Approval Status, values can be "Approved", "Rejected",
-                or "PendingManualApproval" (default: "PendingManualApproval").
-            description (str): Model Package description (default: None).
-        Returns:
-            dict: A dictionary of method argument names and values.
-        """
-        if image_uri:
-            self.image_uri = image_uri
-        container = {
-            "Image": self.image_uri,
-            "ModelDataUrl": self.model_data,
-        }
-
-        model_package_args = {
-            "containers": [container],
-            "content_types": content_types,
-            "response_types": response_types,
-            "inference_instances": inference_instances,
-            "transform_instances": transform_instances,
-            "marketplace_cert": marketplace_cert,
-        }
-
-        if model_package_name is not None:
-            model_package_args["model_package_name"] = model_package_name
-        if model_package_group_name is not None:
-            model_package_args["model_package_group_name"] = model_package_group_name
-        if model_metrics is not None:
-            model_package_args["model_metrics"] = model_metrics._to_request_dict()
-        if metadata_properties is not None:
-            model_package_args["metadata_properties"] = metadata_properties._to_request_dict()
-        if approval_status is not None:
-            model_package_args["approval_status"] = approval_status
-        if description is not None:
-            model_package_args["description"] = description
-        if tags is not None:
-            model_package_args["tags"] = tags
-        return model_package_args
-
     def _init_sagemaker_session_if_does_not_exist(self, instance_type):
         """Set ``self.sagemaker_session`` to ``LocalSession`` or ``Session`` if it's not already.
 
@@ -1128,6 +1055,10 @@ def _upload_code(self, key_prefix, repack=False):
             )
 
         if repack and self.model_data is not None and self.entry_point is not None:
+            if isinstance(self.model_data, sagemaker.workflow.properties.Properties):
+                # model is not yet there, defer repacking to later during pipeline execution
+                return
+
             bucket = self.bucket or self.sagemaker_session.default_bucket()
             repacked_model_data = "s3://" + "/".join([bucket, key_prefix, "model.tar.gz"])