Merge branch 'master' into fix-localmode-subprocess-termination

Author: ranzvi

buildspec-deploy.yml

@@ -16,7 +16,3 @@ Model
     :undoc-members:
     :show-inheritance:
 
-.. autoclass:: sagemaker.serverless.model.LambdaModel
-    :members:
-    :undoc-members:
-    :show-inheritance:

buildspec-localmodetests.yml

@@ -7,8 +7,3 @@ Make real-time predictions against SageMaker endpoints with Python objects
     :members:
     :undoc-members:
     :show-inheritance:
-
-.. autoclass:: sagemaker.serverless.predictor.LambdaPredictor
-    :members:
-    :undoc-members:
-    :show-inheritance:

buildspec-notebooktests.yml

@@ -1063,50 +1063,6 @@ You can also find these notebooks in the **Advanced Functionality** section of t
 For information about using sample notebooks in a SageMaker notebook instance, see `Use Example Notebooks <https://docs.aws.amazon.com/sagemaker/latest/dg/howitworks-nbexamples.html>`__
 in the AWS documentation.
 
-********************
-Serverless Inference
-********************
-
-You can use the SageMaker Python SDK to perform serverless inference on Lambda.
-
-To deploy models to Lambda, you must complete the following prerequisites:
-
-- `Package your model and inference code as a container image. <https://docs.aws.amazon.com/lambda/latest/dg/images-create.html>`_
-- `Create a role that lists Lambda as a trusted entity. <https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html#permissions-executionrole-console>`_
-
-After completing the prerequisites, you can deploy your model to Lambda using
-the `LambdaModel`_ class.
-
-.. code:: python
-
-   from sagemaker.serverless import LambdaModel
-
-   image_uri = "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-lambda-repository:latest"
-   role = "arn:aws:iam::123456789012:role/MyLambdaExecutionRole"
-
-   model = LambdaModel(image_uri=image_uri, role=role)
-   predictor = model.deploy("my-lambda-function", timeout=20, memory_size=4092)
-
-The ``deploy`` method returns a `LambdaPredictor`_  instance. Use the
-`LambdaPredictor`_ ``predict`` method to perform inference on Lambda.
-
-.. code:: python
-
-   url = "https://example.com/cat.jpeg"
-   predictor.predict({"url": url})  # {'class': 'tabby'}
-
-Once you are done performing inference on Lambda, free the `LambdaModel`_ and
-`LambdaPredictor`_ resources using the ``delete_model`` and ``delete_predictor``
-methods.
-
-.. code:: python
-
-   model.delete_model()
-   predictor.delete_predictor()
-
-.. _LambdaModel : https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.serverless.model.LambdaModel
-.. _LambdaPredictor : https://sagemaker.readthedocs.io/en/stable/api/inference/predictors.html#sagemaker.serverless.predictor.LambdaPredictor
-
 ******************
 SageMaker Workflow
 ******************

buildspec-release.yml

@@ -38,6 +38,7 @@ def __init__(
         dataset_type="text/csv",
         s3_data_distribution_type="FullyReplicated",
         s3_compression_type="None",
+        joinsource=None,
     ):
         """Initializes a configuration of both input and output datasets.
 
@@ -57,6 +58,11 @@ def __init__(
             s3_data_distribution_type (str): Valid options are "FullyReplicated" or
                 "ShardedByS3Key".
             s3_compression_type (str): Valid options are "None" or "Gzip".
+            joinsource (str): The name or index of the column in the dataset that acts an
+                identifier column (for instance, while performing a join). This column is only
+                used as an identifier, and not used for any other computations. This is an
+                optional field in all cases except when the dataset contains more than one file,
+                and `save_local_shap_values` is set to true in SHAPConfig.
         """
         if dataset_type not in ["text/csv", "application/jsonlines", "application/x-parquet"]:
             raise ValueError(
@@ -77,6 +83,7 @@ def __init__(
         _set(features, "features", self.analysis_config)
         _set(headers, "headers", self.analysis_config)
         _set(label, "label", self.analysis_config)
+        _set(joinsource, "joinsource_name_or_index", self.analysis_config)
 
     def get_config(self):
         """Returns part of an analysis config dictionary."""
@@ -300,6 +307,37 @@ def get_explainability_config(self):
         return None
 
 
+class PDPConfig(ExplainabilityConfig):
+    """Config class for Partial Dependence Plots (PDP).
+
+    If PDP is requested, the Partial Dependence Plots will be included in the report, and the
+    corresponding values will be included in the analysis output.
+    """
+
+    def __init__(self, features=None, grid_resolution=15, top_k_features=10):
+        """Initializes config for PDP.
+
+        Args:
+            features (None or list): List of features names or indices for which partial dependence
+                plots must be computed and plotted. When ShapConfig is provided, this parameter is
+                optional as Clarify will try to compute the partial dependence plots for top
+                feature based on SHAP attributions. When ShapConfig is not provided, 'features'
+                must be provided.
+            grid_resolution (int): In case of numerical features, this number represents that
+                number of buckets that range of values must be divided into. This decides the
+                granularity of the grid in which the PDP are plotted.
+            top_k_features (int): Set the number of top SHAP attributes to be selected to compute
+                partial dependence plots.
+        """
+        self.pdp_config = {"grid_resolution": grid_resolution, "top_k_features": top_k_features}
+        if features is not None:
+            self.pdp_config["features"] = features
+
+    def get_explainability_config(self):
+        """Returns config."""
+        return copy.deepcopy({"pdp": self.pdp_config})
+
+
 class SHAPConfig(ExplainabilityConfig):
     """Config class of SHAP."""
 
@@ -792,8 +830,9 @@ def run_explainability(
             data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
             model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
                 endpoint to be created.
-            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig`): Config of the
-                specific explainability method. Currently, only SHAP is supported.
+            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig` or list):
+                Config of the specific explainability method or a list of ExplainabilityConfig
+                objects. Currently, SHAP and PDP are the two methods supported.
             model_scores(str|int|ModelPredictedLabelConfig):  Index or JSONPath location in the
                 model output for the predicted scores to be explained. This is not required if the
                 model output is a single score. Alternatively, an instance of
@@ -827,7 +866,30 @@ def run_explainability(
             predictor_config.update(predicted_label_config)
         else:
             _set(model_scores, "label", predictor_config)
-        analysis_config["methods"] = explainability_config.get_explainability_config()
+
+        explainability_methods = {}
+        if isinstance(explainability_config, list):
+            if len(explainability_config) == 0:
+                raise ValueError("Please provide at least one explainability config.")
+            for config in explainability_config:
+                explain_config = config.get_explainability_config()
+                explainability_methods.update(explain_config)
+            if not len(explainability_methods.keys()) == len(explainability_config):
+                raise ValueError("Duplicate explainability configs are provided")
+            if (
+                "shap" not in explainability_methods
+                and explainability_methods["pdp"].get("features", None) is None
+            ):
+                raise ValueError("PDP features must be provided when ShapConfig is not provided")
+        else:
+            if (
+                isinstance(explainability_config, PDPConfig)
+                and explainability_config.get_explainability_config()["pdp"].get("features", None)
+                is None
+            ):
+                raise ValueError("PDP features must be provided when ShapConfig is not provided")
+            explainability_methods = explainability_config.get_explainability_config()
+        analysis_config["methods"] = explainability_methods
         analysis_config["predictor"] = predictor_config
         if job_name is None:
             if self.job_name_prefix:

buildspec-slowtests.yml

@@ -21,24 +21,27 @@
 V2_URL = "https://sagemaker.readthedocs.io/en/stable/v2.html"
 
 
-def _warn(msg):
+def _warn(msg, sdk_version=None):
     """Generic warning raiser referencing V2
 
     Args:
         phrase: The phrase to include in the warning.
+        sdk_version: the sdk version of removal of support.
     """
-    full_msg = f"{msg} in sagemaker>=2.\nSee: {V2_URL} for details."
+    _sdk_version = sdk_version if sdk_version is not None else "2"
+    full_msg = f"{msg} in sagemaker>={_sdk_version}.\nSee: {V2_URL} for details."
     warnings.warn(full_msg, DeprecationWarning, stacklevel=2)
     logger.warning(full_msg)
 
 
-def removed_warning(phrase):
+def removed_warning(phrase, sdk_version=None):
     """Raise a warning for a no-op in sagemaker>=2
 
     Args:
         phrase: the prefix phrase of the warning message.
+        sdk_version: the sdk version of removal of support.
     """
-    _warn(f"{phrase} is a no-op")
+    _warn(f"{phrase} is a no-op", sdk_version)
 
 
 def renamed_warning(phrase):
@@ -146,26 +149,32 @@ def func(*args, **kwargs):  # pylint: disable=W0613
     return func
 
 
-def deprecated(obj):
+def deprecated(sdk_version=None):
     """Decorator for raising deprecated warning for a feature in sagemaker>=2
 
+    Args:
+        sdk_version (str): the sdk version of removal of support.
+
     Usage:
-        @deprecated
+        @deprecated()
         def sample_function():
             print("xxxx....")
 
-        @deprecated
+        @deprecated(sdk_version="2.66")
         class SampleClass():
             def __init__(self):
                 print("xxxx....")
 
     """
 
-    def wrapper(*args, **kwargs):
-        removed_warning(obj.__name__)
-        return obj(*args, **kwargs)
+    def deprecate(obj):
+        def wrapper(*args, **kwargs):
+            removed_warning(obj.__name__, sdk_version)
+            return obj(*args, **kwargs)
+
+        return wrapper
 
-    return wrapper
+    return deprecate
 
 
 def deprecated_function(func, name):