Merge branch 'master' of https://github.com/staubhp/sagemaker-python-sdk into fix-repack-model

Author: Payton Staub

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## v2.68.0 (2021-11-02)
+
+### Features
+
+ * CompilationStep support for Sagemaker Pipelines
+
+## v2.67.0 (2021-11-01)
+
+### Deprecations and Removals
+
+ * deprecate Serverless Lambda model-predictor
+
+### Features
+
+ * add joinsource to DataConfig
+ * Add support for Partial Dependence Plots(PDP) in SageMaker Clarify
+
+### Bug Fixes and Other Changes
+
+ * localmode subprocess parent process not sending SIGTERM to child
+ * remove buildspec from repo
+
+## v2.66.2.post0 (2021-10-28)
+
+### Documentation Changes
+
+ * Update estimator docstrings to add Fast File Mode
+
 ## v2.66.2 (2021-10-27)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.66.3.dev0
+2.68.1.dev0

buildspec-deploy.yml

@@ -16,7 +16,3 @@ Model
     :undoc-members:
     :show-inheritance:
 
-.. autoclass:: sagemaker.serverless.model.LambdaModel
-    :members:
-    :undoc-members:
-    :show-inheritance:

buildspec-localmodetests.yml

@@ -7,8 +7,3 @@ Make real-time predictions against SageMaker endpoints with Python objects
     :members:
     :undoc-members:
     :show-inheritance:
-
-.. autoclass:: sagemaker.serverless.predictor.LambdaPredictor
-    :members:
-    :undoc-members:
-    :show-inheritance:

buildspec-notebooktests.yml

@@ -1063,50 +1063,6 @@ You can also find these notebooks in the **Advanced Functionality** section of t
 For information about using sample notebooks in a SageMaker notebook instance, see `Use Example Notebooks <https://docs.aws.amazon.com/sagemaker/latest/dg/howitworks-nbexamples.html>`__
 in the AWS documentation.
 
-********************
-Serverless Inference
-********************
-
-You can use the SageMaker Python SDK to perform serverless inference on Lambda.
-
-To deploy models to Lambda, you must complete the following prerequisites:
-
-- `Package your model and inference code as a container image. <https://docs.aws.amazon.com/lambda/latest/dg/images-create.html>`_
-- `Create a role that lists Lambda as a trusted entity. <https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html#permissions-executionrole-console>`_
-
-After completing the prerequisites, you can deploy your model to Lambda using
-the `LambdaModel`_ class.
-
-.. code:: python
-
-   from sagemaker.serverless import LambdaModel
-
-   image_uri = "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-lambda-repository:latest"
-   role = "arn:aws:iam::123456789012:role/MyLambdaExecutionRole"
-
-   model = LambdaModel(image_uri=image_uri, role=role)
-   predictor = model.deploy("my-lambda-function", timeout=20, memory_size=4092)
-
-The ``deploy`` method returns a `LambdaPredictor`_  instance. Use the
-`LambdaPredictor`_ ``predict`` method to perform inference on Lambda.
-
-.. code:: python
-
-   url = "https://example.com/cat.jpeg"
-   predictor.predict({"url": url})  # {'class': 'tabby'}
-
-Once you are done performing inference on Lambda, free the `LambdaModel`_ and
-`LambdaPredictor`_ resources using the ``delete_model`` and ``delete_predictor``
-methods.
-
-.. code:: python
-
-   model.delete_model()
-   predictor.delete_predictor()
-
-.. _LambdaModel : https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.serverless.model.LambdaModel
-.. _LambdaPredictor : https://sagemaker.readthedocs.io/en/stable/api/inference/predictors.html#sagemaker.serverless.predictor.LambdaPredictor
-
 ******************
 SageMaker Workflow
 ******************

buildspec-release.yml

@@ -38,6 +38,7 @@ def __init__(
         dataset_type="text/csv",
         s3_data_distribution_type="FullyReplicated",
         s3_compression_type="None",
+        joinsource=None,
     ):
         """Initializes a configuration of both input and output datasets.
 
@@ -57,6 +58,11 @@ def __init__(
             s3_data_distribution_type (str): Valid options are "FullyReplicated" or
                 "ShardedByS3Key".
             s3_compression_type (str): Valid options are "None" or "Gzip".
+            joinsource (str): The name or index of the column in the dataset that acts an
+                identifier column (for instance, while performing a join). This column is only
+                used as an identifier, and not used for any other computations. This is an
+                optional field in all cases except when the dataset contains more than one file,
+                and `save_local_shap_values` is set to true in SHAPConfig.
         """
         if dataset_type not in ["text/csv", "application/jsonlines", "application/x-parquet"]:
             raise ValueError(
@@ -77,6 +83,7 @@ def __init__(
         _set(features, "features", self.analysis_config)
         _set(headers, "headers", self.analysis_config)
         _set(label, "label", self.analysis_config)
+        _set(joinsource, "joinsource_name_or_index", self.analysis_config)
 
     def get_config(self):
         """Returns part of an analysis config dictionary."""
@@ -300,6 +307,37 @@ def get_explainability_config(self):
         return None
 
 
+class PDPConfig(ExplainabilityConfig):
+    """Config class for Partial Dependence Plots (PDP).
+
+    If PDP is requested, the Partial Dependence Plots will be included in the report, and the
+    corresponding values will be included in the analysis output.
+    """
+
+    def __init__(self, features=None, grid_resolution=15, top_k_features=10):
+        """Initializes config for PDP.
+
+        Args:
+            features (None or list): List of features names or indices for which partial dependence
+                plots must be computed and plotted. When ShapConfig is provided, this parameter is
+                optional as Clarify will try to compute the partial dependence plots for top
+                feature based on SHAP attributions. When ShapConfig is not provided, 'features'
+                must be provided.
+            grid_resolution (int): In case of numerical features, this number represents that
+                number of buckets that range of values must be divided into. This decides the
+                granularity of the grid in which the PDP are plotted.
+            top_k_features (int): Set the number of top SHAP attributes to be selected to compute
+                partial dependence plots.
+        """
+        self.pdp_config = {"grid_resolution": grid_resolution, "top_k_features": top_k_features}
+        if features is not None:
+            self.pdp_config["features"] = features
+
+    def get_explainability_config(self):
+        """Returns config."""
+        return copy.deepcopy({"pdp": self.pdp_config})
+
+
 class SHAPConfig(ExplainabilityConfig):
     """Config class of SHAP."""
 
@@ -792,8 +830,9 @@ def run_explainability(
             data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
             model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
                 endpoint to be created.
-            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig`): Config of the
-                specific explainability method. Currently, only SHAP is supported.
+            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig` or list):
+                Config of the specific explainability method or a list of ExplainabilityConfig
+                objects. Currently, SHAP and PDP are the two methods supported.
             model_scores(str|int|ModelPredictedLabelConfig):  Index or JSONPath location in the
                 model output for the predicted scores to be explained. This is not required if the
                 model output is a single score. Alternatively, an instance of
@@ -827,7 +866,30 @@ def run_explainability(
             predictor_config.update(predicted_label_config)
         else:
             _set(model_scores, "label", predictor_config)
-        analysis_config["methods"] = explainability_config.get_explainability_config()
+
+        explainability_methods = {}
+        if isinstance(explainability_config, list):
+            if len(explainability_config) == 0:
+                raise ValueError("Please provide at least one explainability config.")
+            for config in explainability_config:
+                explain_config = config.get_explainability_config()
+                explainability_methods.update(explain_config)
+            if not len(explainability_methods.keys()) == len(explainability_config):
+                raise ValueError("Duplicate explainability configs are provided")
+            if (
+                "shap" not in explainability_methods
+                and explainability_methods["pdp"].get("features", None) is None
+            ):
+                raise ValueError("PDP features must be provided when ShapConfig is not provided")
+        else:
+            if (
+                isinstance(explainability_config, PDPConfig)
+                and explainability_config.get_explainability_config()["pdp"].get("features", None)
+                is None
+            ):
+                raise ValueError("PDP features must be provided when ShapConfig is not provided")
+            explainability_methods = explainability_config.get_explainability_config()
+        analysis_config["methods"] = explainability_methods
         analysis_config["predictor"] = predictor_config
         if job_name is None:
             if self.job_name_prefix: