Merge branch 'dev' of https://github.com/aws/sagemaker-python-sdk into dev

Author: mchoi8739

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## v2.77.1 (2022-02-25)
+
+### Bug Fixes and Other Changes
+
+ * jumpstart model table
+
+## v2.77.0 (2022-02-22)
+
+### Features
+
+ * override jumpstart content bucket
+ * jumpstart model id suggestions
+ * adding customer metadata support to registermodel step
+
+### Bug Fixes and Other Changes
+
+ * Improve Pipeline workflow unit test branch coverage
+ * update lineage_trial_compoment get pipeline execution arn
+ * Add lineage doc
+ * Support primitive types for left value of ConditionSteps
+
+## v2.76.0 (2022-02-17)
+
+### Features
+
+ * Add FailStep Support for Sagemaker Pipeline
+
+### Bug Fixes and Other Changes
+
+ * use recommended inference image uri from Neo API
+ * pin test dependencies
+ * Add exception in test_action
+ * Update Static Endpoint
+ * Add CMH to the non-P3 list
+
+### Documentation Changes
+
+ * Support for generation of Jumpstart model table on build
+
+## v2.75.1 (2022-02-08)
+
+### Bug Fixes and Other Changes
+
+ * Add CMH to the non-P3 list
+
 ## v2.75.0 (2022-02-05)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.75.1.dev0
+2.77.2.dev0

doc/conf.py

@@ -15,6 +15,11 @@
 
 import pkg_resources
 from datetime import datetime
+import sys
+import os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "."))
+from doc_utils.jumpstart_doc_utils import create_jumpstart_model_table  # noqa: E402
 
 project = "sagemaker"
 version = pkg_resources.require(project)[0].version
@@ -71,6 +76,12 @@
 # For Adobe Analytics
 html_js_files = [
     "https://a0.awsstatic.com/s_code/js/3.0/awshome_s_code.js",
+    "https://cdn.datatables.net/1.10.23/js/jquery.dataTables.min.js",
+    "js/datatable.js",
+]
+
+html_css_files = [
+    "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css",
 ]
 
 html_context = {"css_files": ["_static/theme_overrides.css"]}
@@ -83,3 +94,7 @@
 
 # autosectionlabel
 autosectionlabel_prefix_document = True
+
+
+def setup(app):
+    create_jumpstart_model_table()

doc/doc_utils/__init__.py

@@ -0,0 +1,89 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+from urllib import request
+import json
+from packaging.version import Version
+
+JUMPSTART_REGION = "eu-west-2"
+SDK_MANIFEST_FILE = "models_manifest.json"
+JUMPSTART_BUCKET_BASE_URL = "https://jumpstart-cache-prod-{}.s3.{}.amazonaws.com".format(
+    JUMPSTART_REGION, JUMPSTART_REGION
+)
+
+
+def get_jumpstart_sdk_manifest():
+    url = "{}/{}".format(JUMPSTART_BUCKET_BASE_URL, SDK_MANIFEST_FILE)
+    with request.urlopen(url) as f:
+        models_manifest = f.read().decode("utf-8")
+    return json.loads(models_manifest)
+
+
+def get_jumpstart_sdk_spec(key):
+    url = "{}/{}".format(JUMPSTART_BUCKET_BASE_URL, key)
+    with request.urlopen(url) as f:
+        model_spec = f.read().decode("utf-8")
+    return json.loads(model_spec)
+
+
+def create_jumpstart_model_table():
+    sdk_manifest = get_jumpstart_sdk_manifest()
+    sdk_manifest_top_versions_for_models = {}
+
+    for model in sdk_manifest:
+        if model["model_id"] not in sdk_manifest_top_versions_for_models:
+            sdk_manifest_top_versions_for_models[model["model_id"]] = model
+        else:
+            if Version(
+                sdk_manifest_top_versions_for_models[model["model_id"]]["version"]
+            ) < Version(model["version"]):
+                sdk_manifest_top_versions_for_models[model["model_id"]] = model
+
+    file_content = []
+
+    file_content.append("==================================\n")
+    file_content.append("JumpStart Available Model Table\n")
+    file_content.append("==================================\n")
+    file_content.append(
+        """
+    JumpStart for the SageMaker Python SDK uses model ids and model versions to access the necessary
+    utilities. This table serves to provide the core material plus some extra information that can be useful
+    in selecting the correct model id and corresponding parameters.\n
+    """
+    )
+    file_content.append(
+        """
+    If you want to automatically use the latest version of the model, use "*" for the `model_version` attribute.
+    We highly suggest pinning an exact model version however.\n
+    """
+    )
+    file_content.append("\n")
+    file_content.append(".. list-table:: Available Models\n")
+    file_content.append("   :widths: 50 20 20 20\n")
+    file_content.append("   :header-rows: 1\n")
+    file_content.append("   :class: datatable\n")
+    file_content.append("\n")
+    file_content.append("   * - Model ID\n")
+    file_content.append("     - Fine Tunable?\n")
+    file_content.append("     - Latest Version\n")
+    file_content.append("     - Min SDK Version\n")
+
+    for model in sdk_manifest_top_versions_for_models.values():
+        model_spec = get_jumpstart_sdk_spec(model["spec_key"])
+        file_content.append("   * - {}\n".format(model["model_id"]))
+        file_content.append("     - {}\n".format(model_spec["training_supported"]))
+        file_content.append("     - {}\n".format(model["version"]))
+        file_content.append("     - {}\n".format(model["min_version"]))
+
+    f = open("doc_utils/jumpstart.rst", "w")
+    f.writelines(file_content)

doc/doc_utils/jumpstart.rst

@@ -577,6 +577,11 @@ Here is an example:
 Use Prebuilt Models with SageMaker JumpStart
 ********************************************
 
+.. toctree::
+    :maxdepth: 2
+
+    doc_utils/jumpstart
+
 `Amazon SageMaker JumpStart <https://aws.amazon.com/sagemaker/getting-started/>`__ is a
 SageMaker feature that helps users bring machine learning (ML)
 applications to market using prebuilt solutions for common use cases,
@@ -628,11 +633,11 @@ the ``model_id`` and ``model_version`` needed to retrieve the URI.
 
    -  ``model_id``: A unique identifier for the JumpStart model.
    -  ``model_version``: The version of the specifications for the
-      model. To use the latest version, enter ``*``. This is a
+      model. To use the latest version, enter ``"*"``. This is a
       required parameter.
 
 To retrieve a model, first select a ``model id`` and ``version`` from
-the Available Models.
+the :doc:`available models <./doc_utils/jumpstart>`.
 
 .. code:: python
 
@@ -652,7 +657,7 @@ Then use those values to retrieve the model as follows.
 JumpStart scripts
 -----------------
 
-To adapt JumpStart models for the SageMaker Python SDK, a custom
+To adapt JumpStart models for SageMaker, a custom
 script is needed to perform training or inference. JumpStart
 maintains a suite of scripts used for each of the models in the
 JumpStart S3 bucket, which can be accessed using the SageMaker Python
@@ -741,6 +746,7 @@ see `Model <https://sagemaker.readthedocs.io/en/stable/api/inference/model.html
 .. code:: python
 
    from sagemaker.model import Model
+   from sagemaker.predictor import Predictor
    from sagemaker.session import Session
 
    # Create the SageMaker model instance
@@ -750,6 +756,7 @@ see `Model <https://sagemaker.readthedocs.io/en/stable/api/inference/model.html
        source_dir=script_uri,
        entry_point="inference.py",
        role=Session().get_caller_identity_arn(),
+       predictor_cls=Predictor,
    )
 
 Save the output from deploying the model to a variable named
@@ -761,15 +768,12 @@ Deployment may take about 5 minutes.
 
 .. code:: python
 
-   from sagemaker.predictor import Predictor
-
    predictor = model.deploy(
        initial_instance_count=instance_count,
        instance_type=instance_type,
-       predictor_cls=Predictor
    )
 
-Because ``catboost`` relies on the PyTorch Deep Learning Containers
+Because ``catboost`` and ``lightgbm`` rely on the PyTorch Deep Learning Containers
 image, the corresponding Models and Endpoints display the “pytorch”
 prefix when viewed in the AWS console. To verify that these models
 were created successfully with your desired base model, refer to
@@ -780,7 +784,7 @@ Perform Inference
 
 Finally, use the ``predictor`` instance to query your endpoint. For
 ``catboost-classification-model``, for example, the predictor accepts
-a string. For more information about how to use the predictor, see
+a csv. For more information about how to use the predictor, see
 the
 `Appendix <https://sagemaker.readthedocs.io/en/stable/overview.html#appendix>`__.
 
@@ -807,9 +811,8 @@ using “training” as the model scope. Use the utility functions to
 retrieve the URI of each of the three components you need to
 continue. The HuggingFace model in this example requires a GPU
 instance, so use the ``ml.p3.2xlarge`` instance type. For a complete
-list of available SageMaker instance types , see `Available SageMaker
-Studio Instance
-Types <https://docs.aws.amazon.com/sagemaker/latest/dg/notebooks-available-instance-types.html>`__.
+list of available SageMaker instance types, see the `SageMaker On-Demand Pricing
+Table <https://aws.amazon.com/sagemaker/pricing/#On-Demand_Pricing>`__ and select 'Training'.
 
 .. code:: python
 
@@ -970,45 +973,45 @@ ContentType of ``application/list-text``.
 
 .. container::
 
-   +-----------------------+-----------------------+-----------------------+
-   | Task                  | Identifier            | ContentType           |
-   +-----------------------+-----------------------+-----------------------+
-   | Image Classification  | ic                    | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Object Detection      | od, od1               | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Semantic Segmentation | semseg                | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Instance Segmentation | is                    | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Classification   | tc                    | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Sentence Pair         | spc                   | "a                    |
-   | Classification        |                       | pplication/list-text" |
-   +-----------------------+-----------------------+-----------------------+
-   | Extractive Question   | eqa                   | "a                    |
-   | Answering             |                       | pplication/list-text" |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Generation       | textgeneration        | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Image Classification  | icembedding           | "application/x-image" |
-   | Embedding             |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Classification   | tcembedding           | "application/x-text"  |
-   | Embedding             |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Named-entity          | ner                   | "application/x-text"  |
-   | Recognition           |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Summarization    | summarization         | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Translation      | translation           | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Tabular Regression    | regression            | "text/csv"            |
-   +-----------------------+-----------------------+-----------------------+
-   | Tabular               | classification        | "text/csv"            |
-   | Classification        |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
+   +-----------------------+-----------------------+-------------------------+
+   | Task                  | Identifier            | ContentType             |
+   +-----------------------+-----------------------+-------------------------+
+   | Image Classification  | ic                    | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Object Detection      | od, od1               | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Semantic Segmentation | semseg                | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Instance Segmentation | is                    | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Classification   | tc                    | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Sentence Pair         | spc                   | "application/list-text" |
+   | Classification        |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Extractive Question   | eqa                   | "application/list-text" |
+   | Answering             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Generation       | textgeneration        | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Image Classification  | icembedding           | "application/x-image"   |
+   | Embedding             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Classification   | tcembedding           | "application/x-text"    |
+   | Embedding             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Named-entity          | ner                   | "application/x-text"    |
+   | Recognition           |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Summarization    | summarization         | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Translation      | translation           | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Tabular Regression    | regression            | "text/csv"              |
+   +-----------------------+-----------------------+-------------------------+
+   | Tabular               | classification        | "text/csv"              |
+   | Classification        |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
 
 ********************************
 SageMaker Automatic Model Tuning

doc/doc_utils/jumpstart_doc_utils.py

@@ -1,3 +1,4 @@
 sphinx==3.1.1
 sphinx-rtd-theme==0.5.0
 docutils==0.15.2
+packaging==20.9

doc/overview.rst

@@ -10,3 +10,4 @@ The SageMaker Python SDK supports managed training and inference for a variety o
     airflow/index
     step_functions/index
     pipelines/index
+    lineage/index

doc/requirements.txt

@@ -0,0 +1,11 @@
+###################
+SageMaker Lineage
+###################
+Amazon SageMaker ML Lineage Tracking creates and stores information about the steps of a machine learning (ML) workflow from data preparation to model deployment. With the tracking information, you can reproduce the workflow steps, track model and dataset lineage, and establish model governance and audit standards.
+
+SageMaker APIs for creating and managing SageMaker Lineage.
+
+.. toctree::
+    :maxdepth: 2
+
+    sagemaker.lineage