Fix FrameworkProcessor for SageMaker Pipelines

README.rst

@@ -90,6 +90,7 @@ SageMaker Python SDK is tested on:
 - Python 3.6
 - Python 3.7
 - Python 3.8
+- Python 3.9
 
 AWS Permissions
 ~~~~~~~~~~~~~~~

doc/amazon_sagemaker_processing.rst

@@ -221,6 +221,14 @@ For an in-depth look, please see the `Feature Transformation with Spark`_ exampl
 .. _Feature Transformation with Spark: https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker_processing/feature_transformation_with_sagemaker_processing/feature_transformation_with_sagemaker_processing.ipynb
 
 
+Data Processing with Other Frameworks
+=====================================
+
+:class:`FrameworkProcessor`-based classes are also provided for a range of other ML frameworks: For example PyTorch, TensorFlow, and MXNet.
+
+You can use these to run data processing jobs in pre-built container environments, similarly to model training with :class:`Framework`-based Estimators.
+
+
 Learn More
 ==========
 
@@ -229,12 +237,18 @@ Processing class documentation
 
 - :class:`sagemaker.processing.Processor`
 - :class:`sagemaker.processing.ScriptProcessor`
+- :class:`sagemaker.processing.FrameworkProcessor`
 - :class:`sagemaker.sklearn.processing.SKLearnProcessor`
 - :class:`sagemaker.spark.processing.PySparkProcessor`
 - :class:`sagemaker.spark.processing.SparkJarProcessor`
 - :class:`sagemaker.processing.ProcessingInput`
 - :class:`sagemaker.processing.ProcessingOutput`
 - :class:`sagemaker.processing.ProcessingJob`
+- :class:`sagemaker.huggingface.processing.HuggingFaceProcessor`
+- :class:`sagemaker.mxnet.processing.MXNetProcessor`
+- :class:`sagemaker.pytorch.processing.PyTorchProcessor`
+- :class:`sagemaker.tensorflow.processing.TensorFlowProcessor`
+- :class:`sagemaker.xgboost.processing.XGBoostProcessor`
 
 
 Further documentation

doc/conf.py

@@ -15,6 +15,11 @@
 
 import pkg_resources
 from datetime import datetime
+import sys
+import os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "."))
+from doc_utils.jumpstart_doc_utils import create_jumpstart_model_table  # noqa: E402
 
 project = "sagemaker"
 version = pkg_resources.require(project)[0].version
@@ -71,6 +76,12 @@
 # For Adobe Analytics
 html_js_files = [
     "https://a0.awsstatic.com/s_code/js/3.0/awshome_s_code.js",
+    "https://cdn.datatables.net/1.10.23/js/jquery.dataTables.min.js",
+    "js/datatable.js",
+]
+
+html_css_files = [
+    "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css",
 ]
 
 html_context = {"css_files": ["_static/theme_overrides.css"]}
@@ -83,3 +94,7 @@
 
 # autosectionlabel
 autosectionlabel_prefix_document = True
+
+
+def setup(app):
+    create_jumpstart_model_table()

doc/doc_utils/jumpstart_doc_utils.py

@@ -0,0 +1,89 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+from urllib import request
+import json
+from packaging.version import Version
+
+JUMPSTART_REGION = "eu-west-2"
+SDK_MANIFEST_FILE = "models_manifest.json"
+JUMPSTART_BUCKET_BASE_URL = "https://jumpstart-cache-prod-{}.s3.{}.amazonaws.com".format(
+    JUMPSTART_REGION, JUMPSTART_REGION
+)
+
+
+def get_jumpstart_sdk_manifest():
+    url = "{}/{}".format(JUMPSTART_BUCKET_BASE_URL, SDK_MANIFEST_FILE)
+    with request.urlopen(url) as f:
+        models_manifest = f.read().decode("utf-8")
+    return json.loads(models_manifest)
+
+
+def get_jumpstart_sdk_spec(key):
+    url = "{}/{}".format(JUMPSTART_BUCKET_BASE_URL, key)
+    with request.urlopen(url) as f:
+        model_spec = f.read().decode("utf-8")
+    return json.loads(model_spec)
+
+
+def create_jumpstart_model_table():
+    sdk_manifest = get_jumpstart_sdk_manifest()
+    sdk_manifest_top_versions_for_models = {}
+
+    for model in sdk_manifest:
+        if model["model_id"] not in sdk_manifest_top_versions_for_models:
+            sdk_manifest_top_versions_for_models[model["model_id"]] = model
+        else:
+            if Version(
+                sdk_manifest_top_versions_for_models[model["model_id"]]["version"]
+            ) < Version(model["version"]):
+                sdk_manifest_top_versions_for_models[model["model_id"]] = model
+
+    file_content = []
+
+    file_content.append("==================================\n")
+    file_content.append("JumpStart Available Model Table\n")
+    file_content.append("==================================\n")
+    file_content.append(
+        """
+    JumpStart for the SageMaker Python SDK uses model ids and model versions to access the necessary
+    utilities. This table serves to provide the core material plus some extra information that can be useful
+    in selecting the correct model id and corresponding parameters.\n
+    """
+    )
+    file_content.append(
+        """
+    If you want to automatically use the latest version of the model, use "*" for the `model_version` attribute.
+    We highly suggest pinning an exact model version however.\n
+    """
+    )
+    file_content.append("\n")
+    file_content.append(".. list-table:: Available Models\n")
+    file_content.append("   :widths: 50 20 20 20\n")
+    file_content.append("   :header-rows: 1\n")
+    file_content.append("   :class: datatable\n")
+    file_content.append("\n")
+    file_content.append("   * - Model ID\n")
+    file_content.append("     - Fine Tunable?\n")
+    file_content.append("     - Latest Version\n")
+    file_content.append("     - Min SDK Version\n")
+
+    for model in sorted(sdk_manifest, key=lambda elt: elt["model_id"]):
+        model_spec = get_jumpstart_sdk_spec(model["spec_key"])
+        file_content.append("   * - {}\n".format(model["model_id"]))
+        file_content.append("     - {}\n".format(model_spec["training_supported"]))
+        file_content.append("     - {}\n".format(model["version"]))
+        file_content.append("     - {}\n".format(model["min_version"]))
+
+    f = open("doc_utils/jumpstart.rst", "w")
+    f.writelines(file_content)

doc/frameworks/huggingface/sagemaker.huggingface.rst

@@ -32,3 +32,11 @@ Hugging Face Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+Hugging Face Processor
+----------------------
+
+.. autoclass:: sagemaker.huggingface.processing.HuggingFaceProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/mxnet/sagemaker.mxnet.rst

@@ -25,3 +25,11 @@ MXNet Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+MXNet Processor
+---------------------------
+
+.. autoclass:: sagemaker.mxnet.processing.MXNetProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/pytorch/sagemaker.pytorch.rst

@@ -24,3 +24,11 @@ PyTorch Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+PyTorch Processor
+-----------------
+
+.. autoclass:: sagemaker.pytorch.processing.PyTorchProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/tensorflow/sagemaker.tensorflow.rst

@@ -25,3 +25,11 @@ TensorFlow Serving Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+TensorFlow Processor
+--------------------
+
+.. autoclass:: sagemaker.tensorflow.processing.TensorFlowProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/xgboost/xgboost.rst

@@ -16,3 +16,8 @@ The Amazon SageMaker XGBoost open source framework algorithm.
     :members:
     :undoc-members:
     :show-inheritance:
+
+.. autoclass:: sagemaker.xgboost.processing.XGBoostProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/overview.rst

@@ -577,6 +577,11 @@ Here is an example:
 Use Prebuilt Models with SageMaker JumpStart
 ********************************************
 
+.. toctree::
+    :maxdepth: 2
+
+    doc_utils/jumpstart
+
 `Amazon SageMaker JumpStart <https://aws.amazon.com/sagemaker/getting-started/>`__ is a
 SageMaker feature that helps users bring machine learning (ML)
 applications to market using prebuilt solutions for common use cases,
@@ -628,11 +633,11 @@ the ``model_id`` and ``model_version`` needed to retrieve the URI.
 
    -  ``model_id``: A unique identifier for the JumpStart model.
    -  ``model_version``: The version of the specifications for the
-      model. To use the latest version, enter ``*``. This is a
+      model. To use the latest version, enter ``"*"``. This is a
       required parameter.
 
 To retrieve a model, first select a ``model id`` and ``version`` from
-the Available Models.
+the :doc:`available models <./doc_utils/jumpstart>`.
 
 .. code:: python
 
@@ -652,7 +657,7 @@ Then use those values to retrieve the model as follows.
 JumpStart scripts
 -----------------
 
-To adapt JumpStart models for the SageMaker Python SDK, a custom
+To adapt JumpStart models for SageMaker, a custom
 script is needed to perform training or inference. JumpStart
 maintains a suite of scripts used for each of the models in the
 JumpStart S3 bucket, which can be accessed using the SageMaker Python
@@ -769,7 +774,7 @@ Deployment may take about 5 minutes.
        predictor_cls=Predictor
    )
 
-Because ``catboost`` relies on the PyTorch Deep Learning Containers
+Because ``catboost`` and ``lightgbm`` rely on the PyTorch Deep Learning Containers
 image, the corresponding Models and Endpoints display the “pytorch”
 prefix when viewed in the AWS console. To verify that these models
 were created successfully with your desired base model, refer to
@@ -780,7 +785,7 @@ Perform Inference
 
 Finally, use the ``predictor`` instance to query your endpoint. For
 ``catboost-classification-model``, for example, the predictor accepts
-a string. For more information about how to use the predictor, see
+a csv. For more information about how to use the predictor, see
 the
 `Appendix <https://sagemaker.readthedocs.io/en/stable/overview.html#appendix>`__.
 
@@ -807,9 +812,8 @@ using “training” as the model scope. Use the utility functions to
 retrieve the URI of each of the three components you need to
 continue. The HuggingFace model in this example requires a GPU
 instance, so use the ``ml.p3.2xlarge`` instance type. For a complete
-list of available SageMaker instance types , see `Available SageMaker
-Studio Instance
-Types <https://docs.aws.amazon.com/sagemaker/latest/dg/notebooks-available-instance-types.html>`__.
+list of available SageMaker instance types, see the `SageMaker On-Demand Pricing
+Table <https://aws.amazon.com/sagemaker/pricing/#On-Demand_Pricing>`__ and select 'Training'.
 
 .. code:: python
 
@@ -970,45 +974,45 @@ ContentType of ``application/list-text``.
 
 .. container::
 
-   +-----------------------+-----------------------+-----------------------+
-   | Task                  | Identifier            | ContentType           |
-   +-----------------------+-----------------------+-----------------------+
-   | Image Classification  | ic                    | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Object Detection      | od, od1               | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Semantic Segmentation | semseg                | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Instance Segmentation | is                    | "application/x-image" |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Classification   | tc                    | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Sentence Pair         | spc                   | "a                    |
-   | Classification        |                       | pplication/list-text" |
-   +-----------------------+-----------------------+-----------------------+
-   | Extractive Question   | eqa                   | "a                    |
-   | Answering             |                       | pplication/list-text" |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Generation       | textgeneration        | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Image Classification  | icembedding           | "application/x-image" |
-   | Embedding             |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Classification   | tcembedding           | "application/x-text"  |
-   | Embedding             |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Named-entity          | ner                   | "application/x-text"  |
-   | Recognition           |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Summarization    | summarization         | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Text Translation      | translation           | "application/x-text"  |
-   +-----------------------+-----------------------+-----------------------+
-   | Tabular Regression    | regression            | "text/csv"            |
-   +-----------------------+-----------------------+-----------------------+
-   | Tabular               | classification        | "text/csv"            |
-   | Classification        |                       |                       |
-   +-----------------------+-----------------------+-----------------------+
+   +-----------------------+-----------------------+-------------------------+
+   | Task                  | Identifier            | ContentType             |
+   +-----------------------+-----------------------+-------------------------+
+   | Image Classification  | ic                    | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Object Detection      | od, od1               | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Semantic Segmentation | semseg                | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Instance Segmentation | is                    | "application/x-image"   |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Classification   | tc                    | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Sentence Pair         | spc                   | "application/list-text" |
+   | Classification        |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Extractive Question   | eqa                   | "application/list-text" |
+   | Answering             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Generation       | textgeneration        | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Image Classification  | icembedding           | "application/x-image"   |
+   | Embedding             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Classification   | tcembedding           | "application/x-text"    |
+   | Embedding             |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Named-entity          | ner                   | "application/x-text"    |
+   | Recognition           |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Summarization    | summarization         | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Text Translation      | translation           | "application/x-text"    |
+   +-----------------------+-----------------------+-------------------------+
+   | Tabular Regression    | regression            | "text/csv"              |
+   +-----------------------+-----------------------+-------------------------+
+   | Tabular               | classification        | "text/csv"              |
+   | Classification        |                       |                         |
+   +-----------------------+-----------------------+-------------------------+
 
 ********************************
 SageMaker Automatic Model Tuning

doc/requirements.txt

@@ -1,3 +1,4 @@
 sphinx==3.1.1
 sphinx-rtd-theme==0.5.0
 docutils==0.15.2
+packaging==20.9

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -147,3 +147,5 @@ Steps
 .. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckConfig
 
 .. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckStep
+
+.. autoclass:: sagemaker.workflow.fail_step.FailStep