Merge branch 'master' into master

Author: SifeiLi

doc/requirements.txt

@@ -2,5 +2,5 @@ sphinx==5.1.1
 sphinx-rtd-theme==0.5.0
 docutils==0.15.2
 packaging==20.9
-jinja2<3.1
+jinja2==3.1.3
 schema==0.7.5

requirements/extras/test_requirements.txt

@@ -12,13 +12,13 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.7.3
+apache-airflow==2.8.1
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0
 requests==2.31.0
 sagemaker-experiments==0.1.35
-Jinja2==3.0.3
+Jinja2==3.1.3
 pyvis==0.2.1
 pandas>=1.3.5,<1.5
 scikit-learn==1.3.0
@@ -32,7 +32,7 @@ sagemaker-feature-store-pyspark-3.3
 # TODO find workaround
 xgboost>=1.6.2,<=1.7.6
 pillow>=10.0.1,<=11
-transformers==4.32.0
+transformers>=4.36.0
 sentencepiece==0.1.99
 # https://github.com/triton-inference-server/server/issues/6246
 tritonclient[http]<2.37.0

src/sagemaker/image_uri_config/huggingface-llm.json

@@ -10,7 +10,8 @@
             "1.0": "1.0.3",
             "1.1": "1.1.0",
             "1.2": "1.2.0",
-            "1.3": "1.3.3"
+            "1.3": "1.3.3",
+            "1.4": "1.4.0"
         },
         "versions": {
             "0.6.0": {
@@ -388,6 +389,53 @@
                 "container_version": {
                     "gpu": "cu121-ubuntu20.04"
                 }
+            },
+            "1.4.0": {
+                "py_versions": [
+                    "py310"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "il-central-1": "780543022126",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",      
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "me-south-1": "217643126080",
+                    "me-central-1": "914824155844",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-isob-east-1": "094389454867",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884",
+                    "ca-west-1": "204538143572"
+                },
+                "tag_prefix": "2.1.1-tgi1.4.0",
+                "repository": "huggingface-pytorch-tgi-inference",
+                "container_version": {
+                    "gpu": "cu121-ubuntu20.04"
+                }
             }
         }
     }

src/sagemaker/workflow/clarify_check_step.py

@@ -132,8 +132,10 @@ class ModelExplainabilityCheckConfig(ClarifyCheckConfig):
 
     Attributes:
         model_config (ModelConfig): Config of the model and its endpoint to be created.
-        explainability_config (SHAPConfig): Config of the specific explainability method.
-            Currently, only SHAP is supported.
+        explainability_config (SHAPConfig or PDPConfig): Config of the explainability method.
+            Supports SHAP or PDP.
+            For `PDPConfig`, `features` must be specified.
+            `top_k_features` based on SHAP is currently not supported.
         model_scores (str or int or ModelPredictedLabelConfig): Index or JMESPath expression
             to locate the predicted scores in the model output (default: None).
             This is not required if the model output is a single score. Alternatively,
@@ -166,6 +168,10 @@ def __init__(
     ):
         """Constructs a ClarifyCheckStep.
 
+        To understand the `skip_check`, `fail_on_violation`, `register_new_baseline`
+        and `supplied_baseline_constraints` parameters, check the following documentation:
+        https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-quality-clarify-baseline-lifecycle.html
+
         Args:
             name (str): The name of the ClarifyCheckStep step.
             clarify_check_config (ClarifyCheckConfig): A ClarifyCheckConfig instance.

src/sagemaker/workflow/quality_check_step.py

@@ -135,6 +135,11 @@ def __init__(
     ):
         """Constructs a QualityCheckStep.
 
+        To understand the `skip_check`, `fail_on_violation`, `register_new_baseline`,
+        `supplied_baseline_constraints` and `supplied_baseline_constraints` parameters,
+        check the following documentation:
+        https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-quality-clarify-baseline-lifecycle.html
+
         Args:
             name (str): The name of the QualityCheckStep step.
             quality_check_config (QualityCheckConfig): A QualityCheckConfig instance.

tests/data/huggingface_byoc/requirements.txt

@@ -1,2 +1,3 @@
-transformers<=4.28.1
+transformers>=4.36.0
 datasets==2.16.1
+accelerate>=0.21.0

tests/unit/sagemaker/image_uris/test_huggingface_llm.py

@@ -28,6 +28,7 @@
         "1.2.0": "2.1.1-tgi1.2.0-gpu-py310-cu121-ubuntu20.04",
         "1.3.1": "2.1.1-tgi1.3.1-gpu-py310-cu121-ubuntu20.04",
         "1.3.3": "2.1.1-tgi1.3.3-gpu-py310-cu121-ubuntu20.04",
+        "1.4.0": "2.1.1-tgi1.4.0-gpu-py310-cu121-ubuntu20.04",
     },
     "inf2": {
         "0.0.16": "1.13.1-optimum0.0.16-neuronx-py310-ubuntu22.04",

tox.ini

@@ -81,7 +81,7 @@ passenv =
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
     python -c "import os; os.system('install-custom-pkgs --install-boto-wheels')"
-    pip install 'apache-airflow==2.7.3' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.8.txt"
+    pip install 'apache-airflow==2.8.1' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.8.txt"
     pip install 'torch==2.0.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'torchvision==0.15.2+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'