Merge branch 'master' into support_tf2.9_inference

Author: tejaschumbalkar

CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## v2.121.1 (2022-12-09)
+
+### Bug Fixes and Other Changes
+
+ * Pop out ModelPackageName from pipeline definition
+ * Fix failing jumpstart cache unit tests
+
+## v2.121.0 (2022-12-08)
+
+### Features
+
+ * Algorithms Region Expansion OSU/DXB
+
+### Bug Fixes and Other Changes
+
+ * FrameworkProcessor S3 uploads
+ * Add constraints file for apache-airflow
+
+## v2.120.0 (2022-12-07)
+
+### Features
+
+ * Add Neo image uri config for Pytorch 1.12
+ * Adding support for SageMaker Training Compiler in PyTorch estimator starting 1.12
+ * Update registries with new region account number mappings.
+ * Add DXB region to frameworks by DLC
+
+### Bug Fixes and Other Changes
+
+ * support idempotency for framework and spark processors
+
 ## v2.119.0 (2022-12-03)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.119.1.dev0
+2.121.2.dev0

requirements/extras/test_requirements.txt

@@ -11,6 +11,7 @@ contextlib2==21.6.0
 awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
+# Update tox.ini to have correct version of airflow constraints file
 apache-airflow==2.4.1
 apache-airflow-providers-amazon==4.0.0
 attrs==22.1.0

setup.py

@@ -55,7 +55,7 @@ def read_requirements(filename):
     "protobuf3-to-dict>=0.1.5,<1.0",
     "smdebug_rulesconfig==1.0.1",
     "importlib-metadata>=1.4.0,<5.0",
-    "packaging>=20.0",
+    "packaging==20.9",
     "pandas",
     "pathos",
     "schema",

src/sagemaker/fw_utils.py

@@ -494,7 +494,7 @@ def framework_name_from_image(image_uri):
     # We must support both the legacy and current image name format.
     name_pattern = re.compile(
         r"""^(?:sagemaker(?:-rl)?-)?
-        (tensorflow|mxnet|chainer|pytorch|scikit-learn|xgboost
+        (tensorflow|mxnet|chainer|pytorch|pytorch-trcomp|scikit-learn|xgboost
         |huggingface-tensorflow|huggingface-pytorch
         |huggingface-tensorflow-trcomp|huggingface-pytorch-trcomp)(?:-)?
         (scriptmode|training)?

src/sagemaker/image_uri_config/blazingtext.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "811284229777",
                 "us-east-2": "825641698319",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "433757028032"

src/sagemaker/image_uri_config/factorization-machines.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/forecasting-deepar.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "522234722520",
                 "us-east-2": "566113047672",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "156387875391"

src/sagemaker/image_uri_config/image-classification.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "811284229777",
                 "us-east-2": "825641698319",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "433757028032"

src/sagemaker/image_uri_config/ipinsights.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/kmeans.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/knn.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/linear-learner.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/neo-pytorch.json

@@ -11,7 +11,9 @@
         "1.7.0": "1.7",
         "1.7.1": "1.7",
         "1.8.0": "1.8",
-        "1.8.1": "1.8"
+        "1.8.1": "1.8",
+        "1.12.0": "1.12",
+        "1.12.1": "1.12"
     },
     "versions": {
         "1.4": {
@@ -173,6 +175,38 @@
                 "us-west-2": "301217895009"
             },
             "repository": "sagemaker-inference-pytorch"
+        },
+        "1.12": {
+            "py_versions": ["py3"],
+            "registries": {
+                "af-south-1": "774647643957",
+                "ap-east-1": "110948597952",
+                "ap-northeast-1": "941853720454",
+                "ap-northeast-2": "151534178276",
+                "ap-northeast-3": "925152966179",
+                "ap-south-1": "763008648453",
+                "ap-southeast-1": "324986816169",
+                "ap-southeast-2": "355873309152",
+                "ca-central-1": "464438896020",
+                "cn-north-1": "472730292857",
+                "cn-northwest-1": "474822919863",
+                "eu-central-1": "746233611703",
+                "eu-north-1": "601324751636",
+                "eu-south-1": "966458181534",
+                "eu-west-1": "802834080501",
+                "eu-west-2": "205493899709",
+                "eu-west-3": "254080097072",
+                "me-south-1": "836785723513",
+                "sa-east-1": "756306329178",
+                "us-east-1": "785573368785",
+                "us-east-2": "007439368137",
+                "us-gov-west-1": "263933020539",
+                "us-iso-east-1": "167761179201",
+                "us-isob-east-1": "406031935815",
+                "us-west-1": "710691900526",
+                "us-west-2": "301217895009"
+            },
+            "repository": "sagemaker-inference-pytorch"
         }
     }
 }

src/sagemaker/image_uri_config/ntm.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/object-detection.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "811284229777",
                 "us-east-2": "825641698319",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "433757028032"

src/sagemaker/image_uri_config/object2vec.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/pca.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/pytorch-training-compiler.json

@@ -0,0 +1,41 @@
+{
+    "training": {
+        "processors": [
+            "gpu"
+        ],
+        "version_aliases": {
+            "1.12": "1.12.0"
+        },
+        "versions": {
+            "1.12.0": {
+                "py_versions": [
+                    "py38"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "pytorch-trcomp-training"
+            }
+        }
+    }
+}

src/sagemaker/image_uri_config/randomcutforest.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "382416733822",
                 "us-east-2": "404615174143",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "174872318107"

src/sagemaker/image_uri_config/semantic-segmentation.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "811284229777",
                 "us-east-2": "825641698319",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "433757028032"

src/sagemaker/image_uri_config/seq2seq.json

@@ -22,10 +22,12 @@
                 "eu-west-3": "749696950732",
                 "eu-south-1": "257386234256",
                 "me-south-1": "249704162688",
+                "me-central-1": "272398656194",
                 "sa-east-1": "855470959533",
                 "us-east-1": "811284229777",
                 "us-east-2": "825641698319",
                 "us-gov-west-1": "226302683700",
+                "us-gov-east-1": "237065988967",
                 "us-iso-east-1": "490574956308",
                 "us-west-1": "632365934929",
                 "us-west-2": "433757028032"