Merge branch 'update-hf-pt-train-dlc' of https://github.com/JingyaHuang/sagemaker-python-sdk into update-hf-pt-train-dlc

Author: JingyaHuang

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v2.134.0 (2023-02-22)
+
+### Features
+
+ * Add python 3.9 and spark 3.2 support for spark processor
+ * Adding support for Multi Worker Mirrored Strategy in TF estimator
+
+### Bug Fixes and Other Changes
+
+ * tag permission issue - remove describe before create
+
 ## v2.133.0 (2023-02-18)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.133.1.dev0
+2.134.1.dev0

doc/api/training/sdp_versions/latest.rst

@@ -26,8 +26,8 @@ depending on the version of the library you use.
    <https://docs.aws.amazon.com/sagemaker/latest/dg/data-parallel-use-api.html#data-parallel-use-python-skd-api>`_
    for more information.
 
-Version 1.4.0, 1.4.1, 1.5.0, 1.6.0 (Latest)
-===========================================
+For versions between 1.4.0 and 1.7.0 (Latest)
+=============================================
 
 .. toctree::
    :maxdepth: 1

doc/api/training/smd_data_parallel_release_notes/smd_data_parallel_change_log.rst

@@ -7,9 +7,40 @@ Release Notes
 New features, bug fixes, and improvements are regularly made to the SageMaker
 distributed data parallel library.
 
-SageMaker Distributed Data Parallel 1.6.0 Release Notes
+SageMaker Distributed Data Parallel 1.7.0 Release Notes
 =======================================================
 
+*Date: Feb. 10. 2023*
+
+**Currency Updates**
+
+* Added support for PyTorch 1.13.1.
+
+**Migration to AWS Deep Learning Containers**
+
+This version passed benchmark testing and is migrated to the following AWS Deep Learning Containers (DLC):
+
+- PyTorch 1.13.1 DLC
+
+  .. code::
+
+    763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.13.1-gpu-py39-cu117-ubuntu20.04-sagemaker
+
+Binary file of this version of the library for custom container users:
+
+  .. code::
+
+    https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.13.1/cu117/2023-01-09/smdistributed_dataparallel-1.7.0-cp39-cp39-linux_x86_64.whl
+
+
+----
+
+Release History
+===============
+
+SageMaker Distributed Data Parallel 1.6.0 Release Notes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 *Date: Dec. 15. 2022*
 
 **New Features**
@@ -44,11 +75,6 @@ Binary file of this version of the library for `custom container
     https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.12.1/cu113/2022-12-05/smdistributed_dataparallel-1.6.0-cp38-cp38-linux_x86_64.whl
 
 
-----
-
-Release History
-===============
-
 SageMaker Distributed Data Parallel 1.5.0 Release Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/sagemaker/experiments/experiment.py

@@ -15,6 +15,8 @@
 
 import time
 
+from botocore.exceptions import ClientError
+
 from sagemaker.apiutils import _base_types
 from sagemaker.experiments.trial import _Trial
 from sagemaker.experiments.trial_component import _TrialComponent
@@ -154,17 +156,21 @@ def _load_or_create(
         Returns:
             experiments.experiment._Experiment: A SageMaker `_Experiment` object
         """
-        sagemaker_client = sagemaker_session.sagemaker_client
         try:
-            experiment = _Experiment.load(experiment_name, sagemaker_session)
-        except sagemaker_client.exceptions.ResourceNotFound:
             experiment = _Experiment.create(
                 experiment_name=experiment_name,
                 display_name=display_name,
                 description=description,
                 tags=tags,
                 sagemaker_session=sagemaker_session,
             )
+        except ClientError as ce:
+            error_code = ce.response["Error"]["Code"]
+            error_message = ce.response["Error"]["Message"]
+            if not (error_code == "ValidationException" and "already exists" in error_message):
+                raise ce
+            # already exists
+            experiment = _Experiment.load(experiment_name, sagemaker_session)
         return experiment
 
     def list_trials(self, created_before=None, created_after=None, sort_by=None, sort_order=None):

src/sagemaker/experiments/trial.py

@@ -13,6 +13,8 @@
 """Contains the Trial class."""
 from __future__ import absolute_import
 
+from botocore.exceptions import ClientError
+
 from sagemaker.apiutils import _base_types
 from sagemaker.experiments import _api_types
 from sagemaker.experiments.trial_component import _TrialComponent
@@ -268,8 +270,20 @@ def _load_or_create(
         Returns:
             experiments.trial._Trial: A SageMaker `_Trial` object
         """
-        sagemaker_client = sagemaker_session.sagemaker_client
         try:
+            trial = _Trial.create(
+                experiment_name=experiment_name,
+                trial_name=trial_name,
+                display_name=display_name,
+                tags=tags,
+                sagemaker_session=sagemaker_session,
+            )
+        except ClientError as ce:
+            error_code = ce.response["Error"]["Code"]
+            error_message = ce.response["Error"]["Message"]
+            if not (error_code == "ValidationException" and "already exists" in error_message):
+                raise ce
+            # already exists
             trial = _Trial.load(trial_name, sagemaker_session)
             if trial.experiment_name != experiment_name:  # pylint: disable=no-member
                 raise ValueError(
@@ -278,12 +292,4 @@ def _load_or_create(
                         trial.experiment_name  # pylint: disable=no-member
                     )
                 )
-        except sagemaker_client.exceptions.ResourceNotFound:
-            trial = _Trial.create(
-                experiment_name=experiment_name,
-                trial_name=trial_name,
-                display_name=display_name,
-                tags=tags,
-                sagemaker_session=sagemaker_session,
-            )
         return trial

src/sagemaker/experiments/trial_component.py

@@ -15,6 +15,8 @@
 
 import time
 
+from botocore.exceptions import ClientError
+
 from sagemaker.apiutils import _base_types
 from sagemaker.experiments import _api_types
 from sagemaker.experiments._api_types import TrialComponentSearchResult
@@ -326,16 +328,20 @@ def _load_or_create(
             experiments.trial_component._TrialComponent: A SageMaker `_TrialComponent` object.
             bool: A boolean variable indicating whether the trail component already exists
         """
-        sagemaker_client = sagemaker_session.sagemaker_client
         is_existed = False
         try:
-            run_tc = _TrialComponent.load(trial_component_name, sagemaker_session)
-            is_existed = True
-        except sagemaker_client.exceptions.ResourceNotFound:
             run_tc = _TrialComponent.create(
                 trial_component_name=trial_component_name,
                 display_name=display_name,
                 tags=tags,
                 sagemaker_session=sagemaker_session,
             )
+        except ClientError as ce:
+            error_code = ce.response["Error"]["Code"]
+            error_message = ce.response["Error"]["Message"]
+            if not (error_code == "ValidationException" and "already exists" in error_message):
+                raise ce
+            # already exists
+            run_tc = _TrialComponent.load(trial_component_name, sagemaker_session)
+            is_existed = True
         return run_tc, is_existed

src/sagemaker/image_uri_config/spark.json

@@ -91,6 +91,36 @@
                     "ap-southeast-3": "732049463269"
                 },
                 "repository": "sagemaker-spark-processing"
+            },
+            "3.2": {
+                "py_versions": ["py39"],
+                "registries": {
+                    "me-south-1": "750251592176",
+                    "ap-south-1": "105495057255",
+                    "eu-north-1": "330188676905",
+                    "eu-west-3": "136845547031",
+                    "us-east-2": "314815235551",
+                    "eu-west-1": "571004829621",
+                    "eu-central-1": "906073651304",
+                    "sa-east-1": "737130764395",
+                    "ap-east-1": "732049463269",
+                    "us-east-1": "173754725891",
+                    "ap-northeast-2": "860869212795",
+                    "eu-west-2": "836651553127",
+                    "ap-northeast-1": "411782140378",
+                    "us-west-2": "153931337802",
+                    "us-west-1": "667973535471",
+                    "ap-southeast-1": "759080221371",
+                    "ap-southeast-2": "440695851116",
+                    "ca-central-1": "446299261295",
+                    "cn-north-1": "671472414489",
+                    "cn-northwest-1": "844356804704",
+                    "eu-south-1": "753923664805",
+                    "af-south-1": "309385258863",
+                    "us-gov-west-1": "271483468897",
+                    "ap-southeast-3": "732049463269"
+                },
+                "repository": "sagemaker-spark-processing"
             }
         }
     }

src/sagemaker/local/utils.py

@@ -14,16 +14,20 @@
 from __future__ import absolute_import
 
 import os
+import logging
 import shutil
 import subprocess
 import json
 import re
+import errno
 
 from distutils.dir_util import copy_tree
 from six.moves.urllib.parse import urlparse
 
 from sagemaker import s3
 
+logger = logging.getLogger(__name__)
+
 
 def copy_directory_structure(destination_directory, relative_path):
     """Creates intermediate directory structure for relative_path.
@@ -77,7 +81,19 @@ def move_to_destination(source, destination, job_name, sagemaker_session):
     else:
         raise ValueError("Invalid destination URI, must be s3:// or file://, got: %s" % destination)
 
-    shutil.rmtree(source)
+    try:
+        shutil.rmtree(source)
+    except OSError as exc:
+        # on Linux, when docker writes to any mounted volume, it uses the container's user. In most
+        # cases this is root. When the container exits and we try to delete them we can't because
+        # root owns those files. We expect this to happen, so we handle EACCESS. Any other error
+        # we will raise the exception up.
+        if exc.errno == errno.EACCES:
+            logger.warning("Failed to delete: %s Please remove it manually.", source)
+        else:
+            logger.error("Failed to delete: %s", source)
+            raise
+
     return final_uri
 
 

src/sagemaker/model_monitor/dataset_format.py

@@ -71,8 +71,6 @@ def csv(header=True):
         Args:
             header (bool): Whether the csv dataset to baseline and monitor has a header.
                 Default: True.
-            output_columns_position (str): The position of the output columns.
-                Must be one of ("START", "END"). Default: "START".
 
         Returns:
             dict: JSON string containing DatasetFormat to be used by DefaultModelMonitor.

src/sagemaker/model_monitor/model_monitoring.py

@@ -233,6 +233,7 @@ def create_monitoring_schedule(
         monitor_schedule_name=None,
         schedule_cron_expression=None,
         batch_transform_input=None,
+        arguments=None,
     ):
         """Creates a monitoring schedule to monitor an Amazon SageMaker Endpoint.
 
@@ -262,6 +263,7 @@ def create_monitoring_schedule(
             batch_transform_input (sagemaker.model_monitor.BatchTransformInput): Inputs to
                 run the monitoring schedule on the batch transform
                 (default: None)
+            arguments ([str]): A list of string arguments to be passed to a processing job.
 
         """
         if self.monitoring_schedule_name is not None:
@@ -326,6 +328,9 @@ def create_monitoring_schedule(
         if self.network_config is not None:
             network_config_dict = self.network_config._to_request_dict()
 
+        if arguments is not None:
+            self.arguments = arguments
+
         self.sagemaker_session.create_monitoring_schedule(
             monitoring_schedule_name=self.monitoring_schedule_name,
             schedule_expression=schedule_cron_expression,
@@ -2054,6 +2059,21 @@ def _update_data_quality_monitoring_schedule(
             self._update_monitoring_schedule(self.job_definition_name, schedule_cron_expression)
             return
 
+        existing_desc = self.sagemaker_session.describe_monitoring_schedule(
+            monitoring_schedule_name=self.monitoring_schedule_name
+        )
+
+        if (
+            existing_desc.get("MonitoringScheduleConfig") is not None
+            and existing_desc["MonitoringScheduleConfig"].get("ScheduleConfig") is not None
+            and existing_desc["MonitoringScheduleConfig"]["ScheduleConfig"]["ScheduleExpression"]
+            is not None
+            and schedule_cron_expression is None
+        ):
+            schedule_cron_expression = existing_desc["MonitoringScheduleConfig"]["ScheduleConfig"][
+                "ScheduleExpression"
+            ]
+
         # Need to update schedule with a new job definition
         job_desc = self.sagemaker_session.sagemaker_client.describe_data_quality_job_definition(
             JobDefinitionName=self.job_definition_name