Merge branch 'master' into algo_kix_region

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## v2.57.0 (2021-08-30)
+
+### Deprecations and Removals
+
+ * Remove stale S3DownloadMode from test_session.py
+
+### Features
+
+ * update clarify imageURI for KIX
+
+### Bug Fixes and Other Changes
+
+ * propagate KMS key to model.deploy
+ * Propagate tags and VPC configs to repack model steps
+
 ## v2.56.0 (2021-08-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.56.1.dev0
+2.57.1.dev0

setup.py

@@ -52,6 +52,7 @@ def read_version():
         "urllib3>=1.21.1,!=1.25,!=1.25.1",
         "docker-compose>=1.25.2",
         "PyYAML>=5.3, <6",  # PyYAML version has to match docker-compose requirements
+        "psutil",
     ],
     "scipy": ["scipy>=0.19.0"],
 }

src/sagemaker/image_uri_config/debugger.json

@@ -7,6 +7,7 @@
                 "ap-east-1": "199566480951",
                 "ap-northeast-1": "430734990657",
                 "ap-northeast-2": "578805364391",
+                "ap-northeast-3": "479947661362",
                 "ap-south-1": "904829902805",
                 "ap-southeast-1": "972752614525",
                 "ap-southeast-2": "184798709955",

src/sagemaker/local/image.py

@@ -31,7 +31,10 @@
 import tempfile
 
 from distutils.spawn import find_executable
+from signal import SIGTERM
 from threading import Thread
+
+import psutil
 from six.moves.urllib.parse import urlparse
 
 import sagemaker
@@ -840,6 +843,8 @@ def run(self):
 
     def down(self):
         """Placeholder docstring"""
+        for process in psutil.Process(self.process.pid).children():
+            process.send_signal(SIGTERM)
         self.process.terminate()
 
 

src/sagemaker/workflow/_utils.py

@@ -55,6 +55,8 @@ def __init__(
         role,
         model_data: str,
         entry_point: str,
+        display_name: str = None,
+        description: str = None,
         source_dir: str = None,
         dependencies: List = None,
         depends_on: Union[List[str], List[Step]] = None,
@@ -165,7 +167,12 @@ def __init__(
 
         # super!
         super(_RepackModelStep, self).__init__(
-            name=name, depends_on=depends_on, estimator=repacker, inputs=inputs
+            name=name,
+            display_name=display_name,
+            description=description,
+            depends_on=depends_on,
+            estimator=repacker,
+            inputs=inputs,
         )
 
     def _prepare_for_repacking(self):
@@ -285,6 +292,7 @@ def __init__(
         approval_status="PendingManualApproval",
         image_uri=None,
         compile_model_family=None,
+        display_name: str = None,
         description=None,
         depends_on: Union[List[str], List[Step]] = None,
         tags=None,
@@ -326,7 +334,9 @@ def __init__(
                 this step depends on
             **kwargs: additional arguments to `create_model`.
         """
-        super(_RegisterModelStep, self).__init__(name, StepTypeEnum.REGISTER_MODEL, depends_on)
+        super(_RegisterModelStep, self).__init__(
+            name, display_name, description, StepTypeEnum.REGISTER_MODEL, depends_on
+        )
         self.estimator = estimator
         self.model_data = model_data
         self.content_types = content_types

src/sagemaker/workflow/callback_step.py

@@ -83,6 +83,8 @@ def __init__(
         sqs_queue_url: str,
         inputs: dict,
         outputs: List[CallbackOutput],
+        display_name: str = None,
+        description: str = None,
         cache_config: CacheConfig = None,
         depends_on: Union[List[str], List[Step]] = None,
     ):
@@ -94,11 +96,15 @@ def __init__(
             inputs (dict): Input arguments that will be provided
                 in the SQS message body of callback messages.
             outputs (List[CallbackOutput]): Outputs that can be provided when completing a callback.
+            display_name (str): The display name of the callback step.
+            description (str): The description of the callback step.
             cache_config (CacheConfig):  A `sagemaker.workflow.steps.CacheConfig` instance.
             depends_on (List[str] or List[Step]): A list of step names or step instances
                 this `sagemaker.workflow.steps.CallbackStep` depends on
         """
-        super(CallbackStep, self).__init__(name, StepTypeEnum.CALLBACK, depends_on)
+        super(CallbackStep, self).__init__(
+            name, display_name, description, StepTypeEnum.CALLBACK, depends_on
+        )
         self.sqs_queue_url = sqs_queue_url
         self.outputs = outputs
         self.cache_config = cache_config

src/sagemaker/workflow/condition_step.py

@@ -42,6 +42,8 @@ def __init__(
         self,
         name: str,
         depends_on: Union[List[str], List[Step]] = None,
+        display_name: str = None,
+        description: str = None,
         conditions: List[Condition] = None,
         if_steps: List[Union[Step, StepCollection]] = None,
         else_steps: List[Union[Step, StepCollection]] = None,
@@ -53,6 +55,9 @@ def __init__(
         execution.
 
         Args:
+            name (str): The name of the condition step.
+            display_name (str): The display name of the condition step.
+            description (str): The description of the condition step.
             conditions (List[Condition]): A list of `sagemaker.workflow.conditions.Condition`
                 instances.
             if_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
@@ -62,7 +67,9 @@ def __init__(
                 or `sagemaker.workflow.step_collections.StepCollection` instances that are
                 marked as ready for execution if the list of conditions evaluates to False.
         """
-        super(ConditionStep, self).__init__(name, StepTypeEnum.CONDITION, depends_on)
+        super(ConditionStep, self).__init__(
+            name, display_name, description, StepTypeEnum.CONDITION, depends_on
+        )
         self.conditions = conditions or []
         self.if_steps = if_steps or []
         self.else_steps = else_steps or []

src/sagemaker/workflow/lambda_step.py

@@ -82,6 +82,8 @@ def __init__(
         self,
         name: str,
         lambda_func: Lambda,
+        display_name: str = None,
+        description: str = None,
         inputs: dict = None,
         outputs: List[LambdaOutput] = None,
         cache_config: CacheConfig = None,
@@ -91,6 +93,8 @@ def __init__(
 
         Args:
             name (str): The name of the lambda step.
+            display_name (str): The display name of the Lambda step.
+            description (str): The description of the Lambda step.
             lambda_func (str): An instance of sagemaker.lambda_helper.Lambda.
                 If lambda arn is specified in the instance, LambdaStep just invokes the function,
                 else lambda function will be created while creating the pipeline.
@@ -101,7 +105,9 @@ def __init__(
             depends_on (List[str]): A list of step names this `sagemaker.workflow.steps.LambdaStep`
                 depends on
         """
-        super(LambdaStep, self).__init__(name, StepTypeEnum.LAMBDA, depends_on)
+        super(LambdaStep, self).__init__(
+            name, display_name, description, StepTypeEnum.LAMBDA, depends_on
+        )
         self.lambda_func = lambda_func
         self.outputs = outputs if outputs is not None else []
         self.cache_config = cache_config

src/sagemaker/workflow/step_collections.py

@@ -67,6 +67,7 @@ def __init__(
         approval_status=None,
         image_uri=None,
         compile_model_family=None,
+        display_name=None,
         description=None,
         tags=None,
         model: Union[Model, PipelineModel] = None,
@@ -138,6 +139,8 @@ def __init__(
                 tags=tags,
                 subnets=subnets,
                 security_group_ids=security_group_ids,
+                description=description,
+                display_name=display_name,
                 **kwargs,
             )
             steps.append(repack_model_step)
@@ -179,6 +182,8 @@ def __init__(
                         tags=tags,
                         subnets=subnets,
                         security_group_ids=security_group_ids,
+                        description=description,
+                        display_name=display_name,
                         **kwargs,
                     )
                     steps.append(repack_model_step)
@@ -208,6 +213,7 @@ def __init__(
             image_uri=image_uri,
             compile_model_family=compile_model_family,
             description=description,
+            display_name=display_name,
             tags=tags,
             container_def_list=self.container_def_list,
             **kwargs,
@@ -231,6 +237,8 @@ def __init__(
         instance_count,
         instance_type,
         transform_inputs,
+        description: str = None,
+        display_name: str = None,
         # model arguments
         image_uri=None,
         predictor_cls=None,
@@ -302,6 +310,8 @@ def __init__(
                 tags=tags,
                 subnets=estimator.subnets,
                 security_group_ids=estimator.security_group_ids,
+                description=description,
+                display_name=display_name,
             )
             steps.append(repack_model_step)
             model_data = repack_model_step.properties.ModelArtifacts.S3ModelArtifacts
@@ -324,6 +334,8 @@ def predict_wrapper(endpoint, session):
             name=f"{name}CreateModelStep",
             model=model,
             inputs=model_inputs,
+            description=description,
+            display_name=display_name,
         )
         if "entry_point" not in kwargs and depends_on:
             # if the CreateModelStep is the first step in the collection
@@ -351,6 +363,8 @@ def predict_wrapper(endpoint, session):
             name=f"{name}TransformStep",
             transformer=transformer,
             inputs=transform_inputs,
+            description=description,
+            display_name=display_name,
         )
         steps.append(transform_step)