Merge branch 'master' into master

Author: chuyang-deng

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## v1.51.2 (2020-03-11)
+
+### Bug Fixes and Other Changes
+
+ * handle empty inputs/outputs in ProcessingJob.from_processing_name()
+ * use DLC images for GovCloud
+
+### Testing and Release Infrastructure
+
+ * generate test job name at test start instead of module start
+
+## v1.51.1 (2020-03-10)
+
+### Bug Fixes and Other Changes
+
+ * skip pytorch ei test in unsupported regions
+
+### Documentation Changes
+
+ * correct MultiString/MULTI_STRING docstring
+
+## v1.51.0 (2020-03-09)
+
+### Features
+
+ * pytorch 1.3.1 eia support
+
+### Documentation Changes
+
+ * Update Kubernetes Operator default tag
+ * improve docstring for tuner.best_estimator()
+
 ## v1.50.18.post0 (2020-03-05)
 
 ### Documentation Changes

VERSION

@@ -1 +1 @@
-1.50.19.dev0
+1.51.3.dev0

src/sagemaker/algorithm.py

@@ -328,8 +328,8 @@ def transformer(
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in
-                a single request (default: None). Valid values: 'MULTI_RECORD'
-                and 'SINGLE_RECORD'.
+                a single request (default: None). Valid values: 'MultiRecord'
+                and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None).
                 Valid values: 'Line' or 'None'.
             output_path (str): S3 location for saving the transform result. If

src/sagemaker/estimator.py

@@ -823,8 +823,8 @@ def transformer(
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in
-                a single request (default: None). Valid values: 'MULTI_RECORD'
-                and 'SINGLE_RECORD'.
+                a single request (default: None). Valid values: 'MultiRecord'
+                and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None).
                 Valid values: 'Line' or 'None'.
             output_path (str): S3 location for saving the transform result. If
@@ -1895,8 +1895,8 @@ def transformer(
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in
-                a single request (default: None). Valid values: 'MULTI_RECORD'
-                and 'SINGLE_RECORD'.
+                a single request (default: None). Valid values: 'MultiRecord'
+                and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None).
                 Valid values: 'Line' or 'None'.
             output_path (str): S3 location for saving the transform result. If

src/sagemaker/fw_utils.py

@@ -62,7 +62,7 @@
 ]
 PY2_RESTRICTED_EIA_FRAMEWORKS = ["pytorch-serving"]
 VALID_ACCOUNTS_BY_REGION = {"us-gov-west-1": "246785580436", "us-iso-east-1": "744548109606"}
-ASIMOV_VALID_ACCOUNTS_BY_REGION = {"us-iso-east-1": "886529160074"}
+ASIMOV_VALID_ACCOUNTS_BY_REGION = {"us-gov-west-1": "442386744353", "us-iso-east-1": "886529160074"}
 OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "057415533634", "me-south-1": "724002660598"}
 ASIMOV_OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "871362719292", "me-south-1": "217643126080"}
 DEFAULT_ACCOUNT = "520713654638"
@@ -133,25 +133,6 @@ def _is_dlc_version(framework, framework_version, py_version):
     return False
 
 
-def _use_dlc_image(region, framework, py_version, framework_version):
-    """Return if the DLC image should be used for the given framework,
-    framework version, Python version, and region.
-
-    Args:
-        region (str): The AWS region.
-        framework (str): The framework name, e.g. "tensorflow-scriptmode".
-        py_version (str): The Python version, e.g. "py3".
-        framework_version (str): The framework version.
-
-    Returns:
-        bool: Whether or not to use the corresponding DLC image.
-    """
-    is_gov_region = region in VALID_ACCOUNTS_BY_REGION
-    is_dlc_version = _is_dlc_version(framework, framework_version, py_version)
-
-    return ((not is_gov_region) or region in ASIMOV_VALID_ACCOUNTS_BY_REGION) and is_dlc_version
-
-
 def _registry_id(region, framework, py_version, account, framework_version):
     """Return the Amazon ECR registry number (or AWS account ID) for
     the given framework, framework version, Python version, and region.
@@ -168,7 +149,7 @@ def _registry_id(region, framework, py_version, account, framework_version):
             specific one for the framework, framework version, Python version,
             and region, then ``account`` is returned.
     """
-    if _use_dlc_image(region, framework, py_version, framework_version):
+    if _is_dlc_version(framework, framework_version, py_version):
         if region in ASIMOV_OPT_IN_ACCOUNTS_BY_REGION:
             return ASIMOV_OPT_IN_ACCOUNTS_BY_REGION.get(region)
         if region in ASIMOV_VALID_ACCOUNTS_BY_REGION:
@@ -253,7 +234,7 @@ def create_image_uri(
         else:
             device_type = "cpu"
 
-    use_dlc_image = _use_dlc_image(region, framework, py_version, framework_version)
+    use_dlc_image = _is_dlc_version(framework, framework_version, py_version)
 
     if not py_version or (use_dlc_image and framework == "tensorflow-serving-eia"):
         tag = "{}-{}".format(framework_version, device_type)

src/sagemaker/model.py

@@ -504,8 +504,8 @@ def transformer(
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in
-                a single request (default: None). Valid values: 'MULTI_RECORD'
-                and 'SINGLE_RECORD'.
+                a single request (default: None). Valid values: 'MultiRecord'
+                and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None).
                 Valid values: 'Line' or 'None'.
             output_path (str): S3 location for saving the transform result. If

src/sagemaker/pipeline.py

@@ -215,8 +215,8 @@ def transformer(
             instance_type (str): Type of EC2 instance to use, for example,
                 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in
-                a single request (default: None). Valid values: 'MULTI_RECORD'
-                and 'SINGLE_RECORD'.
+                a single request (default: None). Valid values: 'MultiRecord'
+                and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None).
                 Valid values: 'Line' or 'None'.
             output_path (str): S3 location for saving the transform result. If

src/sagemaker/processing.py

@@ -648,10 +648,9 @@ def from_processing_name(cls, sagemaker_session, processing_job_name):
         """
         job_desc = sagemaker_session.describe_processing_job(job_name=processing_job_name)
 
-        return cls(
-            sagemaker_session=sagemaker_session,
-            job_name=processing_job_name,
-            inputs=[
+        inputs = None
+        if job_desc.get("ProcessingInputs"):
+            inputs = [
                 ProcessingInput(
                     source=processing_input["S3Input"]["S3Uri"],
                     destination=processing_input["S3Input"]["LocalPath"],
@@ -664,19 +663,31 @@ def from_processing_name(cls, sagemaker_session, processing_job_name):
                     s3_compression_type=processing_input["S3Input"].get("S3CompressionType"),
                 )
                 for processing_input in job_desc["ProcessingInputs"]
-            ],
-            outputs=[
+            ]
+
+        outputs = None
+        if job_desc.get("ProcessingOutputConfig") and job_desc["ProcessingOutputConfig"].get(
+            "Outputs"
+        ):
+            outputs = [
                 ProcessingOutput(
-                    source=job_desc["ProcessingOutputConfig"]["Outputs"][0]["S3Output"][
-                        "LocalPath"
-                    ],
-                    destination=job_desc["ProcessingOutputConfig"]["Outputs"][0]["S3Output"][
-                        "S3Uri"
-                    ],
-                    output_name=job_desc["ProcessingOutputConfig"]["Outputs"][0]["OutputName"],
+                    source=processing_output["S3Output"]["LocalPath"],
+                    destination=processing_output["S3Output"]["S3Uri"],
+                    output_name=processing_output["OutputName"],
                 )
-            ],
-            output_kms_key=job_desc["ProcessingOutputConfig"].get("KmsKeyId"),
+                for processing_output in job_desc["ProcessingOutputConfig"]["Outputs"]
+            ]
+
+        output_kms_key = None
+        if job_desc.get("ProcessingOutputConfig"):
+            output_kms_key = job_desc["ProcessingOutputConfig"].get("KmsKeyId")
+
+        return cls(
+            sagemaker_session=sagemaker_session,
+            job_name=processing_job_name,
+            inputs=inputs,
+            outputs=outputs,
+            output_kms_key=output_kms_key,
         )
 
     @classmethod

src/sagemaker/session.py

@@ -1968,7 +1968,7 @@ def transform(
             job_name (str): Name of the transform job being created.
             model_name (str): Name of the SageMaker model being used for the transform job.
             strategy (str): The strategy used to decide how to batch records in a single request.
-                Possible values are 'MULTI_RECORD' and 'SINGLE_RECORD'.
+                Possible values are 'MultiRecord' and 'SingleRecord'.
             max_concurrent_transforms (int): The maximum number of HTTP requests to be made to
                 each individual transform container at one time.
             max_payload (int): Maximum size of the payload in a single HTTP request to the

src/sagemaker/tensorflow/estimator.py

@@ -772,7 +772,7 @@ def transformer(
             instance_count (int): Number of EC2 instances to use.
             instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'.
             strategy (str): The strategy used to decide how to batch records in a single request
-                (default: None). Valid values: 'MULTI_RECORD' and 'SINGLE_RECORD'.
+                (default: None). Valid values: 'MultiRecord' and 'SingleRecord'.
             assemble_with (str): How the output is assembled (default: None). Valid values: 'Line'
                 or 'None'.
             output_path (str): S3 location for saving the transform result. If not specified,

src/sagemaker/utils.py

@@ -530,7 +530,9 @@ def _create_or_update_code_dir(
     """
     code_dir = os.path.join(model_dir, "code")
     if os.path.exists(code_dir):
-        shutil.rmtree(code_dir, ignore_errors=True)
+        for filename in os.listdir(code_dir):
+            if filename.endswith(".py"):
+                os.remove(os.path.join(code_dir, filename))
     if source_directory and source_directory.lower().startswith("s3://"):
         local_code_path = os.path.join(tmp, "local_code.tar.gz")
         download_file_from_url(source_directory, local_code_path, sagemaker_session)
@@ -539,9 +541,12 @@ def _create_or_update_code_dir(
             t.extractall(path=code_dir)
 
     elif source_directory:
+        if os.path.exists(code_dir):
+            shutil.rmtree(code_dir)
         shutil.copytree(source_directory, code_dir)
     else:
-        os.mkdir(code_dir)
+        if not os.path.exists(code_dir):
+            os.mkdir(code_dir)
         shutil.copy2(inference_script, code_dir)
 
     for dependency in dependencies:

src/sagemaker/workflow/airflow.py

@@ -802,8 +802,8 @@ def transform_config_from_estimator(
         model_name (str): model name (default: None). If not specified, one will
             be generated.
         strategy (str): The strategy used to decide how to batch records in a
-            single request (default: None). Valid values: 'MULTI_RECORD' and
-            'SINGLE_RECORD'.
+            single request (default: None). Valid values: 'MultiRecord' and
+            'SingleRecord'.
         assemble_with (str): How the output is assembled (default: None). Valid
             values: 'Line' or 'None'.
         output_path (str): S3 location for saving the transform result. If not

tests/integ/test_auto_ml.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import os
-import time
 
 import pytest
 import tests.integ
@@ -34,7 +33,7 @@
 TRAINING_DATA = os.path.join(DATA_DIR, "iris_training.csv")
 TEST_DATA = os.path.join(DATA_DIR, "iris_test.csv")
 PROBLEM_TYPE = "MultiClassClassification"
-JOB_NAME = "auto-ml-{}".format(time.strftime("%y%m%d-%H%M%S"))
+BASE_JOB_NAME = "auto-ml"
 
 # use a succeeded AutoML job to test describe and list candidates method, otherwise tests will run too long
 AUTO_ML_JOB_NAME = "python-sdk-integ-test-base-job"
@@ -119,11 +118,11 @@ def test_auto_ml_fit_optional_args(sagemaker_session):
     )
     inputs = TRAINING_DATA
     with timeout(minutes=AUTO_ML_DEFAULT_TIMEMOUT_MINUTES):
-        auto_ml.fit(inputs, job_name=JOB_NAME)
+        auto_ml.fit(inputs, job_name=unique_name_from_base(BASE_JOB_NAME))
 
-    auto_ml_desc = auto_ml.describe_auto_ml_job(job_name=JOB_NAME)
+    auto_ml_desc = auto_ml.describe_auto_ml_job(job_name=auto_ml.latest_auto_ml_job.job_name)
     assert auto_ml_desc["AutoMLJobStatus"] == "Completed"
-    assert auto_ml_desc["AutoMLJobName"] == JOB_NAME
+    assert auto_ml_desc["AutoMLJobName"] == auto_ml.latest_auto_ml_job.job_name
     assert auto_ml_desc["AutoMLJobObjective"] == job_objective
     assert auto_ml_desc["ProblemType"] == problem_type
     assert auto_ml_desc["OutputDataConfig"]["S3OutputPath"] == output_path

tests/integ/test_processing.py

@@ -20,7 +20,13 @@
 from sagemaker import Session
 from sagemaker.fw_registry import default_framework_uri
 
-from sagemaker.processing import ProcessingInput, ProcessingOutput, ScriptProcessor, Processor
+from sagemaker.processing import (
+    ProcessingInput,
+    ProcessingOutput,
+    ScriptProcessor,
+    Processor,
+    ProcessingJob,
+)
 from sagemaker.sklearn.processing import SKLearnProcessor
 from sagemaker.utils import sts_regional_endpoint
 from tests.integ import DATA_DIR
@@ -475,6 +481,37 @@ def test_script_processor_with_no_inputs_or_outputs(
 
     assert job_description["StoppingCondition"] == {"MaxRuntimeInSeconds": 3600}
 
+    job_from_name = ProcessingJob.from_processing_name(
+        sagemaker_session=sagemaker_session,
+        processing_job_name=job_description["ProcessingJobName"],
+    )
+    job_description = job_from_name.describe()
+
+    assert job_description["ProcessingInputs"][0]["InputName"] == "code"
+
+    assert job_description["ProcessingJobName"].startswith("test-script-processor-with-no-inputs")
+
+    assert job_description["ProcessingJobStatus"] == "Completed"
+
+    assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
+    assert (
+        job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
+    )
+    assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
+
+    assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
+    assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
+        "python3",
+        "/opt/ml/processing/input/code/dummy_script.py",
+    ]
+    assert job_description["AppSpecification"]["ImageUri"] == image_uri
+
+    assert job_description["Environment"] == {"DUMMY_ENVIRONMENT_VARIABLE": "dummy-value"}
+
+    assert ROLE in job_description["RoleArn"]
+
+    assert job_description["StoppingCondition"] == {"MaxRuntimeInSeconds": 3600}
+
 
 @pytest.mark.canary_quick
 def test_processor(sagemaker_session, image_uri, cpu_instance_type, output_kms_key):

tests/integ/test_pytorch_train.py

@@ -12,18 +12,23 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
-import os
-
 import numpy
+import os
 import pytest
-from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
-from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
-
+from sagemaker.pytorch.defaults import LATEST_PY2_VERSION
 from sagemaker.pytorch.estimator import PyTorch
 from sagemaker.pytorch.model import PyTorchModel
-from sagemaker.pytorch.defaults import LATEST_PY2_VERSION
 from sagemaker.utils import sagemaker_timestamp
 
+from tests.integ import (
+    test_region,
+    DATA_DIR,
+    PYTHON_VERSION,
+    TRAINING_DEFAULT_TIMEOUT_MINUTES,
+    EI_SUPPORTED_REGIONS,
+)
+from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
+
 MNIST_DIR = os.path.join(DATA_DIR, "pytorch_mnist")
 MNIST_SCRIPT = os.path.join(MNIST_DIR, "mnist.py")
 
@@ -120,6 +125,9 @@ def test_deploy_model(pytorch_training_job, sagemaker_session, cpu_instance_type
 
 
 @pytest.mark.skipif(PYTHON_VERSION == "py2", reason="PyTorch EIA does not support Python 2.")
+@pytest.mark.skipif(
+    test_region() not in EI_SUPPORTED_REGIONS, reason="EI isn't supported in that specific region."
+)
 def test_deploy_model_with_accelerator(sagemaker_session, cpu_instance_type):
     endpoint_name = "test-pytorch-deploy-eia-{}".format(sagemaker_timestamp())
     model_data = sagemaker_session.upload_data(path=EIA_MODEL)
@@ -134,7 +142,7 @@ def test_deploy_model_with_accelerator(sagemaker_session, cpu_instance_type):
         predictor = pytorch.deploy(
             initial_instance_count=1,
             instance_type=cpu_instance_type,
-            accelerator_type="ml.eia2.medium",
+            accelerator_type="ml.eia1.medium",
             endpoint_name=endpoint_name,
         )