Merge branch 'master' into fix-disable-profiler-settings

Author: Dan

CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## v2.25.1 (2021-02-20)
+
+### Bug Fixes and Other Changes
+
+ * Add tests for visualizer to improve test coverage
+
+### Documentation Changes
+
+ * specify correct return type
+
+### Testing and Release Infrastructure
+
+ * rename canary_quick pytest mark to release
+
+## v2.25.0 (2021-02-19)
+
+### Features
+
+ * Enable step caching
+ * Add other Neo supported regions for Inferentia inference images
+
+### Bug Fixes and Other Changes
+
+ * remove FailStep from pipelines
+ * use sagemaker_session in workflow tests
+ * use ECR public for multidatamodel tests
+ * add the mapping from py3 to cuda11 images
+ * Add 30s cap time for tag tests
+ * add build spec for slow tests
+ * mark top 10 slow tests
+ * remove slow test_run_xxx_monitor_baseline tests
+ * pin astroid to 2.4.2
+
+### Testing and Release Infrastructure
+
+ * unmark more flaky integ tests
+ * remove canary_quick pytest mark from flaky/unnecessary tests
+ * remove python3.8 from buildspec
+ * remove py38 tox env
+ * fix release buildspec typo
+ * unblock regional release builds
+ * lower test TPS for experiment analytics
+ * move package preparation and publishing to the deploy step
+
 ## v2.24.5 (2021-02-12)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.24.6.dev0
+2.25.2.dev0

buildspec-localmodetests.yml

@@ -11,5 +11,5 @@ phases:
 
       # local mode tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
-      - ./ci-scripts/displaytime.sh 'py38 local mode' $start_time
+      - execute-command-if-has-matching-changes "tox -e py37 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
+      - ./ci-scripts/displaytime.sh 'py37 local mode' $start_time

buildspec-release.yml

@@ -15,7 +15,7 @@ phases:
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py37,py38 -- tests/unit
+        tox -e py36,py37 -- tests/unit
 
       # run a subset of the integration tests
-      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2
+      - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m "not (local_mode or slow_test)" -n 32 --boxed --reruns 2

buildspec-slowtests.yml

@@ -0,0 +1,15 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
+  build:
+    commands:
+      - IGNORE_COVERAGE=-
+
+      # slow tests
+      - start_time=`date +%s`
+      - execute-command-if-has-matching-changes "tox -e py37 -- tests/integ -m slow_test -n 16 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - ./ci-scripts/displaytime.sh 'py37 slow tests' $start_time

buildspec-unittests.yml

@@ -18,5 +18,5 @@ phases:
       - start_time=`date +%s`
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py37,py38 --parallel all -- tests/unit
-      - ./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time
+        tox -e py36,py37 --parallel all -- tests/unit
+      - ./ci-scripts/displaytime.sh 'py36,py37 unit' $start_time

buildspec.yml

@@ -11,13 +11,13 @@ phases:
 
       # run integration tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "python3.8 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
+      - execute-command-if-has-matching-changes "python3.7 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'build queue' $start_time
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
-      - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py37 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
+      - ./ci-scripts/displaytime.sh 'py37 tests/integ' $start_time
 
   post_build:
     finally:

doc/frameworks/mxnet/using_mxnet.rst

@@ -377,7 +377,7 @@ It loads the model parameters from a ``model.params`` file in the SageMaker mode
         return net
 
 MXNet on Amazon SageMaker has support for `Elastic Inference <https://docs.aws.amazon.com/sagemaker/latest/dg/ei.html>`__, which allows for inference acceleration to a hosted endpoint for a fraction of the cost of using a full GPU instance.
-In order to load and serve your MXNet model through Amazon Elastic Inference, the MXNet context passed to your MXNet Symbol or Module object within your ``model_fn`` needs to be set to ``eia``, as shown `here <https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-mxnet-elastic-inference.html#ei-mxnet>`__.
+In order to load and serve your MXNet model through Amazon Elastic Inference, import the ``eimx`` Python package and make one change in the code to partition your model and optimize it for the ``EIA`` back end, as shown `here <https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-mxnet-elastic-inference.html#ei-mxnet>`__.
 
 Based on the example above, the following code-snippet shows an example custom ``model_fn`` implementation, which enables loading and serving our MXNet model through Amazon Elastic Inference.
 
@@ -392,11 +392,12 @@ Based on the example above, the following code-snippet shows an example custom `
         Returns:
             mxnet.gluon.nn.Block: a Gluon network (for this example)
         """
-        net = models.get_model('resnet34_v2', ctx=mx.eia(), pretrained=False, classes=10)
-        net.load_params('%s/model.params' % model_dir, ctx=mx.eia())
+        net = models.get_model('resnet34_v2', ctx=mx.cpu(), pretrained=False, classes=10)
+        net.load_params('%s/model.params' % model_dir, ctx=mx.cpu())
+        net.hybridize(backend='EIA', static_alloc=True, static_shape=True)
         return net
 
-The `default_model_fn <https://github.com/aws/sagemaker-mxnet-container/pull/55/files#diff-aabf018d906ed282a3c738377d19a8deR71>`__ loads and serve your model through Elastic Inference, if applicable, within the Amazon SageMaker MXNet containers.
+If you are using MXNet 1.5.1 and earlier, the `default_model_fn <https://github.com/aws/sagemaker-mxnet-container/pull/55/files#diff-aabf018d906ed282a3c738377d19a8deR71>`__ loads and serve your model through Elastic Inference, if applicable, within the Amazon SageMaker MXNet containers.
 
 For more information on how to enable MXNet to interact with Amazon Elastic Inference, see `Use Elastic Inference with MXNet <https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-mxnet-elastic-inference.html>`__.
 

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -110,8 +110,6 @@ Steps
 
 .. autoclass:: sagemaker.workflow.steps.ProcessingStep
 
-.. autoclass:: sagemaker.workflow.steps.FailStep
-
 Utilities
 ---------
 

setup.py

@@ -66,6 +66,7 @@ def read_version():
         "pytest<6.1.0",
         "pytest-cov",
         "pytest-rerunfailures",
+        "pytest-timeout",
         "pytest-xdist",
         "mock",
         "contextlib2",

src/sagemaker/clarify.py

@@ -21,7 +21,7 @@
 import tempfile
 
 from sagemaker.processing import ProcessingInput, ProcessingOutput, Processor
-from sagemaker import image_uris, utils
+from sagemaker import image_uris, s3, utils
 
 
 class DataConfig:
@@ -405,9 +405,15 @@ def _run(
             analysis_config_file = os.path.join(tmpdirname, "analysis_config.json")
             with open(analysis_config_file, "w") as f:
                 json.dump(analysis_config, f)
+            s3_analysis_config_file = _upload_analysis_config(
+                analysis_config_file,
+                data_config.s3_output_path,
+                self.sagemaker_session,
+                kms_key,
+            )
             config_input = ProcessingInput(
                 input_name="analysis_config",
-                source=analysis_config_file,
+                source=s3_analysis_config_file,
                 destination=self._CLARIFY_CONFIG_INPUT,
                 s3_data_type="S3Prefix",
                 s3_input_mode="File",
@@ -638,6 +644,30 @@ def run_explainability(
         self._run(data_config, analysis_config, wait, logs, job_name, kms_key)
 
 
+def _upload_analysis_config(analysis_config_file, s3_output_path, sagemaker_session, kms_key):
+    """Uploads the local analysis_config_file to the s3_output_path.
+
+    Args:
+        analysis_config_file (str): File path to the local analysis config file.
+        s3_output_path (str): S3 prefix to store the analysis config file.
+        sagemaker_session (:class:`~sagemaker.session.Session`):
+            Session object which manages interactions with Amazon SageMaker and
+            any other AWS services needed. If not specified, the processor creates
+            one using the default AWS configuration chain.
+        kms_key (str): The ARN of the KMS key that is used to encrypt the
+            user code file (default: None).
+
+    Returns:
+        The S3 uri of the uploaded file.
+    """
+    return s3.S3Uploader.upload(
+        local_path=analysis_config_file,
+        desired_s3_uri=s3_output_path,
+        sagemaker_session=sagemaker_session,
+        kms_key=kms_key,
+    )
+
+
 def _set(value, key, dictionary):
     """Sets dictionary[key] = value if value is not None."""
     if value is not None:

src/sagemaker/feature_store/feature_group.py

@@ -185,6 +185,7 @@ def _ingest_single_batch(
                     feature_name=data_frame.columns[index], value_as_string=str(row[index])
                 )
                 for index in range(len(row))
+                if pd.notna(row[index])
             ]
             sagemaker_session.put_record(
                 feature_group_name=feature_group_name, record=[value.to_dict() for value in record]

src/sagemaker/image_uris.py

@@ -98,6 +98,8 @@ def retrieve(
             "mxnet-1.8.0-gpu-py37": "cu110-ubuntu16.04",
             "pytorch-1.6-gpu-py36": "cu110-ubuntu18.04-v3",
             "pytorch-1.6.0-gpu-py36": "cu110-ubuntu18.04",
+            "pytorch-1.6-gpu-py3": "cu110-ubuntu18.04-v3",
+            "pytorch-1.6.0-gpu-py3": "cu110-ubuntu18.04",
         }
         key = "-".join([framework, tag])
         if key in container_versions:

src/sagemaker/lineage/visualizer.py

@@ -105,7 +105,7 @@ def _get_start_arn_from_pipeline_execution_step(self, pipeline_execution_step):
             return None
 
         metadata = pipeline_execution_step["Metadata"]
-        jobs = ["TrainingJob", "ProccessingJob", "TransformJob"]
+        jobs = ["TrainingJob", "ProcessingJob", "TransformJob"]
         for job in jobs:
             if job in metadata and metadata[job]:
                 job_arn = metadata[job]["Arn"]

src/sagemaker/model.py

@@ -146,7 +146,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         if self.model_data is None:
             raise ValueError("SageMaker Model Package cannot be created without model data.")

src/sagemaker/mxnet/model.py

@@ -183,7 +183,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)

src/sagemaker/pytorch/model.py

@@ -182,7 +182,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)

src/sagemaker/tensorflow/model.py

@@ -226,7 +226,7 @@ def register(
             description (str): Model Package description (default: None).
 
         Returns:
-            str: A string of SageMaker Model Package ARN.
+            A `sagemaker.model.ModelPackage` instance.
         """
         instance_type = inference_instances[0]
         self._init_sagemaker_session_if_does_not_exist(instance_type)