Merge branch 'master' into update-return-values

Author: ahsan-z-khan

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## v2.25.0 (2021-02-19)
+
+### Features
+
+ * Enable step caching
+ * Add other Neo supported regions for Inferentia inference images
+
+### Bug Fixes and Other Changes
+
+ * remove FailStep from pipelines
+ * use sagemaker_session in workflow tests
+ * use ECR public for multidatamodel tests
+ * add the mapping from py3 to cuda11 images
+ * Add 30s cap time for tag tests
+ * add build spec for slow tests
+ * mark top 10 slow tests
+ * remove slow test_run_xxx_monitor_baseline tests
+ * pin astroid to 2.4.2
+
+### Testing and Release Infrastructure
+
+ * unmark more flaky integ tests
+ * remove canary_quick pytest mark from flaky/unnecessary tests
+ * remove python3.8 from buildspec
+ * remove py38 tox env
+ * fix release buildspec typo
+ * unblock regional release builds
+ * lower test TPS for experiment analytics
+ * move package preparation and publishing to the deploy step
+
 ## v2.24.5 (2021-02-12)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.24.6.dev0
+2.25.1.dev0

tests/integ/test_airflow_config.py

@@ -60,7 +60,6 @@
 SINGLE_INSTANCE_COUNT = 1
 
 
-@pytest.mark.canary_quick
 def test_byo_airflow_config_uploads_data_source_to_s3_when_inputs_provided(
     sagemaker_session, cpu_instance_type
 ):
@@ -92,7 +91,6 @@ def test_byo_airflow_config_uploads_data_source_to_s3_when_inputs_provided(
         )
 
 
-@pytest.mark.canary_quick
 def test_kmeans_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         kmeans = KMeans(
@@ -153,7 +151,6 @@ def test_fm_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_inst
         )
 
 
-@pytest.mark.canary_quick
 def test_ipinsights_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         data_path = os.path.join(DATA_DIR, "ipinsights")
@@ -214,7 +211,6 @@ def test_knn_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
     tests.integ.test_region() in tests.integ.NO_LDA_REGIONS,
     reason="LDA image is not supported in certain regions",
 )
-@pytest.mark.canary_quick
 def test_lda_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         data_path = os.path.join(DATA_DIR, "lda")
@@ -247,7 +243,6 @@ def test_lda_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
         )
 
 
-@pytest.mark.canary_quick
 def test_linearlearner_airflow_config_uploads_data_source_to_s3(
     sagemaker_session, cpu_instance_type
 ):
@@ -312,7 +307,6 @@ def test_linearlearner_airflow_config_uploads_data_source_to_s3(
         )
 
 
-@pytest.mark.canary_quick
 def test_ntm_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         data_path = os.path.join(DATA_DIR, "ntm")
@@ -346,7 +340,6 @@ def test_ntm_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
         )
 
 
-@pytest.mark.canary_quick
 def test_pca_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         pca = PCA(
@@ -373,7 +366,6 @@ def test_pca_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
         )
 
 
-@pytest.mark.canary_quick
 def test_rcf_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_instance_type):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         # Generate a thousand 14-dimensional datapoints.
@@ -439,7 +431,6 @@ def test_chainer_airflow_config_uploads_data_source_to_s3(
         )
 
 
-@pytest.mark.canary_quick
 def test_mxnet_airflow_config_uploads_data_source_to_s3(
     sagemaker_session,
     cpu_instance_type,
@@ -514,7 +505,6 @@ def test_sklearn_airflow_config_uploads_data_source_to_s3(
         )
 
 
-@pytest.mark.canary_quick
 def test_tf_airflow_config_uploads_data_source_to_s3(
     sagemaker_session,
     cpu_instance_type,
@@ -548,7 +538,6 @@ def test_tf_airflow_config_uploads_data_source_to_s3(
         )
 
 
-@pytest.mark.canary_quick
 def test_xgboost_airflow_config_uploads_data_source_to_s3(
     sagemaker_session, cpu_instance_type, xgboost_latest_version
 ):
@@ -574,7 +563,6 @@ def test_xgboost_airflow_config_uploads_data_source_to_s3(
         )
 
 
-@pytest.mark.canary_quick
 def test_pytorch_airflow_config_uploads_data_source_to_s3_when_inputs_not_provided(
     sagemaker_session,
     cpu_instance_type,

tests/integ/test_tuner_multi_algo.py

@@ -97,7 +97,6 @@ def estimator_knn(sagemaker_session, cpu_instance_type):
     return estimator
 
 
-@pytest.mark.canary_quick
 def test_multi_estimator_tuning(
     sagemaker_session, estimator_fm, estimator_knn, data_set, cpu_instance_type
 ):

tests/unit/sagemaker/lineage/test_visualizer.py

@@ -119,3 +119,137 @@ def test_trial_component_name(sagemaker_session, vizualizer):
     )
 
     pd.testing.assert_frame_equal(expected_dataframe, df)
+
+
+def test_model_package_arn(sagemaker_session, vizualizer):
+    name = "model_package_arn"
+
+    sagemaker_session.sagemaker_client.list_artifacts.return_value = {
+        "ArtifactSummaries": [{"ArtifactArn": "artifact-arn"}]
+    }
+
+    sagemaker_session.sagemaker_client.list_associations.side_effect = [
+        {
+            "AssociationSummaries": [
+                {
+                    "SourceArn": "a:b:c:d:e:artifact/src-arn-1",
+                    "SourceName": "source-name-1",
+                    "SourceType": "source-type-1",
+                    "DestinationArn": "a:b:c:d:e:artifact/dest-arn-1",
+                    "DestinationName": "dest-name-1",
+                    "DestinationType": "dest-type-1",
+                    "AssociationType": "type-1",
+                }
+            ]
+        },
+        {
+            "AssociationSummaries": [
+                {
+                    "SourceArn": "a:b:c:d:e:artifact/src-arn-2",
+                    "SourceName": "source-name-2",
+                    "SourceType": "source-type-2",
+                    "DestinationArn": "a:b:c:d:e:artifact/dest-arn-2",
+                    "DestinationName": "dest-name-2",
+                    "DestinationType": "dest-type-2",
+                    "AssociationType": "type-2",
+                }
+            ]
+        },
+    ]
+
+    df = vizualizer.show(model_package_arn=name)
+
+    sagemaker_session.sagemaker_client.list_artifacts.assert_called_with(
+        SourceUri=name,
+    )
+
+    expected_calls = [
+        unittest.mock.call(
+            DestinationArn="artifact-arn",
+        ),
+        unittest.mock.call(
+            SourceArn="artifact-arn",
+        ),
+    ]
+    assert expected_calls == sagemaker_session.sagemaker_client.list_associations.mock_calls
+
+    expected_dataframe = pd.DataFrame.from_dict(
+        OrderedDict(
+            [
+                ("Name/Source", ["source-name-1", "dest-name-2"]),
+                ("Direction", ["Input", "Output"]),
+                ("Type", ["source-type-1", "dest-type-2"]),
+                ("Association Type", ["type-1", "type-2"]),
+                ("Lineage Type", ["artifact", "artifact"]),
+            ]
+        )
+    )
+
+    pd.testing.assert_frame_equal(expected_dataframe, df)
+
+
+def test_endpoint_arn(sagemaker_session, vizualizer):
+    name = "endpoint_arn"
+
+    sagemaker_session.sagemaker_client.list_contexts.return_value = {
+        "ContextSummaries": [{"ContextArn": "context-arn"}]
+    }
+
+    sagemaker_session.sagemaker_client.list_associations.side_effect = [
+        {
+            "AssociationSummaries": [
+                {
+                    "SourceArn": "a:b:c:d:e:context/src-arn-1",
+                    "SourceName": "source-name-1",
+                    "SourceType": "source-type-1",
+                    "DestinationArn": "a:b:c:d:e:context/dest-arn-1",
+                    "DestinationName": "dest-name-1",
+                    "DestinationType": "dest-type-1",
+                    "AssociationType": "type-1",
+                }
+            ]
+        },
+        {
+            "AssociationSummaries": [
+                {
+                    "SourceArn": "a:b:c:d:e:context/src-arn-2",
+                    "SourceName": "source-name-2",
+                    "SourceType": "source-type-2",
+                    "DestinationArn": "a:b:c:d:e:context/dest-arn-2",
+                    "DestinationName": "dest-name-2",
+                    "DestinationType": "dest-type-2",
+                    "AssociationType": "type-2",
+                }
+            ]
+        },
+    ]
+
+    df = vizualizer.show(endpoint_arn=name)
+
+    sagemaker_session.sagemaker_client.list_contexts.assert_called_with(
+        SourceUri=name,
+    )
+
+    expected_calls = [
+        unittest.mock.call(
+            DestinationArn="context-arn",
+        ),
+        unittest.mock.call(
+            SourceArn="context-arn",
+        ),
+    ]
+    assert expected_calls == sagemaker_session.sagemaker_client.list_associations.mock_calls
+
+    expected_dataframe = pd.DataFrame.from_dict(
+        OrderedDict(
+            [
+                ("Name/Source", ["source-name-1", "dest-name-2"]),
+                ("Direction", ["Input", "Output"]),
+                ("Type", ["source-type-1", "dest-type-2"]),
+                ("Association Type", ["type-1", "type-2"]),
+                ("Lineage Type", ["context", "context"]),
+            ]
+        )
+    )
+
+    pd.testing.assert_frame_equal(expected_dataframe, df)