Merge branch 'master' into enable-step-caching

Author: ahsan-z-khan

buildspec-slowtests.yml

@@ -0,0 +1,15 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
+  build:
+    commands:
+      - IGNORE_COVERAGE=-
+
+      # slow tests
+      - start_time=`date +%s`
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 16 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - ./ci-scripts/displaytime.sh 'py38 slow tests' $start_time

buildspec.yml

@@ -16,7 +16,7 @@ phases:
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron and not slow_test\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
 
   post_build:

setup.py

@@ -66,6 +66,7 @@ def read_version():
         "pytest<6.1.0",
         "pytest-cov",
         "pytest-rerunfailures",
+        "pytest-timeout",
         "pytest-xdist",
         "mock",
         "contextlib2",

src/sagemaker/image_uris.py

@@ -98,6 +98,8 @@ def retrieve(
             "mxnet-1.8.0-gpu-py37": "cu110-ubuntu16.04",
             "pytorch-1.6-gpu-py36": "cu110-ubuntu18.04-v3",
             "pytorch-1.6.0-gpu-py36": "cu110-ubuntu18.04",
+            "pytorch-1.6-gpu-py3": "cu110-ubuntu18.04-v3",
+            "pytorch-1.6.0-gpu-py3": "cu110-ubuntu18.04",
         }
         key = "-".join([framework, tag])
         if key in container_versions:

tests/data/multimodel/container/Dockerfile

@@ -1,4 +1,4 @@
-FROM 142577830533.dkr.ecr.us-east-2.amazonaws.com/ubuntu:16.04
+FROM public.ecr.aws/ubuntu/ubuntu:18.04
 
 # Set a docker label to advertise multi-model support on the container
 LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
@@ -15,7 +15,7 @@ RUN apt-get update && \
     curl \
     vim \
     && rm -rf /var/lib/apt/lists/* \
-    && curl -O https://bootstrap.pypa.io/3.5/get-pip.py \
+    && curl -O https://bootstrap.pypa.io/get-pip.py \
     && python3 get-pip.py
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1

tests/integ/sagemaker/lineage/conftest.py

@@ -155,7 +155,8 @@ def artifact_obj_with_association(sagemaker_session, artifact_obj):
 @pytest.fixture
 def trial_component_obj(sagemaker_session):
     trial_component_obj = trial_component.TrialComponent.create(
-        trial_component_name=name(), sagemaker_boto_client=sagemaker_session.sagemaker_client
+        trial_component_name=name(),
+        sagemaker_boto_client=sagemaker_session.sagemaker_client,
     )
     yield trial_component_obj
     time.sleep(0.5)

tests/integ/sagemaker/lineage/test_action.py

@@ -15,6 +15,9 @@
 
 import datetime
 import logging
+import time
+
+import pytest
 
 from sagemaker.lineage import action
 
@@ -80,6 +83,7 @@ def test_list(action_objs, sagemaker_session):
     assert action_names
 
 
+@pytest.mark.timeout(30)
 def test_tag(action_obj, sagemaker_session):
     tag = {"Key": "foo", "Value": "bar"}
     action_obj.set_tag(tag)
@@ -90,12 +94,14 @@ def test_tag(action_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0
     assert actual_tags[0] == tag
 
 
+@pytest.mark.timeout(30)
 def test_tags(action_obj, sagemaker_session):
     tags = [{"Key": "foo1", "Value": "bar1"}]
     action_obj.set_tags(tags)
@@ -106,6 +112,7 @@ def test_tags(action_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0

tests/integ/sagemaker/lineage/test_artifact.py

@@ -17,6 +17,8 @@
 import logging
 import time
 
+import pytest
+
 from sagemaker.lineage import artifact
 
 
@@ -111,6 +113,7 @@ def test_downstream_trials(trial_associated_artifact, trial_obj, sagemaker_sessi
     assert trial_obj.trial_name in trials
 
 
+@pytest.mark.timeout(30)
 def test_tag(artifact_obj, sagemaker_session):
     tag = {"Key": "foo", "Value": "bar"}
     artifact_obj.set_tag(tag)
@@ -121,12 +124,14 @@ def test_tag(artifact_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0
     assert actual_tags[0] == tag
 
 
+@pytest.mark.timeout(30)
 def test_tags(artifact_obj, sagemaker_session):
     tags = [{"Key": "foo1", "Value": "bar1"}]
     artifact_obj.set_tags(tags)
@@ -137,6 +142,7 @@ def test_tags(artifact_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0

tests/integ/sagemaker/lineage/test_association.py

@@ -16,6 +16,8 @@
 import datetime
 import time
 
+import pytest
+
 from sagemaker.lineage import association
 
 
@@ -55,6 +57,7 @@ def test_list(association_objs, sagemaker_session):
     assert association_keys_listed
 
 
+@pytest.mark.timeout(30)
 def test_set_tag(association_obj, sagemaker_session):
     tag = {"Key": "foo", "Value": "bar"}
     association_obj.set_tag(tag)
@@ -65,13 +68,14 @@ def test_set_tag(association_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
-        time.sleep(1)
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0
     assert actual_tags[0] == tag
 
 
+@pytest.mark.timeout(30)
 def test_tags(association_obj, sagemaker_session):
     tags = [{"Key": "foo1", "Value": "bar1"}]
     association_obj.set_tags(tags)
@@ -82,7 +86,7 @@ def test_tags(association_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
-        time.sleep(1)
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0

tests/integ/sagemaker/lineage/test_context.py

@@ -15,6 +15,9 @@
 
 import datetime
 import logging
+import time
+
+import pytest
 
 from sagemaker.lineage import context
 
@@ -78,6 +81,7 @@ def test_list(context_objs, sagemaker_session):
     assert context_names
 
 
+@pytest.mark.timeout(30)
 def test_tag(context_obj, sagemaker_session):
     tag = {"Key": "foo", "Value": "bar"}
     context_obj.set_tag(tag)
@@ -88,12 +92,14 @@ def test_tag(context_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0
     assert actual_tags[0] == tag
 
 
+@pytest.mark.timeout(30)
 def test_tags(context_obj, sagemaker_session):
     tags = [{"Key": "foo1", "Value": "bar1"}]
     context_obj.set_tags(tags)
@@ -104,6 +110,7 @@ def test_tags(context_obj, sagemaker_session):
         )["Tags"]
         if actual_tags:
             break
+        time.sleep(5)
     # When sagemaker-client-config endpoint-url is passed as argument to hit some endpoints,
     # length of actual tags will be greater than 1
     assert len(actual_tags) > 0

tests/integ/test_auto_ml.py

@@ -47,6 +47,7 @@
 }
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
     reason="AutoML is not supported in the region yet.",

tests/integ/test_clarify_model_monitor.py

@@ -285,6 +285,7 @@ def test_bias_monitor(sagemaker_session, scheduled_bias_monitor, endpoint_name,
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
@@ -393,6 +394,7 @@ def test_explainability_monitor(sagemaker_session, scheduled_explainability_moni
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",

tests/integ/test_inference_pipeline.py

@@ -149,6 +149,7 @@ def test_inference_pipeline_model_deploy(sagemaker_session, cpu_instance_type):
         assert "Could not find model" in str(exception.value)
 
 
+@pytest.mark.slow_test
 def test_inference_pipeline_model_deploy_and_update_endpoint(
     sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
 ):

tests/integ/test_lda.py

@@ -26,6 +26,7 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_LDA_REGIONS,
     reason="LDA image is not supported in certain regions",

tests/integ/test_model_quality_monitor.py

@@ -230,6 +230,7 @@ def test_model_quality_monitor(
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
@@ -254,6 +255,7 @@ def test_run_model_quality_monitor(
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",

tests/integ/test_multidatamodel.py

@@ -22,8 +22,6 @@
 import pytest
 from botocore.exceptions import ClientError
 
-import tests
-
 from sagemaker import utils
 from sagemaker.amazon.randomcutforest import RandomCutForest
 from sagemaker.deserializers import StringDeserializer
@@ -53,10 +51,6 @@ def container_image(sagemaker_session):
 
     docker_client = docker.from_env()
 
-    # Base image pull
-    base_image = "142577830533.dkr.ecr.us-east-2.amazonaws.com/ubuntu:16.04"
-    docker_client.images.pull(base_image, auth_config={"username": username, "password": password})
-
     # Build and tag docker image locally
     image, build_log = docker_client.images.build(
         path=os.path.join(DATA_DIR, "multimodel", "container"),
@@ -134,10 +128,6 @@ def _ecr_login(ecr_client):
     return username, password
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models(
     container_image, sagemaker_session, cpu_instance_type
 ):
@@ -198,10 +188,6 @@ def test_multi_data_model_deploy_pretrained_models(
 
 
 @pytest.mark.local_mode
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, sagemaker_session):
     timestamp = sagemaker_timestamp()
     endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
@@ -262,10 +248,6 @@ def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, s
         assert "Could not find endpoint" in str(exception.value)
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_trained_model_from_framework_estimator(
     container_image,
     sagemaker_session,
@@ -380,10 +362,7 @@ def _mxnet_training_job(
         return mx.create_model(image_uri=container_image)
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
+@pytest.mark.slow_test
 def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator(
     container_image, sagemaker_session, cpu_instance_type
 ):
@@ -481,10 +460,6 @@ def __rcf_training_job(
         return rcf_model
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models_update_endpoint(
     container_image, sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
 ):

tests/integ/test_mxnet.py

@@ -83,6 +83,7 @@ def test_attach_deploy(mxnet_training_job, sagemaker_session, cpu_instance_type)
         assert result is not None
 
 
+@pytest.mark.slow_test
 def test_deploy_estimator_with_different_instance_types(
     mxnet_training_job,
     sagemaker_session,
@@ -258,6 +259,7 @@ def test_deploy_model_with_tags_and_kms(
         assert endpoint_config["KmsKeyId"] == kms_key_arn
 
 
+@pytest.mark.slow_test
 def test_deploy_model_and_update_endpoint(
     mxnet_training_job,
     sagemaker_session,

tests/integ/test_tf.py

@@ -155,6 +155,7 @@ def test_mnist_distributed(
     )
 
 
+@pytest.mark.slow_test
 def test_mnist_async(sagemaker_session, cpu_instance_type, tf_full_version, tf_full_py_version):
     estimator = TensorFlow(
         entry_point=SCRIPT,