fix: prevent integration test's timeout functions from hiding failures

tests/integ/timeout.py

@@ -46,7 +46,12 @@ def timeout(seconds=0, minutes=0, hours=0):
 
 @contextmanager
 def timeout_and_delete_endpoint_by_name(
-    endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0
+    endpoint_name,
+    sagemaker_session,
+    seconds=0,
+    minutes=45,
+    hours=0,
+    sleep_between_cleanup_attempts=10,
 ):
     limit = seconds + 60 * minutes + 3600 * hours
 
@@ -67,18 +72,18 @@ def timeout_and_delete_endpoint_by_name(
                     _show_logs(endpoint_name, "Endpoints", sagemaker_session)
                     if no_errors:
                         _cleanup_logs(endpoint_name, "Endpoints", sagemaker_session)
-                    return
+                        return
                 except ClientError as ce:
                     if ce.response["Error"]["Code"] == "ValidationException":
                         # avoids the inner exception to be overwritten
                         pass
                 # trying to delete the resource again in 10 seconds
-                sleep(10)
+                sleep(sleep_between_cleanup_attempts)
 
 
 @contextmanager
 def timeout_and_delete_model_with_transformer(
-    transformer, sagemaker_session, seconds=0, minutes=0, hours=0
+    transformer, sagemaker_session, seconds=0, minutes=0, hours=0, sleep_between_cleanup_attempts=10
 ):
     limit = seconds + 60 * minutes + 3600 * hours
 
@@ -103,7 +108,7 @@ def timeout_and_delete_model_with_transformer(
                 except ClientError as ce:
                     if ce.response["Error"]["Code"] == "ValidationException":
                         pass
-                sleep(10)
+                sleep(sleep_between_cleanup_attempts)
 
 
 def _show_logs(resource_name, resource_type, sagemaker_session):

tests/unit/test_timeout.py

@@ -0,0 +1,70 @@
+# Copyright 2017-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""
+This class tests the timeout.py class in the integration tests.
+This is to prevent regressions that cause the timeout function to hide failed tests.
+"""
+from __future__ import absolute_import
+
+import pytest
+from mock import MagicMock, patch
+
+from tests.integ.timeout import (
+    timeout,
+    timeout_and_delete_endpoint_by_name,
+    timeout_and_delete_model_with_transformer,
+)
+
+
+EXCEPTION_MESSAGE = "This Exception is expected and should not be swallowed by the timeout."
+
+
+def test_timeout_fails_correctly_when_calling_test_throws_exception():
+    with pytest.raises(ValueError) as exception:
+        with timeout(minutes=1):
+            raise ValueError(EXCEPTION_MESSAGE)
+        assert EXCEPTION_MESSAGE in str(exception.value)
+
+
+@patch("sagemaker.session")
+def test_timeout_and_delete_endpoint_by_name_fails_when_calling_test_throws_exception(session):
+    session.delete_endpoint = MagicMock()
+
+    with pytest.raises(ValueError) as exception:
+        with timeout_and_delete_endpoint_by_name(
+            endpoint_name="fake-endpoint_name",
+            sagemaker_session=session,
+            minutes=1,
+            sleep_between_cleanup_attempts=0,
+        ):
+            raise ValueError(EXCEPTION_MESSAGE)
+        assert EXCEPTION_MESSAGE in str(exception.value)
+    assert session.delete_endpoint.call_count == 3
+
+
+@patch("sagemaker.session")
+def test_timeout_and_delete_model_with_transformer_fails_when_calling_test_throws_exception(
+    session
+):
+    transformer = MagicMock()
+
+    with pytest.raises(ValueError) as exception:
+        with timeout_and_delete_model_with_transformer(
+            sagemaker_session=session,
+            transformer=transformer,
+            minutes=1,
+            sleep_between_cleanup_attempts=0,
+        ):
+            raise ValueError(EXCEPTION_MESSAGE)
+        assert EXCEPTION_MESSAGE in str(exception.value)
+    assert transformer.delete_model.call_count == 3