Exponentially sleep during enpoint cleanup

Author: mufaddal-rohawala

tests/integ/test_model_monitor.py

@@ -92,7 +92,11 @@ def predictor(sagemaker_session, tensorflow_inference_latest_version):
         key_prefix="tensorflow-serving/models",
     )
     with tests.integ.timeout.timeout_and_delete_endpoint_by_name(
-        endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, hours=2
+        endpoint_name=endpoint_name,
+        sagemaker_session=sagemaker_session,
+        hours=2,
+        sleep_between_cleanup_attempts=20,
+        exponential_sleep=True,
     ):
         model = TensorFlowModel(
             model_data=model_data,

tests/integ/timeout.py

@@ -54,6 +54,7 @@ def timeout_and_delete_endpoint_by_name(
     minutes=45,
     hours=0,
     sleep_between_cleanup_attempts=10,
+    exponential_sleep=False,
 ):
     limit = seconds + 60 * minutes + 3600 * hours
 
@@ -83,7 +84,11 @@ def timeout_and_delete_endpoint_by_name(
                         # avoids the inner exception to be overwritten
                         pass
                 # trying to delete the resource again in 10 seconds
-                sleep(sleep_between_cleanup_attempts)
+                if exponential_sleep:
+                    _sleep_between_cleanup_attempts = (sleep_between_cleanup_attempts * (3 - attempts))
+                else:
+                    _sleep_between_cleanup_attempts = sleep_between_cleanup_attempts
+                sleep(_sleep_between_cleanup_attempts)
 
 
 @contextmanager
@@ -150,7 +155,7 @@ def _delete_schedules_associated_with_endpoint(sagemaker_session, endpoint_name)
             monitor.delete_monitoring_schedule()
         except Exception as e:
             LOGGER.warning(
-                "Failed to delete monitor {}".format(monitor.monitoring_schedule_name), e
+                "Failed to delete monitor {},\nError: {}".format(monitor.monitoring_schedule_name, e)
             )
 
 

tests/unit/test_timeout.py

@@ -19,7 +19,7 @@
 import time
 
 import pytest
-from mock import Mock, patch
+from mock import Mock, patch, call
 import stopit
 
 from botocore.exceptions import ClientError
@@ -44,6 +44,7 @@
 LONG_DURATION_TO_EXCEED_TIMEOUT = 0.002
 LONG_TIMEOUT_THAT_WILL_NEVER_BE_EXCEEDED = 10
 DURATION_TO_SLEEP_TO_ALLOW_BACKGROUND_THREAD_TO_COMPLETE = 0.2
+DURATION_TO_SLEEP = 0.01
 
 
 @pytest.fixture()
@@ -174,6 +175,39 @@ def test_timeout_and_delete_endpoint_by_name_retries_resource_deletion_on_failur
     assert session.delete_endpoint.call_count == 3
 
 
+@patch("tests.integ.timeout._show_logs", return_value=None, autospec=True)
+@patch("tests.integ.timeout._cleanup_logs", return_value=None, autospec=True)
+@patch(
+    "tests.integ.timeout._delete_schedules_associated_with_endpoint",
+    return_value=None,
+    autospec=True,
+)
+@patch('tests.integ.timeout.sleep', return_value=None)
+def test_timeout_and_delete_endpoint_by_name_retries_resource_deletion_on_failure_with_exp_sleep(
+    mock_sleep, _show_logs, _cleanup_logs, _delete_schedules_associated_with_endpoint, session
+):
+    session.delete_endpoint = Mock(
+        side_effect=ClientError(
+            error_response={"Error": {"Code": 403, "Message": "ValidationException"}},
+            operation_name="Unit Test",
+        )
+    )
+
+    with timeout_and_delete_endpoint_by_name(
+        endpoint_name=ENDPOINT_NAME,
+        sagemaker_session=session,
+        hours=0,
+        minutes=0,
+        seconds=LONG_TIMEOUT_THAT_WILL_NEVER_BE_EXCEEDED,
+        sleep_between_cleanup_attempts=DURATION_TO_SLEEP,
+        exponential_sleep=True,
+    ):
+        pass
+    assert session.delete_endpoint.call_count == 3
+    assert mock_sleep.call_count == 3
+    assert mock_sleep.mock_calls == [call(0.01), call(0.02), call(0.03)]
+
+
 @patch("tests.integ.timeout._show_logs", return_value=None, autospec=True)
 @patch("tests.integ.timeout._cleanup_logs", return_value=None, autospec=True)
 @patch(