increase timeout to 120 and ping interval to 5

Author: Dan Choi

src/sagemaker/local/entities.py

@@ -29,7 +29,7 @@
 logger.setLevel(logging.WARNING)
 
 _UNUSED_ARN = 'local:arn-does-not-matter'
-HEALTH_CHECK_TIMEOUT_LIMIT = 30
+HEALTH_CHECK_TIMEOUT_LIMIT = 120
 
 
 class _LocalTrainingJob(object):
@@ -405,7 +405,7 @@ def _wait_for_serving_container(serving_port):
 
     endpoint_url = 'http://localhost:%s/ping' % serving_port
     while True:
-        i += 1
+        i += 5
         if i >= HEALTH_CHECK_TIMEOUT_LIMIT:
             raise RuntimeError('Giving up, endpoint didn\'t launch correctly')
 
@@ -416,7 +416,7 @@ def _wait_for_serving_container(serving_port):
         else:
             return
 
-        time.sleep(1)
+        time.sleep(5)
 
 
 def _perform_request(endpoint_url, pool_manager=None):

src/sagemaker/local/image.py

@@ -36,10 +36,12 @@
 import sagemaker.local.data
 import sagemaker.local.utils
 import sagemaker.utils
+from sagemaker.local.entities import HEALTH_CHECK_TIMEOUT_LIMIT
 
 CONTAINER_PREFIX = 'algo'
 DOCKER_COMPOSE_FILENAME = 'docker-compose.yaml'
 
+
 # Environment variables to be set during training
 REGION_ENV_NAME = 'AWS_REGION'
 TRAINING_JOB_NAME_ENV_NAME = 'TRAINING_JOB_NAME'
@@ -359,6 +361,9 @@ def _generate_compose_file(self, command, additional_volumes=None, additional_en
         additional_env_var_list = ['{}={}'.format(k, v) for k, v in additional_env_vars.items()]
         environment.extend(additional_env_var_list)
 
+        if os.environ.get('COMPOSE_HTTP_TIMEOUT') is None:
+            os.environ['COMPOSE_HTTP_TIMEOUT'] = str(HEALTH_CHECK_TIMEOUT_LIMIT)
+
         if command == 'train':
             optml_dirs = {'output', 'output/data', 'input'}