Set S3 environment variables (aws#112)

* Setting S3 environment variables before training starts

* Remove S3 environment variable setting in test training script

* Add unit tests

Author: icywang86rui

src/sagemaker_tensorflow_container/s3_utils.py

@@ -20,8 +20,6 @@
 
 def configure(model_dir, job_region):
 
-<<<<<<< HEAD
-<<<<<<< HEAD
     os.environ['S3_REGION'] = _s3_region(job_region, model_dir)
 
     # setting log level to WARNING
@@ -42,37 +40,4 @@ def _s3_region(job_region, model_dir):
 
         return bucket_location or job_region
     else:
-        return job_region
-=======
-    if not model_dir:
-        return
-=======
-    os.environ['S3_REGION'] = _s3_region(job_region, model_dir)
->>>>>>> Add Keras support (#126)
-
-    # setting log level to WARNING
-    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
-    os.environ['S3_USE_HTTPS'] = '1'
-
-
-def _s3_region(job_region, model_dir):
-    if model_dir and model_dir.startswith('s3://'):
-        s3 = boto3.client('s3', region_name=job_region)
-
-        # We get the AWS region of the checkpoint bucket, which may be different from
-        # the region this container is currently running in.
-        parsed_url = urlparse(model_dir)
-        bucket_name = parsed_url.netloc
-
-<<<<<<< HEAD
-    # setting log level to WARNING
-    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
-    os.environ['S3_USE_HTTPS'] = '1'
->>>>>>> Set S3 environment variables (#112)
-=======
-        bucket_location = s3.get_bucket_location(Bucket=bucket_name)['LocationConstraint']
-
-        return bucket_location or job_region
-    else:
-        return job_region
->>>>>>> Add Keras support (#126)
+        return job_region

test/resources/mnist/distributed_mnist.py

@@ -136,11 +136,6 @@ def _parse_args():
     tf_logger = tf_logging._get_logger()
     tf_logger.handlers = [_handler]
 
-    if args.checkpoint_path.startswith('s3://'):
-        os.environ['S3_REGION'] = 'us-west-2'
-        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
-        os.environ['S3_USE_HTTPS'] = '1'
-
     train_data, train_labels = _load_training_data(args.train)
     eval_data, eval_labels = _load_testing_data(args.train)
 

test/unit/test_s3_utils.py

@@ -32,8 +32,6 @@ def test_configure(client):
     client.return_value = s3
     loc = {'LocationConstraint': BUCKET_REGION}
     s3.get_bucket_location.return_value = loc
-<<<<<<< HEAD
-<<<<<<< HEAD
 
     s3_utils.configure(MODEL_DIR, JOB_REGION)
 
@@ -47,25 +45,4 @@ def test_configure_local_dir():
 
     assert os.environ['S3_REGION'] == JOB_REGION
     assert os.environ['TF_CPP_MIN_LOG_LEVEL'] == '1'
-    assert os.environ['S3_USE_HTTPS'] == '1'
-=======
-=======
-
->>>>>>> Add Keras support (#126)
-    s3_utils.configure(MODEL_DIR, JOB_REGION)
-
-    assert os.environ['S3_REGION'] == BUCKET_REGION
-    assert os.environ['TF_CPP_MIN_LOG_LEVEL'] == '1'
-    assert os.environ['S3_USE_HTTPS'] == '1'
-<<<<<<< HEAD
->>>>>>> Set S3 environment variables (#112)
-=======
-
-
-def test_configure_local_dir():
-    s3_utils.configure('/opt/ml/model', JOB_REGION)
-
-    assert os.environ['S3_REGION'] == JOB_REGION
-    assert os.environ['TF_CPP_MIN_LOG_LEVEL'] == '1'
-    assert os.environ['S3_USE_HTTPS'] == '1'
->>>>>>> Add Keras support (#126)
+    assert os.environ['S3_USE_HTTPS'] == '1'