Unify usage of /tmp folder and .tmp extension for writing temp files (aws#163)

* unify-tmp-path

* ignore tmp files

* fix incorrect path bug

* renamed SAGEMAKER_TEMP_PATH_SUFFIX to SMDEBUG_TEMP_PATH_SUFFIX

Author: NihalHarish

smdebug/core/access_layer/file.py

@@ -4,13 +4,11 @@
 
 # First Party
 from smdebug.core.logger import get_logger
-from smdebug.core.sagemaker_utils import is_sagemaker_job
 
 # Local
 from .base import TSAccessBase
 
-NON_SAGEMAKER_TEMP_PATH_PREFIX = "/tmp"
-SAGEMAKER_TEMP_PATH_SUFFIX = ".tmp"
+SMDEBUG_TEMP_PATH_SUFFIX = ".tmp"
 
 
 def ensure_dir(file_path, is_file=True):
@@ -23,13 +21,7 @@ def ensure_dir(file_path, is_file=True):
 
 
 def get_temp_path(file_path):
-    directory = os.path.dirname(file_path)
-    if is_sagemaker_job():
-        temp_path = file_path + SAGEMAKER_TEMP_PATH_SUFFIX
-    else:
-        if len(file_path) > 0 and file_path[0] == "/":
-            file_path = file_path[1:]
-        temp_path = os.path.join(NON_SAGEMAKER_TEMP_PATH_PREFIX, file_path)
+    temp_path = file_path + SMDEBUG_TEMP_PATH_SUFFIX
     return temp_path
 
 

smdebug/core/utils.py

@@ -84,11 +84,10 @@ def list_files_in_directory(directory, file_regex=None):
 
 
 def list_collection_files_in_directory(directory):
-    collections_directory = get_path_to_collections(directory)
     import re
 
-    collections_file_regex = re.compile(".*_?collections.json")
-    return list_files_in_directory(collections_directory, file_regex=collections_file_regex)
+    collections_file_regex = re.compile(".*_?collections.json$")
+    return list_files_in_directory(directory, file_regex=collections_file_regex)
 
 
 def serialize_tf_device(device: str) -> str:

smdebug/trials/local_trial.py

@@ -4,7 +4,7 @@
 # First Party
 from smdebug.core.collection_manager import CollectionManager
 from smdebug.core.index_reader import LocalIndexReader
-from smdebug.core.utils import get_path_to_collections, list_files_in_directory
+from smdebug.core.utils import get_path_to_collections, list_collection_files_in_directory
 
 # Local
 from .trial import Trial
@@ -37,7 +37,7 @@ def __init__(
         self._load_tensors()
 
     def _get_collection_files(self) -> list:
-        return list_files_in_directory(get_path_to_collections(self.path))
+        return list_collection_files_in_directory(get_path_to_collections(self.path))
 
     def _load_tensors_from_index_tensors(self, index_tensors_dict):
         for tname in index_tensors_dict:

tests/core/test_paths.py

@@ -5,11 +5,7 @@
 
 # First Party
 import smdebug.pytorch as smd
-from smdebug.core.access_layer.file import (
-    NON_SAGEMAKER_TEMP_PATH_PREFIX,
-    SAGEMAKER_TEMP_PATH_SUFFIX,
-    get_temp_path,
-)
+from smdebug.core.access_layer.file import SMDEBUG_TEMP_PATH_SUFFIX, get_temp_path
 from smdebug.core.access_layer.utils import training_has_ended
 from smdebug.core.hook_utils import verify_and_get_out_dir
 from smdebug.core.utils import SagemakerSimulator, ScriptSimulator
@@ -93,14 +89,12 @@ def test_temp_paths():
             "/opt/ml/output/tensors/events/a/b",
         ]:
             temp_path = get_temp_path(path)
-            assert temp_path.endswith(SAGEMAKER_TEMP_PATH_SUFFIX)
-            assert not temp_path.startswith(NON_SAGEMAKER_TEMP_PATH_PREFIX)
+            assert temp_path.endswith(SMDEBUG_TEMP_PATH_SUFFIX)
 
     with ScriptSimulator() as sim:
         for path in ["/a/b/c", "/opt/ml/output/a", "a/b/c"]:
             temp_path = get_temp_path(path)
-            assert not SAGEMAKER_TEMP_PATH_SUFFIX in temp_path
-            assert temp_path.startswith(NON_SAGEMAKER_TEMP_PATH_PREFIX)
+            assert temp_path.endswith(SMDEBUG_TEMP_PATH_SUFFIX)
 
 
 def test_s3_path_that_exists_without_end_of_job():