Fix typo and mark some slow tests (aws#265)

* Fix typo and mark some slow tests

* Fix flaky tests

Author: jarednielsen

.gitignore

@@ -115,6 +115,7 @@ log
 cm.json
 cm.ts
 collections.json
+worker*.json
 
 # Wheel and tar files from running `python bin/build_containers.py`
 *.whl

tests/analysis/rules/test_confusion.py

@@ -1,9 +1,10 @@
 from tornasole.core.writer import FileWriter
 from tornasole.core.collection_manager import CollectionManager
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 
 from tornasole.rules.generic import Confusion
 from tornasole.trials import create_trial
+import pytest
 import os
 
 import uuid
@@ -22,9 +23,10 @@ def gen_y_and_y_hat( path, trial, step, y, y_name, y_hat, y_hat_name, colls={}):
     for coll in colls:
         c.add(coll)
         c.get(coll).tensor_names = colls[coll]
-    c.export(os.path.join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    c.export(os.path.join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
 
 
+@pytest.mark.slow # 0:06 to run
 def test_confusion():
     base_path = 'ts_output1/rule_invoker/'
     path = base_path

tests/analysis/rules/test_invoker.py

@@ -4,6 +4,7 @@
 from tornasole.trials import create_trial
 import uuid
 import numpy as np
+import pytest
 from tornasole.exceptions import *
 from tornasole.rules.rule_invoker import invoke_rule
 
@@ -62,6 +63,7 @@ def test_invoker_rule_pass_kwargs():
   assert rcode == 0
   shutil.rmtree(path)
 
+@pytest.mark.slow # 0:04 to run
 def test_invoker_rule_pass_other_trials():
   path1 = dump_data()
   path2 = dump_data()

tests/analysis/trials/test_modes.py

@@ -6,7 +6,7 @@
 from datetime import datetime
 from tornasole.core.writer import FileWriter
 from tornasole.core.collection_manager import CollectionManager
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 
 
 def test_modes_on_global_data():
@@ -20,7 +20,7 @@ def test_mode_data():
   c = CollectionManager()
   c.add("default")
   c.get("default").tensor_names = ["arr"]
-  c.export(os.path.join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+  c.export(os.path.join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
   tr = create_trial(trial_dir)
   for s in range(0, 10):
     fw = FileWriter(trial_dir=trial_dir, step=s)

tests/analysis/trials/test_s3.py

@@ -1,6 +1,6 @@
 from tornasole.trials import S3Trial
 from tornasole.core.collection_manager import CollectionManager
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 import uuid
 import os
 import pytest
@@ -33,8 +33,8 @@ def help_test_multiple_trials(num_steps = 20, num_tensors = 10):
     c = CollectionManager()
     c.add("default")
     c.get("default").tensor_names = ["foo_" + str(i) for i in range(num_tensors)]
-    c.export(path + trial_name + "/" + TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
-    c.export(path + trial_name + "/" + TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
+    c.export(path + trial_name + "/" + TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
+    c.export(path + trial_name + "/" + TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
     for i in range(num_steps):
         generate_data(path=path, trial=trial_name, num_tensors=num_tensors,
                       step=i, tname_prefix='foo', worker='algo-1', shape=(3, 3, 3), rank=0)

tests/analysis/trials/test_trial_modes.py

@@ -22,6 +22,7 @@ def test_local_index_mode():
     assert bool(trial_obj.index_tensors_dict) is True
 
 
+@pytest.mark.slow # 0:03 to run
 def test_local_event_mode():
     trial_name = str(uuid.uuid4())
     path = 'ts_output/train/'

tests/analysis/utils.py

@@ -1,7 +1,7 @@
 from tornasole.core.writer import FileWriter
 import numpy as np
 from tornasole.core.collection_manager import CollectionManager
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 import os
 import aioboto3
 import asyncio
@@ -25,7 +25,7 @@ def generate_data(path, trial, step, tname_prefix,
         c.get("default").tensor_names = [f'{tname_prefix}_{i}' for i in range(num_tensors)]
         c.add('gradients')
         c.get("gradients").tensor_names = [f'{tname_prefix}_{i}' for i in range(num_tensors)]
-        c.export(os.path.join(path, trial, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+        c.export(os.path.join(path, trial, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
 
 
 def check_trial(trial_obj, num_steps, num_tensors):

tests/core/test_collections.py

@@ -1,6 +1,6 @@
 from tornasole.core.collection import Collection
 from tornasole.core.collection_manager import CollectionManager
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 from tornasole.core.reduction_config import ReductionConfig
 from tornasole.core.save_config import SaveConfig, SaveConfigMode
 from tornasole.core.modes import ModeKeys
@@ -42,8 +42,8 @@ def test_manager_export_load():
   cm.add(Collection('trial1'))
   cm.add('trial2')
   cm.get('trial2').include('total_loss')
-  cm.export(TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
-  cm2 = CollectionManager.load(TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
+  cm.export(TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
+  cm2 = CollectionManager.load(TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME)
   assert cm == cm2
 
 def test_manager():

tests/mxnet/test_hook.py

@@ -41,6 +41,7 @@ def test_hook_from_json_config_full():
 
 def test_default_hook():
   reset_collections()
+  shutil.rmtree('/opt/ml/output/tensors', ignore_errors=True)
   if TORNASOLE_CONFIG_FILE_PATH_ENV_STR in os.environ:
     del os.environ[TORNASOLE_CONFIG_FILE_PATH_ENV_STR]
   hook = t_hook.hook_from_config()

tests/tensorflow/hooks/test_dist_horovod.py

@@ -1,6 +1,7 @@
 from tornasole.trials import create_trial
+import pytest
 
-
+@pytest.mark.slow # 0:11 to run
 def test_s3_read():
     path = "s3://tornasole-testing/dist-logs-10/"
     trial = create_trial(path)

tests/tensorflow/hooks/test_save_all_full.py

@@ -3,7 +3,7 @@
 import shutil, glob
 from tornasole.core.reader import FileReader
 from tornasole.core.json_config import TORNASOLE_CONFIG_FILE_PATH_ENV_STR
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 
 
 def test_save_all_full(hook=None, trial_dir=None):
@@ -29,8 +29,8 @@ def test_save_all_full(hook=None, trial_dir=None):
     assert len(coll['gradients'].tensor_names) == 1
     assert len(coll['losses'].tensor_names) == 1
 
-    assert TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
-    cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    assert TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
+    cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
 
     assert len(cm.collections) == 6
     assert len(cm.collections['weights'].tensor_names) == 1

tests/tensorflow/hooks/test_save_reductions.py

@@ -4,7 +4,7 @@
 import glob
 from tornasole.core.reader import FileReader
 from tornasole.core.json_config import TORNASOLE_CONFIG_FILE_PATH_ENV_STR
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 
 def helper_save_reductions(trial_dir, hook):
   simple_model(hook)
@@ -14,8 +14,8 @@ def helper_save_reductions(trial_dir, hook):
   assert len(coll['weights'].tensor_names) == 1
   assert len(coll['gradients'].tensor_names) == 1
 
-  assert TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
-  cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+  assert TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
+  cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
   assert len(cm.collections) == 5
   assert len(cm.collections['weights'].tensor_names) == 1
   assert len(cm.collections['gradients'].tensor_names) == 1

tests/tensorflow/hooks/test_simple_include.py

@@ -4,7 +4,7 @@
 import glob, shutil
 from tornasole.core.reader import FileReader
 from tornasole.core.json_config import TORNASOLE_CONFIG_FILE_PATH_ENV_STR
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 
 
 
@@ -14,7 +14,7 @@ def helper_test_simple_include(trial_dir, hook):
     _, files = get_dirs_files(trial_dir)
     steps, _ = get_dirs_files(os.path.join(trial_dir, 'events'))
 
-    cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
     assert len(cm.collections['default'].tensor_names) == 1
     assert len(steps) == 5
     for step in steps:
@@ -58,7 +58,7 @@ def helper_test_simple_include_regex(trial_dir, hook):
     _, files = get_dirs_files(trial_dir)
     steps, _ = get_dirs_files(os.path.join(trial_dir, 'events'))
 
-    cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
     assert len(cm.collections['default'].tensor_names) == 1
     assert len(steps) == 5
 
@@ -105,7 +105,7 @@ def helper_test_multi_collection_match(trial_dir, hook):
     _, files = get_dirs_files(trial_dir)
     steps, _ = get_dirs_files(os.path.join(trial_dir, 'events'))
 
-    cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
     assert len(cm.collections['default'].tensor_names) == 1
     assert len(cm.collections['trial'].tensor_names) == 1
     assert len(steps) == 5

tests/tensorflow/hooks/test_weights_gradients.py

@@ -1,7 +1,7 @@
 from .utils import *
 from tornasole.tensorflow import reset_collections
 import tensorflow.compat.v1 as tf
-from tornasole.core.config_constants import TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME
+from tornasole.core.config_constants import TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME
 from tornasole.core.json_config import TORNASOLE_CONFIG_FILE_PATH_ENV_STR
 import tornasole.tensorflow as ts
 import shutil
@@ -12,8 +12,8 @@ def helper_test_only_w_g(trial_dir, hook):
     steps, _ = get_dirs_files(os.path.join(trial_dir, 'events'))
     _, files = get_dirs_files(trial_dir)
 
-    assert TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
-    cm = CollectionManager.load(join(trial_dir, TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
+    assert TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME in files
+    cm = CollectionManager.load(join(trial_dir, TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME))
     num_tensors_loaded_collection = len(cm.collections['weights'].tensor_names) + \
                                     len(cm.collections['gradients'].tensor_names) + \
                                     len(cm.collections['default'].tensor_names)

tests/xgboost/test_hook.py

@@ -2,6 +2,7 @@
 import uuid
 import numpy as np
 import pytest
+import shutil
 import xgboost
 
 from .run_xgboost_model import run_xgboost_model
@@ -49,6 +50,7 @@ def test_hook_from_json_config_full(tmpdir, monkeypatch):
 
 def test_default_hook(monkeypatch):
     reset_collections()
+    shutil.rmtree('/opt/ml/output/tensors', ignore_errors=True)
     monkeypatch.delenv(TORNASOLE_CONFIG_FILE_PATH_ENV_STR, raising=False)
     hook = TornasoleHook.hook_from_config()
     assert hook.out_dir == DEFAULT_SAGEMAKER_TORNASOLE_PATH

tornasole/core/config_constants.py

@@ -9,4 +9,4 @@
 TORNASOLE_CONFIG_INCLUDE_REGEX_KEY = "include_regex"
 TORNASOLE_CONFIG_SAVE_ALL_KEY = "save_all"
 DEFAULT_SAGEMAKER_TORNASOLE_PATH = "/opt/ml/output/tensors"
-TORASOLE_DEFAULT_COLLECTIONS_FILE_NAME = 'worker_0_collections.json'
+TORNASOLE_DEFAULT_COLLECTIONS_FILE_NAME = 'worker_0_collections.json'

tornasole/trials/trial.py

@@ -112,7 +112,7 @@ def _wait():
             else:
                 collection_files = list_collection_files_in_directory(self.trial_dir)
 
-            time.sleep(2)
+            # time.sleep(2)
             num_times_before_warning -= 1
             if num_times_before_warning < 0:
                 self.logger.warning('Waiting to read collections')