Tornasole hook from config json (aws#104)

* creating tornasole hook from config

* making a quick variance fix (aws#99)

* Adding the change to convert ndarray to np.ndarray when operator is not available in mxnet.

* Cleanup and tests for TF and mxnet

* remove rmtree from s3 test

* Fixed the function invocation of get_numpy_reduction

* Changes to read from hardcoded path

* fixing pytorch test

* Setting SaveConfig per mode (aws#94)

* add doc for passing saveconfig specific to modes

* add save config for collection

* Create an option to build tornasole with no framework, TORNASOLE_FOR_RULES=1 (aws#95)

* add option to build only for rules

* Adding support to set save config per mode through json, also copying load collection method to all frameworks as that was missed

* remove set -ex from tests script since it prevents upload of reports

* move json config out of hooks

Author: Vikas-kum

bin/build_binaries.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 export VERSION='0.3'
 
 for FRAMEWORK in tensorflow mxnet pytorch
@@ -9,3 +10,10 @@ do
 #    aws s3 cp dist/tornasole-$VERSION-py2.py3-none-any.whl s3://tornasole-binaries-use1/tornasole_$FRAMEWORK/py3/
     rm -rf dist build *.egg-info
 done
+
+
+export TORNASOLE_FOR_RULES=1
+python setup.py bdist_wheel --universal
+unset TORNASOLE_FOR_RULES
+# aws s3 cp dist/tornasole-$VERSION-py2.py3-none-any.whl s3://tornasole-binaries-use1/tornasole_rules/py3/
+rm -rf dist build *.egg-info

config/buildspec.yml

@@ -26,10 +26,10 @@ phases:
     commands:
       - cd $CODEBUILD_SRC_DIR  && python setup.py bdist_wheel --universal && pip install dist/*.whl && cd ..
       - cd $CODEBUILD_SRC_DIR  && chmod +x config/tests.sh && ./config/tests.sh  && mkdir -p upload/$CURRENT_COMMIT_PATH/wheels && cp ./dist/*.whl upload/$CURRENT_COMMIT_PATH/wheels && cd ..
-      - aws s3 cp $CODEBUILD_SRC_DIR/upload s3://tornasolecodebuildtest/ --recursive
 #if [ "$CODEBUILD_GIT_BRANCH" = "master" ] && [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ] ; then
   post_build:
     commands:
-      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then echo "ERROR BUILD FAILED , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK:$CODEBUILD_BUILD_URL" ; fi
-      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 1 ]; then echo "INFO BUILD SUCCEEDED !!! , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK:$CODEBUILD_BUILD_URL" ; fi
-
+      - cat $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/reports/*.html >> $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/reports/all_tests.html
+      - aws s3 cp $CODEBUILD_SRC_DIR/upload s3://tornasolecodebuildtest/ --recursive
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then echo "ERROR BUILD FAILED , Access build logs for PR:$GITHUB_PR_URL at the below codebuild url. Test logs are on S3 here:$S3_TEST_REPORT_URL" ; fi
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 1 ]; then echo "INFO BUILD SUCCEEDED , Access build logs for PR:$GITHUB_PR_URL at the below codebuild url. Test logs are on S3 here:$S3_TEST_REPORT_URL" ; fi

config/get-branch.sh

@@ -6,9 +6,10 @@ fi
 
 cd $CODEBUILD_SRC_DIR && git checkout $CODEBUILD_GIT_BRANCH
 export CURRENT_COMMIT_HASH=$(git log -1 --pretty=%h);
-export CURRENT_COMMIT_DATE="$(git show -s --format=%ci | cut -d' ' -f 1)_$(git show -s --format=%ci | cut -d' ' -f 2)";
+#export CURRENT_COMMIT_DATE="$(git show -s --format=%ci | cut -d' ' -f 1)$(git show -s --format=%ci | cut -d' ' -f 2)";
+export CURRENT_DATETIME=$(date +'%Y%m%d_%H%M%S')
 export CURRENT_REPO_NAME=$(basename `git rev-parse --show-toplevel`) ;
-export CURRENT_COMMIT_PATH="$CURRENT_COMMIT_DATE/$CURRENT_REPO_NAME/$CURRENT_COMMIT_HASH"
+export CURRENT_COMMIT_PATH="$CURRENT_DATETIME/$CURRENT_COMMIT_HASH"
 cd ..
 
 
@@ -20,9 +21,14 @@ export CODEBUILD_BUILD_URL=https://$AWS_DEFAULT_REGION.console.aws.amazon.com/co
 echo "INFO =============================BUILD STARTED==================================="
 echo "INFO =============================Build details========================== ::"
 echo "INFO CODEBUILD_CURRENT_BUILD_URL = $CODEBUILD_BUILD_URL"
-echo "INFO CURRENT_REPO_NAME = $CURRENT_REPO_NAME"
+#echo "INFO CURRENT_REPO_NAME = $CURRENT_REPO_NAME"
 echo "INFO CURRENT_COMMIT_DATE = $CURRENT_COMMIT_DATE"
 echo "INFO CODEBUILD_ACCOUNT_ID = $CODEBUILD_ACCOUNT_ID"
 echo "INFO CURRENT_GIT_BRANCH = $CODEBUILD_GIT_BRANCH"
-#echo "INFO CURRENT_GIT_COMMIT = $CODEBUILD_GIT_COMMIT"
+echo "INFO CURRENT_GIT_COMMIT = $CODEBUILD_GIT_COMMIT"
 echo "INFO CODEBUILD_PROJECT = $CODEBUILD_PROJECT"
+
+PR_ID=$(echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d ';' -f 2-)
+export GITHUB_PR_URL=https://github.com/awslabs/$CURRENT_REPO_NAME/pull/$PR_ID
+#https://s3.console.aws.amazon.com/s3/object/tornasolecodebuildtest/20190817_215022/c24a121/reports/all_tests.html?region=us-east-1&tab=overview
+export S3_TEST_REPORT_URL=https://s3.console.aws.amazon.com/s3/object/tornasolecodebuildtest/$CURRENT_COMMIT_PATH/reports/all_tests.html?region=us-east-1&tab=overview

config/tests.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+
+set -ex
+
 check_logs() {
   if grep "AssertionError" $1;
    then
@@ -9,31 +12,30 @@ check_logs() {
   fi
 }
 
-set -ex
 export TORNASOLE_LOG_LEVEL=debug
 python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html --self-contained-html tests/analysis
 python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_core.html --self-contained-html tests/core
 
 if [ "$run_pytest_tensorflow" = "enable" ] ; then
     python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html --self-contained-html tests/tensorflow
-    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode tensorflow --path_to_config ./tests/analysis/config.yaml
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode tensorflow --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.log
 fi
 
 if [ "$run_pytest_mxnet" = "enable" ] ; then
     python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html --self-contained-html tests/mxnet
-    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode mxnet --path_to_config ./tests/analysis/config.yaml
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode mxnet --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.log
 fi
 
 if [ "$run_pytest_pytorch" = "enable" ] ; then
     python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html --self-contained-html tests/pytorch
-    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode pytorch --path_to_config ./tests/analysis/config.yaml
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode pytorch --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log
 fi
 
 check_logs upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html
 check_logs upload/$CURRENT_COMMIT_PATH/reports/report_core.html
 check_logs upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html
 check_logs upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html
 check_logs upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html
-check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html
-check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html
-check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.log
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log

docs/mxnet/README.md

@@ -175,8 +175,28 @@ and make it easy for analysis. To leverage this functionality you have to
 call the `set_mode` function of hook such as the following call `hook.set_mode(modes.TRAIN)`.
 The different modes available are `modes.TRAIN`, `modes.EVAL` and `modes.PREDICT`. 
 
+
 If the mode was not set, all steps will be available together.
 
+You can choose to have different save configurations (SaveConfigs) 
+for different modes. You can configure this by passing a 
+dictionary from mode to SaveConfig object. 
+The hook's `save_config` parameter accepts such a dictionary, as well as collection's `set_save_config` method.
+```
+from tornasole.tensorflow import TornasoleHook, get_collection, modes, SaveConfig
+scm = {modes.TRAIN: SaveConfig(save_interval=100), 
+        modes.EVAL: SaveConfig(save_interval=10)}
+        
+hook = TornasoleHook(..., 
+                     save_config=scm,
+                     ...)
+```
+
+```
+from tornasole.tensorflow import get_collection, modes, SaveConfig
+get_collection('weights').set_save_config({modes.TRAIN: SaveConfig(save_interval=10), 
+                                           modes.EVAL: SaveConfig(save_interval=1000)}
+```
 #### Collection
 Collection object helps group tensors for easier handling of tensors being saved. 
 A collection has its own list of tensors, include regex patterns, [reduction config](#reductionconfig) and [save config](#saveconfig). 

docs/mxnet/api.md

@@ -28,12 +28,16 @@ TornasoleHook is the entry point for Tornasole into your program.
         name of worker in a multi process training job
         outputs and tensors are organized by this name during retrieval.
         
-    save_config: SaveConfig object
+    save_config: SaveConfig object or a dictionary from mode to SaveConfig objects
         SaveConfig allows you to customize when tensors are saved. 
         Hook takes SaveConfig object which is applied as 
         default for all included tensors.
         A collection can optionally have its own SaveConfig object 
         which overrides this for its tensors.
+        If you pass a dictionary from mode->SaveConfig, then that
+        SaveConfig is applied to tensors included for that mode.
+        example: {modes.TRAIN: SaveConfig(save_interval=10), 
+                  modes.EVAL:SaveConfig(save_interval=1)}
         Refer to documentation for SaveConfig.
     
     reduction_config: ReductionConfig object
@@ -121,7 +125,7 @@ The following methods can be called on a collection object.
 | ```coll.add_block_tensors(block, input=False, output=False)```  | Takes an instance Gluon block, input and output flags. Users can use this Collection to log input/output tensors for a specific block  |
 | ```coll.get_include_regex()```  | Returns include_regex for the collection  |
 | ```coll.get_save_config()```  | Returns save config for the collection  |
-| ```coll.set_save_config(s)```  | Sets save config for the collection  |
+| ```coll.set_save_config(s)```  | Sets save config for the collection. You can either pass a SaveConfig instance or a dictionary from mode to SaveConfig |
 | ```coll.get_reduction_config()```  | Returns reduction config for the collection  |
 | ```coll.set_reduction_config()```  | Sets reduction config for the collection  |
 

docs/pytorch/README.md

@@ -196,6 +196,26 @@ The different modes available are `modes.TRAIN`, `modes.EVAL` and `modes.PREDICT
 
 If the mode was not set, all steps will be available together. 
 
+You can choose to have different save configurations (SaveConfigs) 
+for different modes. You can configure this by passing a 
+dictionary from mode to SaveConfig object. 
+The hook's `save_config` parameter accepts such a dictionary, as well as collection's `set_save_config` method.
+```
+from tornasole.tensorflow import TornasoleHook, get_collection, modes, SaveConfig
+scm = {modes.TRAIN: SaveConfig(save_interval=100), 
+        modes.EVAL: SaveConfig(save_interval=10)}
+        
+hook = TornasoleHook(..., 
+                     save_config=scm,
+                     ...)
+```
+
+```
+from tornasole.tensorflow import get_collection, modes, SaveConfig
+get_collection('weights').set_save_config({modes.TRAIN: SaveConfig(save_interval=10), 
+                                           modes.EVAL: SaveConfig(save_interval=1000)}
+```
+
 #### Collection
 Collection object helps group tensors for easier handling of tensors being saved. 
 A collection has its own list of tensors, include regex patterns, [reduction config](#reductionconfig) and [save config](#saveconfig). 

docs/pytorch/api.md

@@ -22,12 +22,16 @@ A class used to represent the hook which gets attached to the
         when dry_run is set to True, behavior is only described in the log file.
         The tensors are not actually saved. 
     
-    save_config: SaveConfig object
+    save_config: SaveConfig object or a dictionary from mode to SaveConfig objects
         SaveConfig allows you to customize when tensors are saved. 
-        Hook Ttkes SaveConfig object which is applied as 
+        Hook takes SaveConfig object which is applied as 
         default for all included tensors.
         A collection can optionally have its own SaveConfig object 
         which overrides this for its tensors.
+        If you pass a dictionary from mode->SaveConfig, then that
+        SaveConfig is applied to tensors included for that mode.
+        example: {modes.TRAIN: SaveConfig(save_interval=10), 
+                  modes.EVAL:SaveConfig(save_interval=1)}
         Refer to documentation for SaveConfig.
         
     include_regex: list of (str or tensor variables)
@@ -93,7 +97,7 @@ The following methods can be called on a collection object.
 | ```coll.add(t)```  | Takes an instance or list or set of tf.Operation/tf.Variable/tf.Tensor to add to the collection  |
 | ```coll.get_include_regex()```  | Returns include_regex for the collection  |
 | ```coll.get_save_config()```  | Returns save config for the collection  |
-| ```coll.set_save_config(s)```  | Sets save config for the collection  |
+| ```coll.set_save_config(s)```  | Sets save config for the collection. You can either pass a SaveConfig instance or a dictionary from mode to SaveConfig |
 | ```coll.get_reduction_config()```  | Returns reduction config for the collection  |
 | ```coll.set_reduction_config()```  | Sets reduction config for the collection  |
 | ```coll.add_module_tensors(module, input=False, output=False)```  | Takes an instance of a module, along with input and output flags. Users can use this Collection to log input/output tensors for a specific module  |

docs/tensorflow/README.md

@@ -283,6 +283,26 @@ The different modes available are `modes.TRAIN`, `modes.EVAL` and `modes.PREDICT
 
 If the mode was not set, all steps will be available together.
 
+You can choose to have different save configurations (SaveConfigs) 
+for different modes. You can configure this by passing a 
+dictionary from mode to SaveConfig object. 
+The hook's `save_config` parameter accepts such a dictionary, as well as collection's `set_save_config` method.
+```
+from tornasole.tensorflow import TornasoleHook, get_collection, modes, SaveConfig
+scm = {modes.TRAIN: SaveConfig(save_interval=100), 
+        modes.EVAL: SaveConfig(save_interval=10)}
+        
+hook = TornasoleHook(..., 
+                     save_config=scm,
+                     ...)
+```
+
+```
+from tornasole.tensorflow import get_collection, modes, SaveConfig
+get_collection('weights').set_save_config({modes.TRAIN: SaveConfig(save_interval=10), 
+                                           modes.EVAL: SaveConfig(save_interval=1000)}
+```
+
 #### Collection
 Collection object helps group tensors for easier handling of tensors being saved. 
 A collection has its own list of tensors, include/exclude regex patterns, reduction config and save config. 

docs/tensorflow/api.md

@@ -29,12 +29,16 @@ class TornasoleHook:
         name of worker in a multi process training job
         outputs and tensors are organized by this name during retrieval.
         
-    save_config: SaveConfig object
+    save_config: SaveConfig object or a dictionary from mode to SaveConfig objects
         SaveConfig allows you to customize when tensors are saved. 
         Hook takes SaveConfig object which is applied as 
         default for all included tensors.
         A collection can optionally have its own SaveConfig object 
         which overrides this for its tensors.
+        If you pass a dictionary from mode->SaveConfig, then that
+        SaveConfig is applied to tensors included for that mode.
+        example: {modes.TRAIN: SaveConfig(save_interval=10), 
+                  modes.EVAL:SaveConfig(save_interval=1)}
         Refer to documentation for SaveConfig.
     
     reduction_config: ReductionConfig object
@@ -109,7 +113,7 @@ The following methods can be called on a collection object.
 | ```coll.add(t)```  | Takes an instance or list or set of tf.Operation/tf.Variable/tf.Tensor to add to the collection  |
 | ```coll.get_include_regex()```  | Returns include_regex for the collection  |
 | ```coll.get_save_config()```  | Returns save config for the collection  |
-| ```coll.set_save_config(s)```  | Sets save config for the collection  |
+| ```coll.set_save_config(s)```  | Sets save config for the collection. You can either pass a SaveConfig instance or a dictionary from mode to SaveConfig |
 | ```coll.get_reduction_config()```  | Returns reduction config for the collection  |
 | ```coll.set_reduction_config()```  | Sets reduction config for the collection  |
 

setup.py

@@ -16,18 +16,29 @@ def get_framework_packages(f):
     return ['tornasole.' + f + '*', 'tests.' + f + '*']
 
 def get_frameworks_to_build():
+    only_rules = os.environ.get('TORNASOLE_FOR_RULES', False)
+    if only_rules in ['1', 'True', 'true']:
+        only_rules = True
+    else:
+        only_rules = False
+
     with_frameworks = {}
-    for f in FRAMEWORKS:
-        with_frameworks[f] = os.environ.get('TORNASOLE_WITH_' + f.upper(), False)
-        if with_frameworks[f] in ['1', 'True', 'true']:
-            with_frameworks[f] = True
-        else:
-            with_frameworks[f] = False
-    enabled_some_framework = any(with_frameworks.values())
-    if not enabled_some_framework:
-        print('Building for all frameworks in one package')
+    if not only_rules:
         for f in FRAMEWORKS:
-            with_frameworks[f] = True
+            with_frameworks[f] = os.environ.get('TORNASOLE_WITH_' + f.upper(), False)
+            if with_frameworks[f] in ['1', 'True', 'true']:
+                with_frameworks[f] = True
+            else:
+                with_frameworks[f] = False
+        enabled_some_framework = any(with_frameworks.values())
+
+        if not enabled_some_framework:
+            print('Building for all frameworks in one package')
+            for f in FRAMEWORKS:
+                with_frameworks[f] = True
+    else:
+        for f in FRAMEWORKS:
+            with_frameworks[f] = False
     return with_frameworks
 
 def get_packages_to_include(frameworks_to_build):

tests/analysis/config.yaml

@@ -104,7 +104,7 @@
   - mxnet
   - *Enable
   - [*mnist_gluon_vg_demo, # this training script will generate 1300 steps
-     --random_seed True  --num_steps 33 --tornasole_frequency 30,
+     --random_seed True --num_steps 33 --tornasole_frequency 30,
      *invoker,
      --rule_name vanishinggradient --flag True --start_step 1 --end_step 31
     ] # notice, step 0 always returns False

tests/core/test_training_end.py

@@ -1,32 +1,30 @@
-from tornasole.core.access_layer.utils import has_training_ended
-from tornasole.core.access_layer.utils import training_has_ended
+from tornasole.core.access_layer.utils import has_training_ended, \
+    training_has_ended, delete_s3_prefixes
 import shutil
-import boto3
 from tornasole.core.utils import is_s3
 from tornasole.core.access_layer.file import ensure_dir
-
-def del_s3(bucket,file_path):
-    s3_client = boto3.client('s3')
-    s3_client.delete_object(Bucket=bucket, Key=file_path)
+from tornasole.core.access_layer.s3 import TSAccessS3
 
 def test_local_training_end():
-    localdir = "./training_end_test_dir"
+    localdir = "/tmp/training_end_test_dir"
+    ensure_dir(localdir, is_file=False)
     training_has_ended(localdir)
     assert has_training_ended(localdir) == True
     shutil.rmtree(localdir)
 
 def test_negative_local_training_end():
-    localdir = "./training_end_test_dir_negative"
+    localdir = "/tmp/training_end_test_dir_negative"
     assert has_training_ended(localdir) == False
 
 def test_s3_training_end():
     s3dir = 's3://tornasolecodebuildtest/training_end_test_dir'
-    bucket = 'tornasolecodebuildtest'
+    _, bucket, key = is_s3(s3dir)
+    f = TSAccessS3(bucket_name=bucket, key_name=key)
+    f.close()
     training_has_ended(s3dir)
     assert has_training_ended(s3dir) == True
-    del_s3(bucket, s3dir)
+    delete_s3_prefixes(bucket, key)
 
 def test_negative_s3_training_end():
-    bucket = 'tornasolecodebuildtest'
     s3dir = 's3://tornasolecodebuildtest/training_end_test_dir_negative'
     assert has_training_ended(s3dir) == False