changes in test.sh (aws#89)

* changes

* fixing import error in local trial created by some pr

* fixing import error in local trial created by some pr

* changes

* changes

* changes

* changes

* changes

* changes

Author: kaustubhsardar

config/buildspec.yml

@@ -1,6 +1,11 @@
 # Build Spec for AWS CodeBuild CI
 
 version: 0.2
+env:
+  variables:
+    run_pytest_pytorch: "enable"
+    run_pytest_mxnet: "enable"
+    run_pytest_tensorflow: "enable"
 phases:
   install:
     commands:
@@ -12,6 +17,7 @@ phases:
         - cd $CODEBUILD_SRC_DIR  && chmod +x config/protoc_downloader.sh && ./config/protoc_downloader.sh
         - pip install pytest wheel pyYaml pytest-html tensorflow mxnet torch
         - pip uninstall -y boto3 && pip uninstall -y aiobotocore && pip uninstall -y botocore
+
 
   pre_build:
     commands:

config/tests.sh

@@ -1,8 +1,39 @@
 #!/usr/bin/env bash
+check_logs() {
+  if grep "AssertionError" $1;
+   then
+    echo "Integration tests:FAILED."
+    exit 1
+  else
+        echo "Integration tests: SUCCESS."
+  fi
+}
 
-#export TORNASOLE_LOG_LEVEL=debug
-TORNASOLE_LOG_LEVEL=debug python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report.html --self-contained-html tests/
-TORNASOLE_LOG_LEVEL=debug python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode tensorflow --path_to_config ./tests/analysis/config.yaml
-TORNASOLE_LOG_LEVEL=debug python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode mxnet --path_to_config ./tests/analysis/config.yaml
-TORNASOLE_LOG_LEVEL=debug python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode pytorch --path_to_config ./tests/analysis/config.yaml
+set -ex
+export TORNASOLE_LOG_LEVEL=debug
+python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html --self-contained-html tests/analysis
+python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_core.html --self-contained-html tests/core
 
+if [ "$run_pytest_tensorflow" = "enable" ] ; then
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html --self-contained-html tests/tensorflow
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode tensorflow --path_to_config ./tests/analysis/config.yaml
+fi
+
+if [ "$run_pytest_mxnet" = "enable" ] ; then
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html --self-contained-html tests/mxnet
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode mxnet --path_to_config ./tests/analysis/config.yaml
+fi
+
+if [ "$run_pytest_pytorch" = "enable" ] ; then
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html --self-contained-html tests/pytorch
+    python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode pytorch --path_to_config ./tests/analysis/config.yaml
+fi
+
+check_logs upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/report_core.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html
+check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html