Skip to content

Commit 5a8d8b6

Browse files
NihalHarishVikas-kum
NihalHarish
and
Vikas-kum
authored
CodeCov Support (aws#233)
* update tests to use pytest * Added cov-append Co-authored-by: Vikas-kum <[email protected]>
1 parent 36aab25 commit 5a8d8b6

8 files changed

+93
-63
lines changed

‎config/buildspec_build_wheel.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ phases:
1616
- sudo apt-get install unzip -qq -o=Dpkg::Use-Pty=0
1717
- cd $CODEBUILD_SRC_DIR && chmod +x config/protoc_downloader.sh && ./config/protoc_downloader.sh
1818
- pip install --upgrade pip==19.3.1
19-
- pip install -q pytest wheel pyYaml pytest-html pre-commit
19+
- pip install -q pytest wheel pyYaml pytest-html pre-commit pytest-cov
2020
- pip uninstall -y boto3 && pip uninstall -y aiobotocore && pip uninstall -y botocore
2121

2222
build:

‎config/buildspec_zero_code_change.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ phases:
2525
- sudo apt-get install unzip -qq -o=Dpkg::Use-Pty=0
2626
- cd $CODEBUILD_SRC_DIR && chmod +x config/protoc_downloader.sh && ./config/protoc_downloader.sh
2727
- pip install --upgrade pip==19.3.1
28-
- pip install -q pytest wheel pyYaml pytest-html pre-commit awscli tensorflow_datasets
28+
- pip install -q pytest wheel pyYaml pytest-html pre-commit awscli tensorflow_datasets pytest-cov
2929
- cd $CODEBUILD_SRC_DIR && chmod +x config/install_smdebug.sh && chmod +x config/check_smdebug_install.sh && ./config/install_smdebug.sh;
3030

3131
build:

‎config/tests.sh

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,26 @@ check_logs() {
1717
run_for_framework() {
1818
if [ "$zero_code_change_test" = "enable" ] ; then
1919
# ignoring some test becuase they require multiple frmaeworks to be installed, these tests need to be broken down
20-
python -m pytest --cov=./ --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html --ignore=tests/core/test_paths.py --ignore=tests/core/test_index_utils.py --ignore=tests/core/test_collections.py tests/$1
20+
python -m pytest --cov=./ --cov-append --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html --ignore=tests/core/test_paths.py --ignore=tests/core/test_index_utils.py --ignore=tests/core/test_collections.py tests/$1
2121
if [ "$1" = "mxnet" ] ; then
22-
python tests/zero_code_change/mxnet_gluon_integration_test.py
22+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_mxnet_gluon_integration.py
2323
elif [ "$1" = "pytorch" ] ; then
24-
python tests/zero_code_change/pytorch_integration_tests.py
24+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_pytorch_integration.py
25+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_pytorch_multiprocessing.py
26+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_training_with_no_grad_updates.py
2527
elif [ "$1" = "tensorflow" ] ; then
26-
python tests/zero_code_change/tensorflow_integration_tests.py
28+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_tensorflow_integration.py
2729
elif [ "$1" = "tensorflow2" ] ; then
28-
python tests/zero_code_change/tensorflow2_gradtape_integration_tests.py
29-
python tests/zero_code_change/tensorflow2_integration_tests.py
30+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_tensorflow2_gradtape_integration.py
31+
python -m pytest --cov=./ --cov-append tests/zero_code_change/test_tensorflow2_integration.py
3032
fi
3133

3234
else
3335
if [ "$1" = "tensorflow2" ] ; then
34-
python -m pytest --cov=./ --durations=50 --html=$REPORT_DIR/report_$1/eager_mode.html -v -s --self-contained-html tests/$1
35-
python -m pytest --cov=./ --durations=50 --non-eager --html=$REPORT_DIR/report_$1/non_eager_mode.html -v -s --self-contained-html tests/$1
36+
python -m pytest --cov=./ --cov-append --durations=50 --html=$REPORT_DIR/report_$1/eager_mode.html -v -s --self-contained-html tests/$1
37+
python -m pytest --cov=./ --cov-append --durations=50 --non-eager --html=$REPORT_DIR/report_$1/non_eager_mode.html -v -s --self-contained-html tests/$1
3638
else
37-
python -m pytest --cov=./ --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html tests/$1
39+
python -m pytest --cov=./ --cov-append --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html tests/$1
3840
fi
3941
fi
4042
}
@@ -46,7 +48,7 @@ export SMDEBUG_LOG_LEVEL=info
4648

4749
export OUT_DIR=upload/$CURRENT_COMMIT_PATH
4850
export REPORT_DIR=$OUT_DIR/pytest_reports
49-
python -m pytest --cov=./ -v -W=ignore --durations=50 --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
51+
python -m pytest --cov=./ --cov-append -v -W=ignore --durations=50 --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
5052

5153
run_for_framework core
5254

‎tests/zero_code_change/mxnet_gluon_integration_test.py renamed to ‎tests/zero_code_change/test_mxnet_gluon_integration.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def parse_args():
3030
return opt
3131

3232

33-
def test(ctx, net, val_data):
33+
def fn_test(ctx, net, val_data):
3434
metric = mx.metric.Accuracy()
3535
for i, (data, label) in enumerate(val_data):
3636
data = data.as_in_context(ctx)
@@ -75,7 +75,7 @@ def train_model(net, epochs, ctx, learning_rate, momentum, train_data, val_data)
7575

7676
name, acc = metric.get()
7777
print("[Epoch %d] Training: %s=%f" % (epoch, name, acc))
78-
name, val_acc = test(ctx, net, val_data)
78+
name, val_acc = fn_test(ctx, net, val_data)
7979
print("[Epoch %d] Validation: %s=%f" % (epoch, name, val_acc))
8080

8181

@@ -145,18 +145,21 @@ def validate():
145145
print("Validation Complete")
146146

147147

148-
def main():
149-
opt = parse_args()
148+
def test_integration_mxnet():
149+
# DEFAULT CONSTANTS
150+
batch_size = 256
151+
epochs = 1
152+
learning_rate = 0.1
150153
mx.random.seed(128)
151154
random.seed(12)
152155
np.random.seed(2)
153156

154-
context = mx.cpu() if opt.context.lower() == "cpu" else mx.gpu()
157+
context = mx.cpu()
155158
# Create a Gluon Model.
156159
net = create_gluon_model()
157160

158161
# Start the training.
159-
train_data, val_data = prepare_data(opt.batch_size)
162+
train_data, val_data = prepare_data(batch_size)
160163

161164
json_file_contents = """
162165
{
@@ -175,16 +178,16 @@ def main():
175178
with SagemakerSimulator(json_file_contents=json_file_contents) as sim:
176179
train_model(
177180
net=net,
178-
epochs=opt.epochs,
181+
epochs=epochs,
179182
ctx=context,
180-
learning_rate=opt.learning_rate,
183+
learning_rate=learning_rate,
181184
momentum=0.9,
182185
train_data=train_data,
183186
val_data=val_data,
184187
)
185-
if opt.validate:
186-
validate()
188+
189+
validate()
187190

188191

189192
if __name__ == "__main__":
190-
main()
193+
test_integration_mxnet()

‎tests/zero_code_change/pytorch_integration_tests.py renamed to ‎tests/zero_code_change/test_pytorch_integration.py

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,20 @@
1111
import argparse
1212

1313
# Third Party
14+
import pytest
1415
import torch.nn as nn
1516
import torch.nn.functional as F
1617
import torch.optim as optim
17-
from pt_utils import Net, get_dataloaders
18+
from tests.zero_code_change.pt_utils import Net, get_dataloaders
1819

1920
# First Party
2021
import smdebug.pytorch as smd
2122
from smdebug.core.utils import SagemakerSimulator, ScriptSimulator
2223

2324

24-
def test_pytorch(script_mode: bool = False, use_loss_module=False):
25+
@pytest.mark.parametrize("script_mode", [False])
26+
@pytest.mark.parametrize("use_loss_module", [True, False])
27+
def test_pytorch(script_mode, use_loss_module):
2528
smd.del_hook()
2629

2730
sim_class = ScriptSimulator if script_mode else SagemakerSimulator
@@ -82,17 +85,13 @@ def test_pytorch(script_mode: bool = False, use_loss_module=False):
8285
)
8386

8487

85-
def test_pytorch_loss_module(script_mode: bool = False):
86-
test_pytorch(script_mode=script_mode, use_loss_module=True)
87-
88-
8988
if __name__ == "__main__":
9089
parser = argparse.ArgumentParser()
9190
parser.add_argument(
9291
"--script-mode", help="Manually create hooks instead of relying on ZCC", action="store_true"
9392
)
9493
args = parser.parse_args()
95-
script_mode = args.script_mode
94+
use_script_mode = args.script_mode
9695

97-
test_pytorch(script_mode=script_mode, use_loss_module=True)
98-
test_pytorch(script_mode=script_mode, use_loss_module=False)
96+
test_pytorch(script_mode=use_script_mode, use_loss_module=True)
97+
test_pytorch(script_mode=use_script_mode, use_loss_module=False)

‎tests/zero_code_change/tensorflow2_gradtape_integration_tests.py renamed to ‎tests/zero_code_change/test_tensorflow2_gradtape_integration.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import argparse
1111

1212
# Third Party
13+
import pytest
1314
import tensorflow.compat.v2 as tf
1415

1516
# First Party
@@ -99,12 +100,14 @@ def helper_test_keras_v2_gradienttape(script_mode: bool = False, json_file_conte
99100
assert not hook
100101

101102

102-
def test_keras_v2_default(script_mode: bool = False):
103+
@pytest.mark.parametrize("script_mode", [False])
104+
def test_keras_v2_default(script_mode):
103105
# Test default ZCC behavior
104106
helper_test_keras_v2_gradienttape(script_mode=script_mode)
105107

106108

107-
def test_keras_v2_multi_collections(script_mode: bool = False):
109+
@pytest.mark.parametrize("script_mode", [False])
110+
def test_keras_v2_multi_collections(script_mode):
108111
# Test multiple collections included in hook json
109112
json_file_contents = """
110113
{
@@ -138,7 +141,8 @@ def test_keras_v2_multi_collections(script_mode: bool = False):
138141
)
139142

140143

141-
def test_keras_v2_save_all(script_mode: bool = False):
144+
@pytest.mark.parametrize("script_mode", [False])
145+
def test_keras_v2_save_all(script_mode):
142146
# Test save all through hook config
143147
json_file_contents = """
144148
{
@@ -158,7 +162,10 @@ def test_keras_v2_save_all(script_mode: bool = False):
158162
if __name__ == "__main__":
159163
parser = argparse.ArgumentParser()
160164
parser.add_argument(
161-
"--script-mode", help="Manually create hooks instead of relying on ZCC", action="store_true"
165+
"--script-mode",
166+
help="Manually create hooks instead of relying on ZCC",
167+
action="store_true",
168+
default=False,
162169
)
163170
args = parser.parse_args()
164171
script_mode = args.script_mode

‎tests/zero_code_change/tensorflow2_integration_tests.py renamed to ‎tests/zero_code_change/test_tensorflow2_integration.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import argparse
1919

2020
# Third Party
21+
import pytest
2122
import tensorflow.compat.v2 as tf
2223

2324
# First Party
@@ -137,12 +138,16 @@ def helper_test_keras_v2_json_config(
137138
assert len(trial.tensor_names(collection="losses")) > 0
138139

139140

140-
def test_keras_v2_default(script_mode: bool = False, eager_mode: bool = True):
141+
@pytest.mark.parametrize("script_mode", [False])
142+
@pytest.mark.parametrize("eager_mode", [True, False])
143+
def test_keras_v2_default(script_mode, eager_mode):
141144
# Test default ZCC behavior
142145
helper_test_keras_v2(script_mode=script_mode, eager_mode=eager_mode)
143146

144147

145-
def test_keras_v2_multi_collections(script_mode: bool = False, eager_mode: bool = True):
148+
@pytest.mark.parametrize("script_mode", [False])
149+
@pytest.mark.parametrize("eager_mode", [True, False])
150+
def test_keras_v2_multi_collections(script_mode, eager_mode):
146151
# Test multiple collections included in hook json
147152
json_file_contents = """
148153
{
@@ -176,7 +181,9 @@ def test_keras_v2_multi_collections(script_mode: bool = False, eager_mode: bool
176181
)
177182

178183

179-
def test_keras_v2_save_all(script_mode: bool = False, eager_mode: bool = True):
184+
@pytest.mark.parametrize("script_mode", [False])
185+
@pytest.mark.parametrize("eager_mode", [True, False])
186+
def test_keras_v2_save_all(script_mode, eager_mode):
180187
# Test save all through hook config
181188
json_file_contents = """
182189
{
@@ -196,7 +203,10 @@ def test_keras_v2_save_all(script_mode: bool = False, eager_mode: bool = True):
196203
if __name__ == "__main__":
197204
parser = argparse.ArgumentParser()
198205
parser.add_argument(
199-
"--script-mode", help="Manually create hooks instead of relying on ZCC", action="store_true"
206+
"--script-mode",
207+
help="Manually create hooks instead of relying on ZCC",
208+
action="store_true",
209+
default=False,
200210
)
201211
args = parser.parse_args()
202212
script_mode = args.script_mode

0 commit comments

Comments
 (0)