Skip to content

infra: use fixture for Python version in MXNet integ tests #1613

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 24, 2020
Merged

infra: use fixture for Python version in MXNet integ tests #1613

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cb2c160
infra: use fixture for Python version in MXNet integ tests
laurenyu Jun 10, 2020
458a375
fix formatting
laurenyu Jun 23, 2020
542f9d1
Merge branch 'zwei' into clean-up-tests-bc-no-more-python-2-mx
laurenyu Jun 23, 2020
110bb60
fix param names
laurenyu Jun 23, 2020
cbba80d
black format
laurenyu Jun 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ def pytest_addoption(parser):
parser.addoption("--sagemaker-runtime-config", action="store", default=None)
parser.addoption("--boto-config", action="store", default=None)
parser.addoption("--chainer-full-version", action="store", default="5.0.0")
parser.addoption("--mxnet-full-version", action="store", default="1.6.0")
parser.addoption("--ei-mxnet-full-version", action="store", default="1.5.1")
parser.addoption(
"--rl-coach-mxnet-full-version",
Expand Down Expand Up @@ -255,8 +254,13 @@ def chainer_full_version(request):


@pytest.fixture(scope="module")
def mxnet_full_version(request):
return request.config.getoption("--mxnet-full-version")
def mxnet_full_version():
return "1.6.0"


@pytest.fixture(scope="module")
def mxnet_full_py_version():
return "py3"


@pytest.fixture(scope="module")
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_airflow_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ def test_chainer_airflow_config_uploads_data_source_to_s3(

@pytest.mark.canary_quick
def test_mxnet_airflow_config_uploads_data_source_to_s3(
sagemaker_session, cpu_instance_type, mxnet_full_version
sagemaker_session, cpu_instance_type, mxnet_full_version, mxnet_full_py_version
):
with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
Expand All @@ -452,7 +452,7 @@ def test_mxnet_airflow_config_uploads_data_source_to_s3(
entry_point=script_path,
role=ROLE,
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=SINGLE_INSTANCE_COUNT,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -573,7 +573,7 @@ def test_xgboost_airflow_config_uploads_data_source_to_s3(

@pytest.mark.canary_quick
def test_pytorch_airflow_config_uploads_data_source_to_s3_when_inputs_not_provided(
sagemaker_session, cpu_instance_type, pytorch_full_version, pytorch_full_py_version,
sagemaker_session, cpu_instance_type, pytorch_full_version, pytorch_full_py_version
):
with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
estimator = PyTorch(
Expand Down
48 changes: 26 additions & 22 deletions tests/integ/test_debugger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,9 @@

import pytest

from sagemaker.debugger import Rule
from sagemaker.debugger import DebuggerHookConfig
from sagemaker.debugger import TensorBoardOutputConfig

from sagemaker.debugger import rule_configs
from sagemaker.debugger import DebuggerHookConfig, Rule, rule_configs, TensorBoardOutputConfig
from sagemaker.mxnet.estimator import MXNet
from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.retry import retries
from tests.integ.timeout import timeout

Expand Down Expand Up @@ -60,7 +56,9 @@
# TODO-reinvent-2019: test get_debugger_artifacts_path and get_tensorboard_artifacts_path


def test_mxnet_with_rules(sagemaker_session, mxnet_full_version, cpu_instance_type):
def test_mxnet_with_rules(
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
rules = [
Rule.sagemaker(rule_configs.vanishing_gradient()),
Expand All @@ -77,7 +75,7 @@ def test_mxnet_with_rules(sagemaker_session, mxnet_full_version, cpu_instance_ty
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -119,7 +117,9 @@ def test_mxnet_with_rules(sagemaker_session, mxnet_full_version, cpu_instance_ty
_wait_and_assert_that_no_rule_jobs_errored(training_job=mx.latest_training_job)


def test_mxnet_with_custom_rule(sagemaker_session, mxnet_full_version, cpu_instance_type):
def test_mxnet_with_custom_rule(
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
rules = [_get_custom_rule(sagemaker_session)]

Expand All @@ -130,7 +130,7 @@ def test_mxnet_with_custom_rule(sagemaker_session, mxnet_full_version, cpu_insta
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -166,7 +166,9 @@ def test_mxnet_with_custom_rule(sagemaker_session, mxnet_full_version, cpu_insta
_wait_and_assert_that_no_rule_jobs_errored(training_job=mx.latest_training_job)


def test_mxnet_with_debugger_hook_config(sagemaker_session, mxnet_full_version, cpu_instance_type):
def test_mxnet_with_debugger_hook_config(
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
debugger_hook_config = DebuggerHookConfig(
s3_output_path=os.path.join(
Expand All @@ -181,7 +183,7 @@ def test_mxnet_with_debugger_hook_config(sagemaker_session, mxnet_full_version,
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand All @@ -204,7 +206,7 @@ def test_mxnet_with_debugger_hook_config(sagemaker_session, mxnet_full_version,


def test_mxnet_with_rules_and_debugger_hook_config(
sagemaker_session, mxnet_full_version, cpu_instance_type
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
rules = [
Expand All @@ -227,7 +229,7 @@ def test_mxnet_with_rules_and_debugger_hook_config(
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -272,7 +274,7 @@ def test_mxnet_with_rules_and_debugger_hook_config(


def test_mxnet_with_custom_rule_and_debugger_hook_config(
sagemaker_session, mxnet_full_version, cpu_instance_type
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
rules = [_get_custom_rule(sagemaker_session)]
Expand All @@ -289,7 +291,7 @@ def test_mxnet_with_custom_rule_and_debugger_hook_config(
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -328,7 +330,7 @@ def test_mxnet_with_custom_rule_and_debugger_hook_config(


def test_mxnet_with_tensorboard_output_config(
sagemaker_session, mxnet_full_version, cpu_instance_type
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
tensorboard_output_config = TensorBoardOutputConfig(
Expand All @@ -344,7 +346,7 @@ def test_mxnet_with_tensorboard_output_config(
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand All @@ -370,7 +372,9 @@ def test_mxnet_with_tensorboard_output_config(


@pytest.mark.canary_quick
def test_mxnet_with_all_rules_and_configs(sagemaker_session, mxnet_full_version, cpu_instance_type):
def test_mxnet_with_all_rules_and_configs(
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
rules = [
Rule.sagemaker(rule_configs.vanishing_gradient()),
Expand Down Expand Up @@ -398,7 +402,7 @@ def test_mxnet_with_all_rules_and_configs(sagemaker_session, mxnet_full_version,
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down Expand Up @@ -441,7 +445,7 @@ def test_mxnet_with_all_rules_and_configs(sagemaker_session, mxnet_full_version,


def test_mxnet_with_debugger_hook_config_disabled(
sagemaker_session, mxnet_full_version, cpu_instance_type
sagemaker_session, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_gluon.py")
Expand All @@ -451,7 +455,7 @@ def test_mxnet_with_debugger_hook_config_disabled(
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
Expand Down
10 changes: 5 additions & 5 deletions tests/integ/test_git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from sagemaker.pytorch.estimator import PyTorch
from sagemaker.sklearn.estimator import SKLearn
from sagemaker.sklearn.model import SKLearnModel
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ import DATA_DIR


GIT_REPO = "https://github.com/aws/sagemaker-python-sdk.git"
Expand Down Expand Up @@ -81,7 +81,7 @@ def test_github(sagemaker_local_session, pytorch_full_version, pytorch_full_py_v

@pytest.mark.local_mode
@pytest.mark.skip("needs a secure authentication approach")
def test_private_github(sagemaker_local_session, mxnet_full_version):
def test_private_github(sagemaker_local_session, mxnet_full_version, mxnet_full_py_version):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
Expand All @@ -100,7 +100,7 @@ def test_private_github(sagemaker_local_session, mxnet_full_version):
source_dir=source_dir,
dependencies=dependencies,
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
Expand Down Expand Up @@ -219,7 +219,7 @@ def test_github_with_ssh_passphrase_not_configured(sagemaker_local_session, skle

@pytest.mark.local_mode
@pytest.mark.skip("needs a secure authentication approach")
def test_codecommit(sagemaker_local_session, mxnet_full_version):
def test_codecommit(sagemaker_local_session, mxnet_full_version, mxnet_full_py_version):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
Expand All @@ -236,7 +236,7 @@ def test_codecommit(sagemaker_local_session, mxnet_full_version):
source_dir=source_dir,
dependencies=dependencies,
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
Expand Down
34 changes: 20 additions & 14 deletions tests/integ/test_local_mode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import stopit

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ import DATA_DIR

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
Expand Down Expand Up @@ -54,7 +54,7 @@ def _initialize(self, boto_session, sagemaker_client, sagemaker_runtime_client):


@pytest.fixture(scope="module")
def mxnet_model(sagemaker_local_session, mxnet_full_version):
def mxnet_model(sagemaker_local_session, mxnet_full_version, mxnet_full_py_version):
def _create_model(output_path):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
Expand All @@ -66,7 +66,7 @@ def _create_model(output_path):
train_instance_type="local",
output_path=output_path,
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
sagemaker_session=sagemaker_local_session,
)

Expand All @@ -85,7 +85,9 @@ def _create_model(output_path):


@pytest.mark.local_mode
def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model, mxnet_full_version):
def test_local_mode_serving_from_s3_model(
sagemaker_local_session, mxnet_model, mxnet_full_version, mxnet_full_py_version
):
path = "s3://%s" % sagemaker_local_session.default_bucket()
s3_model = mxnet_model(path)
s3_model.sagemaker_session = sagemaker_local_session
Expand Down Expand Up @@ -119,14 +121,14 @@ def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mx


@pytest.mark.local_mode
def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):
def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version, mxnet_full_py_version):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")

mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
Expand All @@ -153,14 +155,16 @@ def test_mxnet_local_mode(sagemaker_local_session, mxnet_full_version):


@pytest.mark.local_mode
def test_mxnet_distributed_local_mode(sagemaker_local_session, mxnet_full_version):
def test_mxnet_distributed_local_mode(
sagemaker_local_session, mxnet_full_version, mxnet_full_py_version
):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")

mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=2,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
Expand All @@ -179,7 +183,7 @@ def test_mxnet_distributed_local_mode(sagemaker_local_session, mxnet_full_versio


@pytest.mark.local_mode
def test_mxnet_local_data_local_script(mxnet_full_version):
def test_mxnet_local_data_local_script(mxnet_full_version, mxnet_full_py_version):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")

Expand All @@ -189,7 +193,7 @@ def test_mxnet_local_data_local_script(mxnet_full_version):
train_instance_count=1,
train_instance_type="local",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
sagemaker_session=LocalNoS3Session(),
)

Expand All @@ -209,14 +213,16 @@ def test_mxnet_local_data_local_script(mxnet_full_version):


@pytest.mark.local_mode
def test_mxnet_training_failure(sagemaker_local_session, mxnet_full_version, tmpdir):
def test_mxnet_training_failure(
sagemaker_local_session, mxnet_full_version, mxnet_full_py_version, tmpdir
):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "failure_script.py")

mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
Expand All @@ -233,7 +239,7 @@ def test_mxnet_training_failure(sagemaker_local_session, mxnet_full_version, tmp

@pytest.mark.local_mode
def test_local_transform_mxnet(
sagemaker_local_session, tmpdir, mxnet_full_version, cpu_instance_type
sagemaker_local_session, tmpdir, mxnet_full_version, mxnet_full_py_version, cpu_instance_type
):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
Expand All @@ -244,7 +250,7 @@ def test_local_transform_mxnet(
train_instance_count=1,
train_instance_type="local",
framework_version=mxnet_full_version,
py_version=PYTHON_VERSION,
py_version=mxnet_full_py_version,
sagemaker_session=sagemaker_local_session,
)

Expand Down
Loading