Merge branch 'master' into migrate-sklearn

Author: jesterhazy

.travis.yml

@@ -15,6 +15,7 @@ CHANGELOG
 * bug-fix: pass accelerator_type in ``deploy`` for REST API TFS ``Model``
 * doc-fix: move content from tf/README.rst into sphynx project
 * doc-fix: move content from sklearn/README.rst into sphynx project
+* doc-fix: Improve new developer experience in README
 
 1.18.3.post1
 ============

CHANGELOG.rst

@@ -97,9 +97,16 @@ Running tests
 
 SageMaker Python SDK has unit tests and integration tests.
 
+You can install the libraries needed to run the tests by running :code:`pip install --upgrade .[test]` or, for Zsh users: :code:`pip install --upgrade .\[test\]`
+
 **Unit tests**
 
-tox is a prerequisite for running unit tests so you need to make sure you have it installed. To run the unit tests:
+
+We run unit tests with tox, which is a program that lets you run unit tests for multiple Python versions, and also make sure the
+code fits our style guidelines. We run tox with Python 2.7 and 3.6, so to run unit tests
+with the same configuration we do, you'll need to have interpreters for Python 2.7 and Python 3.6 installed.
+
+To run the unit tests with tox, run:
 
 ::
 
@@ -109,23 +116,29 @@ tox is a prerequisite for running unit tests so you need to make sure you have i
 
 To run the integration tests, the following prerequisites must be met
 
-1. Access to an AWS account to run the tests on
-2. AWS account credentials available to boto3 clients used in the tests
-3. The AWS account has an IAM role named :code:`SageMakerRole`
-4. The libraries listed in the ``extras_require`` object in ``setup.py`` for ``test`` are installed.
-   You can do this by running the following command: :code:`pip install --upgrade .[test]`
+1. AWS account credentials are available in the environment for the boto3 client to use.
+2. The AWS account has an IAM role named :code:`SageMakerRole` with the AmazonSageMakerFullAccess policy attached.
+
+We recommend selectively running just those integration tests you'd like to run. You can filter by individual test function names with:
+
+::
+
+    pytest -k 'test_i_care_about'
+
 
-You can run integ tests by issuing the following command:
+You can also run all of the integration tests by running the following command, which runs them in sequence, which may take a while:
 
 ::
 
     pytest tests/integ
 
-You can also filter by individual test function names (usable with any of the previous commands):
+
+You can also run them in parallel:
 
 ::
 
-    pytest -k 'test_i_care_about'
+    pytest -n auto tests/integ
+
 
 Building Sphinx docs
 ~~~~~~~~~~~~~~~~~~~~

README.rst

@@ -19,6 +19,33 @@ phases:
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
         tox -e py27,py36 -- tests/unit
 
+      # run notebook test
+      - |
+        if has-matching-changes "src/*.py" "setup.py" "setup.cfg"; then
+          echo "running notebook test"
+          python setup.py sdist
+          aws s3 --region us-west-2 cp ./dist/sagemaker-*.tar.gz s3://sagemaker-python-sdk-pr/
+          aws s3 cp s3://sagemaker-mead-cli/mead-nb-test.tar.gz mead-nb-test.tar.gz
+          tar -xzf mead-nb-test.tar.gz
+          git clone --depth 1 https://github.com/awslabs/amazon-sagemaker-examples.git
+          JAVA_HOME=$(get-java-home)
+          echo "set JAVA_HOME=$JAVA_HOME"
+          SAGEMAKER_ROLE_ARN=$(get-sagemaker-role-arn)
+          echo "set SAGEMAKER_ROLE_ARN=$SAGEMAKER_ROLE_ARN"
+          ./runtime/bin/mead-run-nb-test \
+            --instance-type ml.c4.8xlarge \
+            --region us-west-2 \
+            --lifecycle-config-name install-python-sdk \
+            --notebook-instance-role-arn $SAGEMAKER_ROLE_ARN \
+            ./amazon-sagemaker-examples/sagemaker-python-sdk/tensorflow_distributed_mnist/tensorflow_batch_transform_mnist.ipynb
+         else
+           echo "skipping notebook test"
+         fi
+
       # run integration tests
-      - if-matching-changes "test/" "tests/" "src/*.py" "docker/*/Dockerfile" &&
-        IGNORE_COVERAGE=- tox -e py27,py36 -- -n 24 --boxed tests/integ
+      - |
+        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg"; then
+          IGNORE_COVERAGE=- tox -e py27,py36 -- -n 24 --boxed tests/integ
+        else
+          echo "skipping integration tests"
+        fi

buildspec.yml

@@ -40,7 +40,7 @@ def fm_serializer(data):
     return json.dumps(js)
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_byo_estimator(sagemaker_session, region):
     """Use Factorization Machines algorithm as an example here.
 

tests/integ/test_byo_estimator.py

@@ -66,7 +66,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, chainer_ful
         return chainer.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_attach_deploy(chainer_training_job, sagemaker_session):
     endpoint_name = 'test-chainer-attach-deploy-{}'.format(sagemaker_timestamp())

tests/integ/test_chainer_train.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 from sagemaker import FactorizationMachines, FactorizationMachinesModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_factorization_machines(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_factorization_machines.py

@@ -27,6 +27,7 @@
 horovod_dir = os.path.join(os.path.dirname(__file__), '..', 'data', 'horovod')
 
 
+@pytest.mark.canary_quick
 @pytest.mark.parametrize('instance_type', ['ml.c5.xlarge', 'ml.p3.2xlarge'])
 def test_horovod(sagemaker_session, instance_type, tmpdir):
 

tests/integ/test_horovod.py

@@ -28,7 +28,7 @@
 from sagemaker.utils import sagemaker_timestamp
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_inference_pipeline_model_deploy(sagemaker_session):
     sparkml_data_path = os.path.join(DATA_DIR, 'sparkml_model')

tests/integ/test_inference_pipeline.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import os
-import pytest
 
 from sagemaker import IPInsights, IPInsightsModel
 from sagemaker.predictor import RealTimePredictor
@@ -25,7 +24,6 @@
 FEATURE_DIM = None
 
 
-@pytest.mark.continuous_testing
 def test_ipinsights(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'ipinsights')

tests/integ/test_ipinsights.py

@@ -26,7 +26,6 @@
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_kmeans(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_kmeans.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 from sagemaker import KNN, KNNModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_knn_regressor(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_knn.py

@@ -15,7 +15,6 @@
 import os
 
 import numpy as np
-import pytest
 
 from sagemaker import LDA, LDAModel
 from sagemaker.amazon.common import read_records
@@ -25,7 +24,6 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
-@pytest.mark.continuous_testing
 def test_lda(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'lda')

tests/integ/test_lda.py

@@ -27,7 +27,7 @@
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_linear_learner(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_linear_learner.py

@@ -17,6 +17,7 @@
 import time
 
 import pandas
+import pytest
 
 import sagemaker
 from sagemaker import AlgorithmEstimator, ModelPackage
@@ -42,6 +43,7 @@
                     '154230595-8f00905c1f927a512b73ea29dd09ae30'
 
 
+@pytest.mark.canary_quick
 def test_marketplace_estimator(sagemaker_session):
     with timeout(minutes=15):
         data_path = os.path.join(DATA_DIR, 'marketplace', 'training')
@@ -112,6 +114,7 @@ def test_marketplace_attach(sagemaker_session):
         print(predictor.predict(test_x.values).decode('utf-8'))
 
 
+@pytest.mark.canary_quick
 def test_marketplace_model(sagemaker_session):
 
     def predict_wrapper(endpoint, session):

tests/integ/test_marketplace.py

@@ -45,7 +45,7 @@ def mxnet_training_job(sagemaker_session, mxnet_full_version):
         return mx.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_attach_deploy(mxnet_training_job, sagemaker_session):
     endpoint_name = 'test-mxnet-attach-deploy-{}'.format(sagemaker_timestamp())
@@ -122,7 +122,7 @@ def test_deploy_model_with_update_non_existing_endpoint(mxnet_training_job, sage
             model.deploy(1, 'ml.m4.xlarge', update_endpoint=True, endpoint_name='non-existing-endpoint')
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(tests.integ.test_region() not in tests.integ.EI_SUPPORTED_REGIONS,
                     reason="EI isn't supported in that specific region.")

tests/integ/test_mxnet_train.py

@@ -44,7 +44,7 @@ def mxnet_training_job(sagemaker_session, mxnet_full_version):
         return mx.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skip(reason="This should be enabled along with the Boto SDK release for Neo API changes")
 def test_attach_deploy(mxnet_training_job, sagemaker_session):

tests/integ/test_neo_mxnet.py

@@ -25,7 +25,7 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_ntm(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'ntm')

tests/integ/test_ntm.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import os
-import pytest
 
 from sagemaker.predictor import RealTimePredictor
 from sagemaker import Object2Vec, Object2VecModel
@@ -26,7 +25,6 @@
 FEATURE_NUM = None
 
 
-@pytest.mark.continuous_testing
 def test_object2vec(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'object2vec')

tests/integ/test_object2vec.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 import sagemaker.amazon.pca
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_pca(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_pca.py

@@ -39,7 +39,7 @@ def fixture_training_job(sagemaker_session, pytorch_full_version):
         return pytorch.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_sync_fit_deploy(pytorch_training_job, sagemaker_session):
     # TODO: add tests against local mode when it's ready to be used

tests/integ/test_pytorch_train.py

@@ -13,15 +13,13 @@
 from __future__ import absolute_import
 
 import numpy as np
-import pytest
 
 from sagemaker import RandomCutForest, RandomCutForestModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_randomcutforest(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         # Generate a thousand 14-dimensional datapoints.

tests/integ/test_randomcutforest.py

@@ -25,7 +25,7 @@
 CPU_INSTANCE = 'ml.m4.xlarge'
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="RL images supports only Python 3.")
 def test_coach_mxnet(sagemaker_session, rl_coach_full_version):
     estimator = _test_coach(sagemaker_session, RLFramework.MXNET, rl_coach_full_version)
@@ -89,7 +89,7 @@ def _test_coach(sagemaker_session, rl_framework, rl_coach_version):
                        })
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="RL images supports only Python 3.")
 def test_ray_tf(sagemaker_session, rl_ray_full_version):
     source_dir = os.path.join(DATA_DIR, 'ray_cartpole')

tests/integ/test_rl.py

@@ -54,7 +54,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, sklearn_ful
         return sklearn.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="Scikit-learn image supports only python 3.")
 def test_attach_deploy(sklearn_training_job, sagemaker_session):

tests/integ/test_sklearn_train.py

@@ -23,7 +23,7 @@
 from tests.integ.timeout import timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_sparkml_model_deploy(sagemaker_session):
     # Uploads an MLeap serialized MLeap model to S3 and use that to deploy a SparkML model to perform inference

tests/integ/test_sparkml_serving.py

@@ -52,7 +52,7 @@ def tf_training_job(sagemaker_session, tf_full_version):
         return estimator.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.")
 def test_deploy_model(sagemaker_session, tf_training_job):
@@ -78,7 +78,7 @@ def test_deploy_model(sagemaker_session, tf_training_job):
         assert dict_result == list_result
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(tests.integ.test_region() not in tests.integ.EI_SUPPORTED_REGIONS,
                     reason="EI isn't supported in that specific region.")