Merge branch 'master' into ioFollowup2

Author: laurenyu

.pylintrc

@@ -1,5 +1,128 @@
 # Changelog
 
+## v1.34.0 (2019-07-18)
+
+### Features
+
+ * Git integration for CodeCommit
+ * deal with credentials for Git support for GitHub
+
+### Bug fixes and other changes
+
+ * modify TODO on disabled Pylint check
+ * enable consider-using-ternary Pylint check
+ * enable chained-comparison Pylint check
+ * enable too-many-public-methods Pylint check
+ * enable consider-using-in Pylint check
+ * set num_processes_per_host only if provided by user
+ * fix attach for 1P algorithm estimators
+ * enable ungrouped-imports Pylint check
+ * enable wrong-import-order Pylint check
+ * enable attribute-defined-outside-init Pylint check
+ * enable consider-merging-isinstance Pylint check
+ * enable inconsistent-return-statements Pylint check
+ * enable simplifiable-if-expression pylint checks
+ * fix list serialization for 1P algos
+ * enable no-else-return and no-else-raise pylint checks
+ * enable unidiomatic-typecheck pylint check
+
+## v1.33.0 (2019-07-10)
+
+### Features
+
+ * git support for hosting models
+ * allow custom model name during deploy
+
+### Bug fixes and other changes
+
+ * remove TODO comment on import-error Pylint check
+ * enable wrong-import-position pylint check
+ * Revert "change: enable wrong-import-position pylint check (#907)"
+ * enable signature-differs pylint check
+ * enable wrong-import-position pylint check
+ * enable logging-not-lazy pylint check
+ * reset default output path in Transformer.transform
+ * Add ap-northeast-1 to Neo algorithms region map
+
+## v1.32.2 (2019-07-08)
+
+### Bug fixes and other changes
+
+ * enable logging-format-interpolation pylint check
+ * remove superfluous parens per Pylint rule
+
+### Documentation changes
+
+ * add pypi, rtd, black badges to readme
+
+## v1.32.1 (2019-07-04)
+
+### Bug fixes and other changes
+
+ * correct code per len-as-condition Pylint check
+ * tighten pylint config and expand C and R exceptions
+ * Update displaytime.sh
+ * fix notebook tests
+ * separate unit, local mode, and notebook tests in different buildspecs
+
+### Documentation changes
+
+ * refactor the overview topic in the sphinx project
+
+## v1.32.0 (2019-07-02)
+
+### Features
+
+ * support Endpoint_type for TF transform
+
+### Bug fixes and other changes
+
+ * fix git test in test_estimator.py
+ * Add ap-northeast-1 to Neo algorithms region map
+
+## v1.31.1 (2019-07-01)
+
+### Bug fixes and other changes
+
+ * print build execution time
+ * remove unnecessary failure case tests
+ * build spec improvements.
+
+## v1.31.0 (2019-06-27)
+
+### Features
+
+ * use deep learning images
+
+### Bug fixes and other changes
+
+ * Update buildspec.yml
+ * allow only one integration test run per time
+ * remove unnecessary P3 tests from TFS integration tests
+ * add pytest.mark.local_mode annotation to broken tests
+
+## v1.30.0 (2019-06-25)
+
+### Features
+
+ * add TensorFlow 1.13 support
+ * add git_config and git_clone, validate method
+
+### Bug fixes and other changes
+
+ * add pytest.mark.local_mode annotation to broken tests
+
+## v1.29.0 (2019-06-24)
+
+### Features
+
+ * network isolation mode in training
+
+### Bug fixes and other changes
+
+ * Integrate black into development process
+ * moving not canary TFS tests to local mode
+
 ## v1.28.3 (2019-06-20)
 
 ### Bug fixes and other changes

CHANGELOG.md

@@ -14,6 +14,22 @@ SageMaker Python SDK
    :target: https://codecov.io/gh/aws/sagemaker-python-sdk
    :alt: CodeCov
 
+.. image:: https://img.shields.io/pypi/v/sagemaker.svg
+   :target: https://pypi.python.org/pypi/sagemaker
+   :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/sagemaker.svg
+   :target: https://pypi.python.org/pypi/sagemaker
+   :alt: Supported Python Versions
+
+.. image:: https://img.shields.io/badge/code_style-black-000000.svg
+   :target: https://github.com/python/black
+   :alt: Code style: black
+
+.. image:: https://readthedocs.org/projects/sagemaker/badge/?version=stable
+   :target: https://sagemaker.readthedocs.io/en/stable/
+   :alt: Documentation Status
+
 SageMaker Python SDK is an open source library for training and deploying machine learning models on Amazon SageMaker.
 
 With the SDK, you can train and deploy models using popular deep learning frameworks **Apache MXNet** and **TensorFlow**.
@@ -157,9 +173,9 @@ MXNet SageMaker Estimators
 
 By using MXNet SageMaker Estimators, you can train and host MXNet models on Amazon SageMaker.
 
-Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``.
+Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``.
 
-Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``.
+Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``.
 
 We recommend that you use the latest supported version, because that's where we focus most of our development efforts.
 
@@ -173,7 +189,7 @@ TensorFlow SageMaker Estimators
 
 By using TensorFlow SageMaker Estimators, you can train and host TensorFlow models on Amazon SageMaker.
 
-Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``.
+Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``, ``1.13.1``.
 
 Supported versions of TensorFlow for Elastic Inference: ``1.11.0``, ``1.12.0``.
 

README.rst

@@ -1 +1 @@
-1.28.4.dev0
+1.34.1.dev0

VERSION

@@ -0,0 +1,25 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
+  build:
+    commands:
+      - IGNORE_COVERAGE=-
+
+      # local mode tests
+      - start_time=`date +%s`
+      - |
+        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+          tox -e py36 -- tests/integ -m local_mode --durations 50
+        fi
+      - ./ci-scripts/displaytime.sh 'py36 local mode' $start_time
+
+      - start_time=`date +%s`
+      - |
+        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+          tox -e py27 -- tests/integ -m local_mode --durations 50
+        fi
+      - ./ci-scripts/displaytime.sh 'py27 local mode' $start_time

buildspec-localmodetests.yml

@@ -0,0 +1,13 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # run notebook test
+      - echo "running notebook test"
+      - start_time=`date +%s`
+      - |
+        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "tests/scripts/run-notebook-test.sh"; then
+          ./tests/scripts/run-notebook-test.sh
+        fi
+      - ./ci-scripts/displaytime.sh 'notebook test' $start_time

buildspec-notebooktests.yml

@@ -9,6 +9,9 @@ phases:
       # run linters
       - tox -e flake8,pylint
 
+      # run format verification
+      - tox -e black-check
+
       # run package and docbuild checks
       - tox -e twine
       - tox -e sphinx

buildspec-release.yml

@@ -0,0 +1,22 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # run linters
+      - TOX_PARALLEL_NO_SPINNER=1
+      - PY_COLORS=0
+      - start_time=`date +%s`
+      - tox -e flake8,pylint,twine,black-check --parallel all
+      - ./ci-scripts/displaytime.sh 'flake8,pylint,twine,black-check' $start_time
+
+      - start_time=`date +%s`
+      - tox -e sphinx
+      - ./ci-scripts/displaytime.sh 'sphinx' $start_time
+
+      # run unit tests
+      - start_time=`date +%s`
+      - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
+        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
+        tox -e py36,py27 --parallel all -- tests/unit
+      - ./ci-scripts/displaytime.sh 'py36,py27 unit' $start_time

buildspec-unittests.yml

@@ -1,37 +1,32 @@
 version: 0.2
 
 phases:
-  pre_build:
-    commands:
-      - start-dockerd
-
   build:
     commands:
-      # run linters
-      - tox -e flake8,pylint
-
-      # run package and docbuild checks
-      - tox -e twine
-      - tox -e sphinx
+      - IGNORE_COVERAGE=-
 
-      # run unit tests
-      - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
-        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py27 -- tests/unit
-
-      # run notebook test
+      # run integration tests
+      - start_time=`date +%s`
       - |
-        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          echo "running notebook test"
-          ./tests/scripts/run-notebook-test.sh
-         else
-           echo "skipping notebook test"
-         fi
+        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+          python3 -u ci-scripts/queue_build.py
+        fi
+      - ./ci-scripts/displaytime.sh 'build queue' $start_time
 
-      # run integration tests
+      - start_time=`date +%s`
       - |
         if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          IGNORE_COVERAGE=- tox -e py36,py27 -- tests/integ -n 24 --boxed --reruns 2
-        else
-          echo "skipping integration tests"
+          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50
+        fi
+      - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time
+
+  post_build:
+    finally:
+      - |
+        if [ -d "ci-lock" ]; then
+          FILENAME=$(ls ci-lock/ || true)
+          ACCOUNT=$(aws sts get-caller-identity --output text | awk '{print $1}')
+          S3_BUCKET_DIR=s3://sagemaker-us-west-2-${ACCOUNT}/ci-lock/
+          aws s3 rm ${S3_BUCKET_DIR}${FILENAME}      
         fi
+

buildspec.yml

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+set -euo pipefail
+
+echo =================== $1 execution time ===================
+
+start_time=$2
+end_time=`date +%s`
+total_time=$(expr $end_time - $start_time + 1)
+hours=$((total_time/60/60%24))
+minutes=$((total_time/60%60))
+secs=$((total_time%60))
+
+(( $hours > 0 )) && printf '%d hours ' $hours
+(( $minutes > 0 )) && printf '%d minutes ' $minutes
+(( $hours > 0 || $minutes > 0 )) && printf 'and '
+printf '%d seconds\n\n' $secs