Merge branch 'zwei' into script-path

Author: laurenyu

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = sagemaker/tests/*, sagemaker/tensorflow/tensorflow_serving/*
+concurrency = threading
+omit = sagemaker/tests/*
+timid = True

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,17 +1,17 @@
 ---
 name: Feature request
-about: Suggest an improvement for this library
+about: Suggest new functionality for this library
 title: ''
 labels: ''
 assignees: ''
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+**Describe the feature you'd like**
+A clear and concise description of the functionality you want.
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+**How would this feature be used? Please describe.**
+A clear and concise description of the use case for this feature. Please provide an example, if possible.
 
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ _Put an `x` in the boxes that apply. You can also fill these out after creating
 
 - [ ] I have read the [CONTRIBUTING](https://github.com/aws/sagemaker-python-sdk/blob/master/CONTRIBUTING.md) doc
 - [ ] I used the commit message format described in [CONTRIBUTING](https://github.com/aws/sagemaker-python-sdk/blob/master/CONTRIBUTING.md#committing-your-change)
-- [ ] I have passed the region in to any/all clients that I've initialized as part of this change.
+- [ ] I have passed the region in to all S3 and STS clients that I've initialized as part of this change.
 - [ ] I have updated any necessary documentation, including [READMEs](https://github.com/aws/sagemaker-python-sdk/blob/master/README.rst) and [API docs](https://github.com/aws/sagemaker-python-sdk/tree/master/doc) (if appropriate)
 
 #### Tests

.pylintrc

@@ -76,21 +76,22 @@ confidence=
 # --disable=W"
 disable=
     C0330, # Black disagrees with and explicitly violates this: https://github.com/python/black/issues/48
-    too-many-locals,
+    abstract-method, # TODO: Fix abstract methods
     arguments-differ,
-    too-many-lines,
+    cyclic-import, # TODO: Resolve cyclic imports
     fixme,
-    too-many-arguments,
     invalid-name,
-    too-many-instance-attributes,
-    len-as-condition, # TODO: Enable this check once pylint 2.4.0 is released and consumed due to the fix in https://github.com/PyCQA/pylint/issues/2684
     import-error, # Since we run Pylint before any of our builds in tox, this will always fail
-    protected-access, # TODO: Fix access
-    abstract-method, # TODO: Fix abstract methods
-    useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
-    cyclic-import, # TODO: Resolve cyclic imports
+    import-outside-toplevel,
     no-self-use, # TODO: Convert methods to functions where appropriate
+    protected-access, # TODO: Fix access
+    signature-differs,  # TODO: fix kwargs
+    too-many-arguments,
     too-many-branches, # TODO: Simplify or ignore as appropriate
+    too-many-instance-attributes,
+    too-many-lines,
+    too-many-locals,
+    useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs

CHANGELOG.md

@@ -1,5 +1,229 @@
 # Changelog
 
+## v2.0.0.rc1 (2020-07-08)
+
+### Breaking Changes
+
+ * Move StreamDeserializer to sagemaker.deserializers
+ * Move StringDeserializer to sagemaker.deserializers
+ * rename record_deserializer to RecordDeserializer
+ * remove "train_" where redundant in parameter/variable names
+ * Add BytesDeserializer
+ * rename image to image_uri
+ * rename image_name to image_uri
+ * create new inference resources during model.deploy() and model.transformer()
+ * rename session parameter to sagemaker_session in S3 utility classes
+ * rename distributions to distribution in TF/MXNet estimators
+ * deprecate update_endpoint arg in deploy()
+ * create new inference resources during estimator.deploy() or estimator.transformer()
+ * deprecate delete_endpoint() for estimators and HyperparameterTuner
+ * refactor Predictor attribute endpoint to endpoint_name
+ * make instance_type optional for Airflow model configs
+ * refactor name of RealTimePredictor to Predictor
+ * remove check for Python 2 string in sagemaker.predictor._is_sequence_like()
+ * deprecate sagemaker.utils.to_str()
+ * drop Python 2 support
+
+### Features
+
+ * add BaseSerializer and BaseDeserializer
+ * add Predictor.update_endpoint()
+
+### Bug Fixes and Other Changes
+
+ * handle "train_*" renames in v2 migration tool
+ * handle image_uri rename for Session methods in v2 migration tool
+ * Update BytesDeserializer accept header
+ * handle image_uri rename for estimators and models in v2 migration tool
+ * handle image_uri rename in Airflow model config functions in v2 migration tool
+ * update migration tool for S3 utility functions
+ * set _current_job_name and base_tuning_job_name in HyperparameterTuner.attach()
+ * infer base name from job name in estimator.attach()
+ * ensure generated names are < 63 characters when deploying compiled models
+ * add TF migration documentation to error message
+
+### Documentation Changes
+
+ * update documentation with v2.0.0.rc1 changes
+ * remove 'train_*' prefix from estimator parameters
+ * update documentation for image_name/image --> image_uri
+
+### Testing and Release Infrastructure
+
+ * refactor matching logic in v2 migration tool
+ * add cli modifier for RealTimePredictor and derived classes
+ * change coverage settings to reduce intermittent errors
+ * clean up pickle.load logic in integ tests
+ * use fixture for Python version in framework integ tests
+ * remove assumption of Python 2 unit test runs
+
+## v1.68.0 (2020-07-07)
+
+### Features
+
+ * add spot instance support for AlgorithmEstimator
+
+### Documentation Changes
+
+ * add xgboost documentation for inference
+
+## v1.67.1.post0 (2020-07-01)
+
+### Documentation Changes
+
+ * add Step Functions SDK info
+
+## v1.67.1 (2020-06-30)
+
+### Bug Fixes and Other Changes
+
+ * add deprecation warnings for estimator.delete_endpoint() and tuner.delete_endpoint()
+
+## v1.67.0 (2020-06-29)
+
+### Features
+
+ * Apache Airflow integration for SageMaker Processing Jobs
+
+### Bug Fixes and Other Changes
+
+ * fix punctuation in warning message
+
+### Testing and Release Infrastructure
+
+ * address warnings about pytest custom marks, error message checking, and yaml loading
+ * mark long-running cron tests
+ * fix tox test dependencies and bump coverage threshold to 86%
+
+## v1.66.0 (2020-06-25)
+
+### Features
+
+ * add 3.8 as supported python version
+
+### Testing and Release Infrastructure
+
+ * upgrade airflow to latest stable version
+ * update feature request issue template
+
+## v1.65.1.post1 (2020-06-24)
+
+### Testing and Release Infrastructure
+
+ * add py38 to buildspecs
+
+## v1.65.1.post0 (2020-06-22)
+
+### Documentation Changes
+
+ * document that Local Mode + local code doesn't support dependencies arg
+
+### Testing and Release Infrastructure
+
+ * upgrade Sphinx to 3.1.1
+
+## v1.65.1 (2020-06-18)
+
+### Bug Fixes and Other Changes
+
+ * remove include_package_data=True from setup.py
+
+### Documentation Changes
+
+ * add some clarification to Processing docs
+
+### Testing and Release Infrastructure
+
+ * specify what kinds of clients in PR template
+
+## v1.65.0 (2020-06-17)
+
+### Features
+
+ * support for describing hyperparameter tuning job
+
+### Bug Fixes and Other Changes
+
+ * update distributed GPU utilization warning message
+ * set logs to False if wait is False in AutoML
+ * workflow passing spot training param to training job
+
+## v2.0.0.rc0 (2020-06-17)
+
+### Breaking Changes
+
+ * remove estimator parameters for TF legacy mode
+ * remove legacy `TensorFlowModel` and `TensorFlowPredictor` classes
+ * force image URI to be passed for legacy TF images
+ * rename `sagemaker.tensorflow.serving` to `sagemaker.tensorflow.model`
+ * require `framework_version` and `py_version` for framework estimator and model classes
+ * change `Model` parameter order to make `model_data` optional
+
+### Bug Fixes and Other Changes
+
+ * add v2 migration tool
+
+### Documentation Changes
+
+ * update TF documentation to reflect breaking changes and how to upgrade
+ * start v2 usage and migration documentation
+
+### Testing and Release Infrastructure
+
+ * remove scipy from dependencies
+ * remove TF from optional dependencies
+
+## v1.64.1 (2020-06-16)
+
+### Bug Fixes and Other Changes
+
+ * include py38 tox env and some dependency upgrades
+
+## v1.64.0 (2020-06-15)
+
+### Features
+
+ * add support for SKLearn 0.23
+
+## v1.63.0 (2020-06-12)
+
+### Features
+
+ * Allow selecting inference response content for automl generated models
+ * Support for multi variant endpoint invocation with target variant param
+
+### Documentation Changes
+
+ * improve docstring and remove unavailable links
+
+## v1.62.0 (2020-06-11)
+
+### Features
+
+ * Support for multi variant endpoint invocation with target variant param
+
+### Bug Fixes and Other Changes
+
+ * Revert "feature: Support for multi variant endpoint invocation with target variant param (#1571)"
+ * make instance_type optional for prepare_container_def
+ * docs: workflows navigation
+
+### Documentation Changes
+
+ * fix typo in MXNet documentation
+
+## v1.61.0 (2020-06-09)
+
+### Features
+
+ * Use boto3 DEFAULT_SESSION when no boto3 session specified.
+
+### Bug Fixes and Other Changes
+
+ * remove v2 Session warnings
+ * upgrade smdebug-rulesconfig to 0.1.4
+ * explicitly handle arguments in create_model for sklearn and xgboost
+
 ## v1.60.2 (2020-05-29)
 
 ### Bug Fixes and Other Changes

MANIFEST.in

@@ -1,4 +1,6 @@
-recursive-include src/sagemaker *
+recursive-include src/sagemaker *.py
+
+include src/sagemaker/image_uri_config/*.json
 
 include VERSION
 include LICENSE.txt

README.rst

@@ -93,9 +93,9 @@ Supported Python Versions
 
 SageMaker Python SDK is tested on:
 
-- Python 2.7
 - Python 3.6
 - Python 3.7
+- Python 3.8
 
 AWS Permissions
 ~~~~~~~~~~~~~~~
@@ -121,10 +121,9 @@ You can install the libraries needed to run the tests by running :code:`pip inst
 
 **Unit tests**
 
-
 We run unit tests with tox, which is a program that lets you run unit tests for multiple Python versions, and also make sure the
-code fits our style guidelines. We run tox with Python 2.7, 3.6 and 3.7, so to run unit tests
-with the same configuration we do, you'll need to have interpreters for Python 2.7, Python 3.6 and Python 3.7 installed.
+code fits our style guidelines. We run tox with `all of our supported Python versions <#supported-python-versions>`_, so to run unit tests
+with the same configuration we do, you need to have interpreters for those Python versions installed.
 
 To run the unit tests with tox, run:
 
@@ -164,23 +163,26 @@ You can also run them in parallel:
 Building Sphinx docs
 ~~~~~~~~~~~~~~~~~~~~
 
-Setup a Python environment with ``sphinx`` and ``sagemaker``:
+Setup a Python environment, and install the dependencies listed in ``doc/requirements.txt``:
 
 ::
 
+    # conda
     conda create -n sagemaker python=3.7
     conda activate sagemaker
-    conda install sphinx==2.2.2
-    pip install sagemaker --user
+    conda install --file doc/requirements.txt
 
-Install the Read The Docs theme:
+    # pip
+    pip install -r doc/requirements.txt
 
-::
 
-    pip install sphinx_rtd_theme --user
+Clone/fork the repo, and install your local version:
+
+::
 
+    pip install --upgrade .
 
-Clone/fork the repo, ``cd`` into the ``sagemaker-python-sdk/doc`` directory and run:
+Then ``cd`` into the ``sagemaker-python-sdk/doc`` directory and run:
 
 ::
 

VERSION

@@ -1 +1 @@
-1.60.3.dev0
+2.0.0.rc1

bin/sagemaker-submit

@@ -56,4 +56,4 @@ if __name__ == '__main__':
                                            hyperparameters=hyperparameters,
                                            instance_count=args.instance_count,
                                            instance_type=args.instance_type)
-    estimator.fit(sagemaker.s3_input(args.data))
+    estimator.fit(sagemaker.TrainingInput(args.data))

buildspec-localmodetests.yml

@@ -11,5 +11,5 @@ phases:
 
       # local mode tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py27,py37 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
-      - ./ci-scripts/displaytime.sh 'py27,py37 local mode' $start_time
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
+      - ./ci-scripts/displaytime.sh 'py38 local mode' $start_time

buildspec-release.yml

@@ -18,7 +18,7 @@ phases:
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py27,py36,py37 -- tests/unit
+        tox -e py36,py37,py38 -- tests/unit
 
       # run a subset of the integration tests
       - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2

buildspec-unittests.yml

@@ -18,5 +18,5 @@ phases:
       - start_time=`date +%s`
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py27,py36,py37 --parallel all -- tests/unit
-      - ./ci-scripts/displaytime.sh 'py27,py36,py37 unit' $start_time
+        tox -e py36,py37,py38 --parallel all -- tests/unit
+      - ./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time