breaking: merge v2 changes into master

Author: laurenyu

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = sagemaker/tests/*, sagemaker/tensorflow/tensorflow_serving/*
+concurrency = threading
+omit = sagemaker/tests/*
+timid = True

CHANGELOG.md

@@ -74,6 +74,63 @@
 
  * add KFP Processing component
 
+## v2.0.0.rc1 (2020-07-08)
+
+### Breaking Changes
+
+ * Move StreamDeserializer to sagemaker.deserializers
+ * Move StringDeserializer to sagemaker.deserializers
+ * rename record_deserializer to RecordDeserializer
+ * remove "train_" where redundant in parameter/variable names
+ * Add BytesDeserializer
+ * rename image to image_uri
+ * rename image_name to image_uri
+ * create new inference resources during model.deploy() and model.transformer()
+ * rename session parameter to sagemaker_session in S3 utility classes
+ * rename distributions to distribution in TF/MXNet estimators
+ * deprecate update_endpoint arg in deploy()
+ * create new inference resources during estimator.deploy() or estimator.transformer()
+ * deprecate delete_endpoint() for estimators and HyperparameterTuner
+ * refactor Predictor attribute endpoint to endpoint_name
+ * make instance_type optional for Airflow model configs
+ * refactor name of RealTimePredictor to Predictor
+ * remove check for Python 2 string in sagemaker.predictor._is_sequence_like()
+ * deprecate sagemaker.utils.to_str()
+ * drop Python 2 support
+
+### Features
+
+ * add BaseSerializer and BaseDeserializer
+ * add Predictor.update_endpoint()
+
+### Bug Fixes and Other Changes
+
+ * handle "train_*" renames in v2 migration tool
+ * handle image_uri rename for Session methods in v2 migration tool
+ * Update BytesDeserializer accept header
+ * handle image_uri rename for estimators and models in v2 migration tool
+ * handle image_uri rename in Airflow model config functions in v2 migration tool
+ * update migration tool for S3 utility functions
+ * set _current_job_name and base_tuning_job_name in HyperparameterTuner.attach()
+ * infer base name from job name in estimator.attach()
+ * ensure generated names are < 63 characters when deploying compiled models
+ * add TF migration documentation to error message
+
+### Documentation Changes
+
+ * update documentation with v2.0.0.rc1 changes
+ * remove 'train_*' prefix from estimator parameters
+ * update documentation for image_name/image --> image_uri
+
+### Testing and Release Infrastructure
+
+ * refactor matching logic in v2 migration tool
+ * add cli modifier for RealTimePredictor and derived classes
+ * change coverage settings to reduce intermittent errors
+ * clean up pickle.load logic in integ tests
+ * use fixture for Python version in framework integ tests
+ * remove assumption of Python 2 unit test runs
+
 ## v1.68.0 (2020-07-07)
 
 ### Features
@@ -165,6 +222,31 @@
  * set logs to False if wait is False in AutoML
  * workflow passing spot training param to training job
 
+## v2.0.0.rc0 (2020-06-17)
+
+### Breaking Changes
+
+ * remove estimator parameters for TF legacy mode
+ * remove legacy `TensorFlowModel` and `TensorFlowPredictor` classes
+ * force image URI to be passed for legacy TF images
+ * rename `sagemaker.tensorflow.serving` to `sagemaker.tensorflow.model`
+ * require `framework_version` and `py_version` for framework estimator and model classes
+ * change `Model` parameter order to make `model_data` optional
+
+### Bug Fixes and Other Changes
+
+ * add v2 migration tool
+
+### Documentation Changes
+
+ * update TF documentation to reflect breaking changes and how to upgrade
+ * start v2 usage and migration documentation
+
+### Testing and Release Infrastructure
+
+ * remove scipy from dependencies
+ * remove TF from optional dependencies
+
 ## v1.64.1 (2020-06-16)
 
 ### Bug Fixes and Other Changes

MANIFEST.in

@@ -1,4 +1,6 @@
-recursive-include src/sagemaker *
+recursive-include src/sagemaker *.py
+
+include src/sagemaker/image_uri_config/*.json
 
 include VERSION
 include LICENSE.txt

README.rst

@@ -93,7 +93,6 @@ Supported Python Versions
 
 SageMaker Python SDK is tested on:
 
-- Python 2.7
 - Python 3.6
 - Python 3.7
 - Python 3.8
@@ -122,10 +121,9 @@ You can install the libraries needed to run the tests by running :code:`pip inst
 
 **Unit tests**
 
-
 We run unit tests with tox, which is a program that lets you run unit tests for multiple Python versions, and also make sure the
-code fits our style guidelines. We run tox with Python 2.7, 3.6, 3.7, and 3.8, so to run unit tests
-with the same configuration we do, you'll need to have interpreters for Python 2.7, Python 3.6, Python 3.7, and Python 3.8 installed.
+code fits our style guidelines. We run tox with `all of our supported Python versions <#supported-python-versions>`_, so to run unit tests
+with the same configuration we do, you need to have interpreters for those Python versions installed.
 
 To run the unit tests with tox, run:
 

VERSION

@@ -1 +1 @@
-1.72.1.dev0
+2.0.0.rc1

bin/sagemaker-submit

@@ -11,5 +11,5 @@ phases:
 
       # local mode tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py27,py38 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
-      - ./ci-scripts/displaytime.sh 'py27,py38 local mode' $start_time
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml"
+      - ./ci-scripts/displaytime.sh 'py38 local mode' $start_time

buildspec-localmodetests.yml

@@ -18,7 +18,7 @@ phases:
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py27,py36,py37,py38 -- tests/unit
+        tox -e py36,py37,py38 -- tests/unit
 
       # run a subset of the integration tests
       - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2

buildspec-release.yml

@@ -18,5 +18,5 @@ phases:
       - start_time=`date +%s`
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py27,py36,py37,py38 --parallel all -- tests/unit
-      - ./ci-scripts/displaytime.sh 'py27,py36,py37,py38 unit' $start_time
+        tox -e py36,py37,py38 --parallel all -- tests/unit
+      - ./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time

buildspec-unittests.yml

@@ -8,7 +8,7 @@ The Amazon SageMaker Factorization Machines algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_factors, predictor_type, epochs, clip_gradient, mini_batch_size, feature_dim, eps, rescale_grad, bias_lr, linear_lr, factors_lr, bias_wd, linear_wd, factors_wd, bias_init_method, bias_init_scale, bias_init_sigma, bias_init_value, linear_init_method, linear_init_scale, linear_init_sigma, linear_init_value, factors_init_method, factors_init_scale, factors_init_sigma, factors_init_value
+    :exclude-members: image_uri, num_factors, predictor_type, epochs, clip_gradient, mini_batch_size, feature_dim, eps, rescale_grad, bias_lr, linear_lr, factors_lr, bias_wd, linear_wd, factors_wd, bias_init_method, bias_init_scale, bias_init_sigma, bias_init_value, linear_init_method, linear_init_scale, linear_init_sigma, linear_init_value, factors_init_method, factors_init_scale, factors_init_sigma, factors_init_value
 
 
 .. autoclass:: sagemaker.FactorizationMachinesModel

doc/algorithms/factorization_machines.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker IP Insights algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_entity_vectors, vector_dim, batch_metrics_publish_interval, epochs, learning_rate,
+    :exclude-members: image_uri, num_entity_vectors, vector_dim, batch_metrics_publish_interval, epochs, learning_rate,
                       num_ip_encoder_layers, random_negative_sampling_rate, shuffled_negative_sampling_rate, weight_decay
 
 .. autoclass:: sagemaker.IPInsightsModel

doc/algorithms/ipinsights.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker K-means algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, k, init_method, max_iterations, tol, num_trials, local_init_method, half_life_time_size, epochs, center_factor, mini_batch_size, feature_dim, MAX_DEFAULT_BATCH_SIZE
+    :exclude-members: image_uri, k, init_method, max_iterations, tol, num_trials, local_init_method, half_life_time_size, epochs, center_factor, mini_batch_size, feature_dim, MAX_DEFAULT_BATCH_SIZE
 
 .. autoclass:: sagemaker.KMeansModel
     :members:

doc/algorithms/kmeans.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker K-Nearest Neighbors (k-NN) algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, k, sample_size, predictor_type, dimension_reduction_target, dimension_reduction_type,
+    :exclude-members: image_uri, k, sample_size, predictor_type, dimension_reduction_target, dimension_reduction_type,
                       index_metric, index_type, faiss_index_ivf_nlists, faiss_index_pq_m
 
 .. autoclass:: sagemaker.KNNModel

doc/algorithms/knn.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker LDA algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_topics, alpha0, max_restarts, max_iterations, mini_batch_size, feature_dim, tol
+    :exclude-members: image_uri, num_topics, alpha0, max_restarts, max_iterations, mini_batch_size, feature_dim, tol
 
 
 .. autoclass:: sagemaker.LDAModel

doc/algorithms/lda.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker LinearLearner algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, train_instance_count, train_instance_type, predictor_type, binary_classifier_model_selection_criteria, target_recall, target_precision, positive_example_weight_mult, epochs, use_bias, num_models, parameter, num_calibration_samples, calibration, init_method, init_scale, init_sigma, init_bias, optimizer, loss, wd, l1, momentum, learning_rate, beta_1, beta_2, bias_lr_mult, use_lr_scheduler, lr_scheduler_step, lr_scheduler_factor, lr_scheduler_minimum_lr, lr_scheduler_minimum_lr, mini_batch_size, feature_dim, bias_wd_mult, MAX_DEFAULT_BATCH_SIZE
+    :exclude-members: image_uri, instance_count, instance_type, predictor_type, binary_classifier_model_selection_criteria, target_recall, target_precision, positive_example_weight_mult, epochs, use_bias, num_models, parameter, num_calibration_samples, calibration, init_method, init_scale, init_sigma, init_bias, optimizer, loss, wd, l1, momentum, learning_rate, beta_1, beta_2, bias_lr_mult, use_lr_scheduler, lr_scheduler_step, lr_scheduler_factor, lr_scheduler_minimum_lr, lr_scheduler_minimum_lr, mini_batch_size, feature_dim, bias_wd_mult, MAX_DEFAULT_BATCH_SIZE
 
 .. autoclass:: sagemaker.LinearLearnerModel
     :members:

doc/algorithms/linear_learner.rst

@@ -8,8 +8,8 @@ The Amazon SageMaker NTM algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_topics, encoder_layers, epochs, encoder_layers_activation, optimizer, tolerance,
-                 num_patience_epochs, batch_norm, rescale_gradient, clip_gradient, weight_decay, learning_rate
+    :exclude-members: image_uri, num_topics, encoder_layers, epochs, encoder_layers_activation, optimizer, tolerance,
+                      num_patience_epochs, batch_norm, rescale_gradient, clip_gradient, weight_decay, learning_rate
 
 
 .. autoclass:: sagemaker.NTMModel

doc/algorithms/ntm.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker Object2Vec algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, enc_dim, mini_batch_size, epochs, early_stopping_patience, early_stopping_tolerance,
+    :exclude-members: image_uri, enc_dim, mini_batch_size, epochs, early_stopping_patience, early_stopping_tolerance,
                       dropout, weight_decay, bucket_width, num_classes, mlp_layers, mlp_dim, mlp_activation,
                       output_layer, optimizer, learning_rate, enc0_network, enc1_network, enc0_cnn_filter_width,
                       enc1_cnn_filter_width, enc0_max_seq_len, enc1_max_seq_len, enc0_token_embedding_dim,

doc/algorithms/object2vec.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker PCA algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_components, algorithm_mode, subtract_mean, extra_components, mini_batch_size, feature_dim, MAX_DEFAULT_BATCH_SIZE
+    :exclude-members: image_uri, num_components, algorithm_mode, subtract_mean, extra_components, mini_batch_size, feature_dim, MAX_DEFAULT_BATCH_SIZE
 
 
 .. autoclass:: sagemaker.PCAModel

doc/algorithms/pca.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker Random Cut Forest algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image, num_trees, num_samples_per_tree, eval_metrics, feature_dim, MINI_BATCH_SIZE
+    :exclude-members: image_uri, num_trees, num_samples_per_tree, eval_metrics, feature_dim, MINI_BATCH_SIZE
 
 
 .. autoclass:: sagemaker.RandomCutForestModel

doc/algorithms/randomcutforest.rst

@@ -54,8 +54,8 @@ The ``DebuggerHookConfig`` accepts one or more objects of type ``CollectionConfi
 
     estimator = TensorFlow(
         role=role,
-        train_instance_count=1,
-        train_instance_type=train_instance_type,
+        instance_count=1,
+        instance_type=instance_type,
         debugger_hook_config=debugger_hook_config
     )
 
@@ -215,8 +215,8 @@ Sample Usages
 
     estimator = TensorFlow(
             role=role,
-            train_instance_count=1,
-            train_instance_type=train_instance_type,
+            instance_count=1,
+            instance_type=instance_type,
             rules=[Rule.sagemaker(vanishing_gradient())]
     )
 
@@ -232,8 +232,8 @@ In the example above, Amazon SageMaker pulls the collection configuration best s
 
     estimator = TensorFlow(
             role=role,
-            train_instance_count=1,
-            train_instance_type=train_instance_type,
+            instance_count=1,
+            instance_type=instance_type,
             rules=[Rule.sagemaker(vanishing_gradient()), Rule.sagemaker(weight_update_ratio())]
     )
 
@@ -269,8 +269,8 @@ Here we modify the ``weight_update_ratio`` rule to store a custom collection rat
 
     estimator = TensorFlow(
             role=role,
-            train_instance_count=1,
-            train_instance_type=train_instance_type,
+            instance_count=1,
+            instance_type=instance_type,
             rules=[
                 Rule.sagemaker(vanishing_gradient()),
                 wur_with_customization
@@ -317,8 +317,8 @@ To evaluate the custom rule against the training:
 
     estimator = TensorFlow(
         role=role,
-        train_instance_count=1,
-        train_instance_type=train_instance_type,
+        instance_count=1,
+        instance_type=instance_type,
         rules=[
             custom_gradient_rule
         ]
@@ -344,8 +344,8 @@ To enable the debugging hook to emit TensorBoard data, you need to specify the n
 
     estimator = TensorFlow(
         role=role,
-        train_instance_count=1,
-        train_instance_type=train_instance_type,
+        instance_count=1,
+        instance_type=instance_type,
         tensorboard_output_config=tensorboard_output_config
     )
 
@@ -392,8 +392,8 @@ To disable the hook initialization, you can do so by specifying ``False`` for va
 
     estimator = TensorFlow(
         role=role,
-        train_instance_count=1,
-        train_instance_type=train_instance_type,
+        instance_count=1,
+        instance_type=instance_type,
         debugger_hook_config=False
     )
 

doc/amazon_sagemaker_debugger.rst

@@ -125,7 +125,7 @@ Using ``DefaultMonitor.create_monitoring_schedule()``, you can create a model mo
 
     my_monitor.create_monitoring_schedule(
         monitor_schedule_name='my-monitoring-schedule',
-        endpoint_input=predictor.endpoint,
+        endpoint_input=predictor.endpoint_name,
         statistics=my_monitor.baseline_statistics(),
         constraints=my_monitor.suggested_constraints(),
         schedule_cron_expression=CronExpressionGenerator.hourly(),