Merge branch 'master' into fix-pytorch-inference-test

Author: ajaykarpur

CHANGELOG.md

@@ -1,5 +1,56 @@
 # Changelog
 
+## v2.5.4 (2020-09-08)
+
+### Bug Fixes and Other Changes
+
+ * update max_run_wait to max_wait in v2.rst for estimator parameters
+ * Updating regional account ids for af-south-1 and eu-south-1
+ * add account ids for af-south-1 and eu-south-1 for debugger rules
+
+## v2.5.3 (2020-09-02)
+
+### Bug Fixes and Other Changes
+
+ * Revert "change: update image uri config for pytorch 1.6.0 inference (#1864)"
+ * update image uri config for pytorch 1.6.0 inference
+ * add missing framework version image uri config
+
+## v2.5.2 (2020-08-31)
+
+### Bug Fixes and Other Changes
+
+ * refactor normalization of args for processing
+ * set TF 2.1.1 as highest py2 version for TF
+ * decrease integ test concurrency and increase delay between retries
+
+## v2.5.1 (2020-08-27)
+
+### Bug Fixes and Other Changes
+
+ * formatting changes from updates to black
+
+## v2.5.0 (2020-08-25)
+
+### Features
+
+ * add mypy tox target
+
+### Bug Fixes and Other Changes
+
+ * break out methods to get processing arguments
+ * break out methods to get train arguments
+
+## v2.4.2 (2020-08-24)
+
+### Bug Fixes and Other Changes
+
+ * check ast node on later renamers for cli v2 updater
+
+### Documentation Changes
+
+ * Clarify removals in v2
+
 ## v2.4.1 (2020-08-19)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.4.2.dev0
+2.5.5.dev0

buildspec.yml

@@ -16,7 +16,7 @@ phases:
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "src/sagemaker/image_uri_config/*.json" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time
 
   post_build:

doc/v2.rst

@@ -66,13 +66,13 @@ Please instantiate the objects instead.
 +--------------------------------------------+------------------------------------------------+
 | ``sagemaker.predictor.json_serializer``    | ``sagemaker.serializers.JSONSerializer()``     |
 +--------------------------------------------+------------------------------------------------+
-| ``sagemaker.predictor.npy_serializer``     | ``sagemaker.deserializers.NumpySerializer()``  |
+| ``sagemaker.predictor.npy_serializer``     | ``sagemaker.serializers.NumpySerializer()``    |
 +--------------------------------------------+------------------------------------------------+
 | ``sagemaker.predictor.csv_deserializer``   | ``sagemaker.deserializers.CSVDeserializer()``  |
 +--------------------------------------------+------------------------------------------------+
 | ``sagemaker.predictor.json_deserializer``  | ``sagemaker.deserializers.JSONDeserializer()`` |
 +--------------------------------------------+------------------------------------------------+
-| ``sagemaker.predictor.numpy_deserializer`` | ``sagemaker.serializers.NumpyDeserializer()``  |
+| ``sagemaker.predictor.numpy_deserializer`` | ``sagemaker.deserializers.NumpyDeserializer()``|
 +--------------------------------------------+------------------------------------------------+
 
 ``update_endpoint`` in ``deploy()``
@@ -231,7 +231,7 @@ The following estimator parameters have been renamed:
 +------------------------------+------------------------+
 | ``train_use_spot_instances`` | ``use_spot_instances`` |
 +------------------------------+------------------------+
-| ``train_max_run_wait``       | ``max_run_wait``       |
+| ``train_max_run_wait``       | ``max_wait``           |
 +------------------------------+------------------------+
 | ``train_volume_size``        | ``volume_size``        |
 +------------------------------+------------------------+

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+ignore_missing_imports = True

src/sagemaker/amazon/amazon_estimator.py

@@ -94,7 +94,9 @@ def __init__(
     def training_image_uri(self):
         """Placeholder docstring"""
         return image_uris.retrieve(
-            self.repo_name, self.sagemaker_session.boto_region_name, version=self.repo_version,
+            self.repo_name,
+            self.sagemaker_session.boto_region_name,
+            version=self.repo_version,
         )
 
     def hyperparameters(self):

src/sagemaker/amazon/kmeans.py

@@ -29,7 +29,7 @@ class KMeans(AmazonAlgorithmEstimatorBase):
 
     As the result of KMeans, members of a group are as similar as possible to one another and as
     different as possible from members of other groups. You define the attributes that you want
-    the algorithm to use to determine similarity. """
+    the algorithm to use to determine similarity."""
 
     repo_name = "kmeans"
     repo_version = 1
@@ -257,7 +257,9 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
         """
         sagemaker_session = sagemaker_session or Session()
         image_uri = image_uris.retrieve(
-            KMeans.repo_name, sagemaker_session.boto_region_name, version=KMeans.repo_version,
+            KMeans.repo_name,
+            sagemaker_session.boto_region_name,
+            version=KMeans.repo_version,
         )
         super(KMeansModel, self).__init__(
             image_uri,

src/sagemaker/amazon/knn.py

@@ -246,7 +246,9 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
         """
         sagemaker_session = sagemaker_session or Session()
         image_uri = image_uris.retrieve(
-            KNN.repo_name, sagemaker_session.boto_region_name, version=KNN.repo_version,
+            KNN.repo_name,
+            sagemaker_session.boto_region_name,
+            version=KNN.repo_version,
         )
         super(KNNModel, self).__init__(
             image_uri,

src/sagemaker/amazon/lda.py

@@ -27,10 +27,10 @@
 class LDA(AmazonAlgorithmEstimatorBase):
     """An unsupervised learning algorithm attempting to describe data as distinct categories.
 
-     LDA is most commonly used to discover a
-     user-specified number of topics shared by documents within a text corpus. Here each
-     observation is a document, the features are the presence (or occurrence count) of each
-     word, and the categories are the topics."""
+    LDA is most commonly used to discover a
+    user-specified number of topics shared by documents within a text corpus. Here each
+    observation is a document, the features are the presence (or occurrence count) of each
+    word, and the categories are the topics."""
 
     repo_name = "lda"
     repo_version = 1
@@ -230,7 +230,9 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
         """
         sagemaker_session = sagemaker_session or Session()
         image_uri = image_uris.retrieve(
-            LDA.repo_name, sagemaker_session.boto_region_name, version=LDA.repo_version,
+            LDA.repo_name,
+            sagemaker_session.boto_region_name,
+            version=LDA.repo_version,
         )
         super(LDAModel, self).__init__(
             image_uri,

src/sagemaker/amazon/ntm.py

@@ -259,7 +259,9 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
         """
         sagemaker_session = sagemaker_session or Session()
         image_uri = image_uris.retrieve(
-            NTM.repo_name, sagemaker_session.boto_region_name, version=NTM.repo_version,
+            NTM.repo_name,
+            sagemaker_session.boto_region_name,
+            version=NTM.repo_version,
         )
         super(NTMModel, self).__init__(
             image_uri,

src/sagemaker/amazon/pca.py

@@ -240,7 +240,9 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
         """
         sagemaker_session = sagemaker_session or Session()
         image_uri = image_uris.retrieve(
-            PCA.repo_name, sagemaker_session.boto_region_name, version=PCA.repo_version,
+            PCA.repo_name,
+            sagemaker_session.boto_region_name,
+            version=PCA.repo_version,
         )
         super(PCAModel, self).__init__(
             image_uri,

src/sagemaker/analytics.py

@@ -416,8 +416,7 @@ def _metric_names_for_training_job(self):
 
 
 class ExperimentAnalytics(AnalyticsMetricsBase):
-    """Fetch trial component data and make them accessible for analytics.
-    """
+    """Fetch trial component data and make them accessible for analytics."""
 
     MAX_TRIAL_COMPONENTS = 10000
 
@@ -477,16 +476,14 @@ def __init__(
 
     @property
     def name(self):
-        """Name of the Experiment being analyzed
-        """
+        """Name of the Experiment being analyzed"""
         return self._experiment_name
 
     def __repr__(self):
         return "<sagemaker.ExperimentAnalytics for %s>" % self.name
 
     def clear_cache(self):
-        """Clear the object of all local caches of API methods.
-        """
+        """Clear the object of all local caches of API methods."""
         super(ExperimentAnalytics, self).clear_cache()
         self._trial_components = None
 
@@ -570,13 +567,13 @@ def _reshape(self, trial_component):
 
     def _fetch_dataframe(self):
         """Return a pandas dataframe with all the trial_components,
-            along with their parameters and metrics.
+        along with their parameters and metrics.
         """
         df = pd.DataFrame([self._reshape(component) for component in self._get_trial_components()])
         return df
 
     def _get_trial_components(self, force_refresh=False):
-        """ Get all trial components matching the given search query expression.
+        """Get all trial components matching the given search query expression.
 
         Args:
             force_refresh (bool): Set to True to fetch the latest data from SageMaker API.

src/sagemaker/automl/automl.py

@@ -26,8 +26,7 @@
 
 
 class AutoML(object):
-    """A class for creating and interacting with SageMaker AutoML jobs
-    """
+    """A class for creating and interacting with SageMaker AutoML jobs"""
 
     def __init__(
         self,

src/sagemaker/cli/framework_upgrade.py

@@ -158,7 +158,13 @@ def add_region(existing_content, region, account):
 
 
 def add_version(
-    existing_content, short_version, full_version, scope, processors, py_versions, tag_prefix,
+    existing_content,
+    short_version,
+    full_version,
+    scope,
+    processors,
+    py_versions,
+    tag_prefix,
 ):
     """Read framework image uri information from json file to a dictionary, update it with new
     framework version information, then write the dictionary back to json file.
@@ -172,7 +178,7 @@ def add_version(
         processors (str): Supported processors (e.g. "cpu,gpu").
         py_versions (str): Supported Python versions (e.g. "py3,py37").
         tag_prefix (str): Algorithm image's tag prefix.
-     """
+    """
     if py_versions:
         py_versions = py_versions.split(",")
     processors = processors.split(",")

src/sagemaker/estimator.py

@@ -1037,11 +1037,31 @@ def start_new(cls, estimator, inputs, experiment_config):
                 :meth:`~sagemaker.estimator.EstimatorBase.fit`.  Dictionary contains
                 three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
 
-
         Returns:
             sagemaker.estimator._TrainingJob: Constructed object that captures
             all information about the started training job.
         """
+        train_args = cls._get_train_args(estimator, inputs, experiment_config)
+        estimator.sagemaker_session.train(**train_args)
+
+        return cls(estimator.sagemaker_session, estimator._current_job_name)
+
+    @classmethod
+    def _get_train_args(cls, estimator, inputs, experiment_config):
+        """Constructs a dict of arguments for an Amazon SageMaker training job from the estimator.
+
+        Args:
+            estimator (sagemaker.estimator.EstimatorBase): Estimator object
+                created by the user.
+            inputs (str): Parameters used when called
+                :meth:`~sagemaker.estimator.EstimatorBase.fit`.
+            experiment_config (dict[str, str]): Experiment management configuration used when called
+                :meth:`~sagemaker.estimator.EstimatorBase.fit`.  Dictionary contains
+                three optional keys, 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
+
+        Returns:
+            Dict: dict for `sagemaker.session.Session.train` method
+        """
 
         local_mode = estimator.sagemaker_session.local_mode
         model_uri = estimator.model_uri
@@ -1102,9 +1122,7 @@ def start_new(cls, estimator, inputs, experiment_config):
         if estimator.enable_sagemaker_metrics is not None:
             train_args["enable_sagemaker_metrics"] = estimator.enable_sagemaker_metrics
 
-        estimator.sagemaker_session.train(**train_args)
-
-        return cls(estimator.sagemaker_session, estimator._current_job_name)
+        return train_args
 
     @classmethod
     def _add_spot_checkpoint_args(cls, local_mode, estimator, train_args):

src/sagemaker/image_uri_config/debugger.json

@@ -3,6 +3,7 @@
     "versions": {
         "latest": {
             "registries": {
+                "af-south-1": "314341159256",
                 "ap-east-1": "199566480951",
                 "ap-northeast-1": "430734990657",
                 "ap-northeast-2": "578805364391",
@@ -14,6 +15,7 @@
                 "cn-northwest-1": "658757709296",
                 "eu-central-1": "482524230118",
                 "eu-north-1": "314864569078",
+                "eu-south-1": "563282790590",
                 "eu-west-1": "929884845733",
                 "eu-west-2": "250201462417",
                 "eu-west-3": "447278800020",