Merge branch 'aws:master' into master

jayatalr · web-flow · commit 3d4ffa5029a6 · 2021-08-25T13:50:20.000-07:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,24 @@
 # Changelog
 
+## v2.55.0 (2021-08-25)
+
+### Features
+
+ * Add information of Amazon-provided analysis image used by Model Monitor
+
+### Bug Fixes and Other Changes
+
+ * Update Changelog to fix release
+ * Fixing the order of populating container list
+ * pass network isolation config to pipelineModel
+ * Deference symbolic link when create tar file
+ * multiprocess issue in feature_group.py
+ * deprecate tag logic on Association
+
+### Documentation Changes
+
+ * add dataset_definition to processing page
+
 ## v2.54.0 (2021-08-16)
 
 ### Features
@@ -8,7 +27,7 @@
 
 ### Bug Fixes and Other Changes
 
- * issue #2253 where Processing job in Local mode would call Describe…
+ * issue #2253 where Processing job in Local mode would call Describe API
 
 ## v2.53.0 (2021-08-12)
 
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.54.1.dev0
+2.55.1.dev0
diff --git a/doc/api/utility/inputs.rst b/doc/api/utility/inputs.rst
@@ -5,4 +5,8 @@ Inputs
     :members:
     :undoc-members:
     :show-inheritance:
-    :noindex:
+
+.. automodule:: sagemaker.dataset_definition.inputs
+    :members:
+    :undoc-members:
+    :show-inheritance:
diff --git a/src/sagemaker/automl/automl.py b/src/sagemaker/automl/automl.py
@@ -328,6 +328,7 @@ def create_model(
             predictor_cls=predictor_cls,
             name=name,
             vpc_config=vpc_config,
+            enable_network_isolation=enable_network_isolation,
             sagemaker_session=sagemaker_session or self.sagemaker_session,
         )
         return pipeline
diff --git a/src/sagemaker/dataset_definition/inputs.py b/src/sagemaker/dataset_definition/inputs.py
@@ -27,7 +27,7 @@ class RedshiftDatasetDefinition(ApiObject):
 
     With this input, SQL queries will be executed using Redshift to generate datasets to S3.
 
-    Attributes:
+    Parameters:
         cluster_id (str): The Redshift cluster Identifier.
         database (str): The name of the Redshift database used in Redshift query execution.
         db_user (str): The database user name used in Redshift query execution.
@@ -60,7 +60,7 @@ class AthenaDatasetDefinition(ApiObject):
 
     With this input, SQL queries will be executed using Athena to generate datasets to S3.
 
-    Attributes:
+    Parameters:
         catalog (str): The name of the data catalog used in Athena query execution.
         database (str): The name of the database used in the Athena query execution.
         query_string (str): The SQL query statements, to be executed.
@@ -87,7 +87,7 @@ class AthenaDatasetDefinition(ApiObject):
 class DatasetDefinition(ApiObject):
     """DatasetDefinition input.
 
-    Attributes:
+    Parameters:
         data_distribution_type (str): Whether the generated dataset is FullyReplicated or
             ShardedByS3Key (default).
         input_mode (str): Whether to use File or Pipe input mode. In File (default) mode, Amazon
@@ -98,9 +98,8 @@ class DatasetDefinition(ApiObject):
         local_path (str): The local path where you want Amazon SageMaker to download the Dataset
             Definition inputs to run a processing job. LocalPath is an absolute path to the input
             data. This is a required parameter when `AppManaged` is False (default).
-        redshift_dataset_definition
-            (:class:`~sagemaker.dataset_definition.inputs.RedshiftDatasetDefinition`): Redshift
-            dataset definition.
+        redshift_dataset_definition (:class:`~sagemaker.dataset_definition.inputs.RedshiftDatasetDefinition`):
+            Configuration for Redshift Dataset Definition input.
         athena_dataset_definition (:class:`~sagemaker.dataset_definition.inputs.AthenaDatasetDefinition`):
             Configuration for Athena Dataset Definition input.
     """
@@ -126,7 +125,7 @@ class S3Input(ApiObject):
     S3 list operations are not strongly consistent.
     Use ManifestFile if strong consistency is required.
 
-    Attributes:
+    Parameters:
         s3_uri (str): the path to a specific S3 object or a S3 prefix
         local_path (str): the path to a local directory. If not provided, skips data download
             by SageMaker platform.
diff --git a/src/sagemaker/feature_store/feature_group.py b/src/sagemaker/feature_store/feature_group.py
@@ -207,7 +207,8 @@ def _ingest_single_batch(
         for row in data_frame[start_index:end_index].itertuples():
             record = [
                 FeatureValue(
-                    feature_name=data_frame.columns[index - 1], value_as_string=str(row[index])
+                    feature_name=data_frame.columns[index - 1],
+                    value_as_string=str(row[index]),
                 )
                 for index in range(1, len(row))
                 if pd.notna(row[index])
@@ -270,13 +271,24 @@ def _run_multi_process(self, data_frame: DataFrame, wait=True, timeout=None):
             timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
                 if timeout is reached.
         """
+        # pylint: disable=I1101
         batch_size = math.ceil(data_frame.shape[0] / self.max_processes)
+        # pylint: enable=I1101
 
         args = []
         for i in range(self.max_processes):
             start_index = min(i * batch_size, data_frame.shape[0])
             end_index = min(i * batch_size + batch_size, data_frame.shape[0])
-            args += [(data_frame[start_index:end_index], start_index, timeout)]
+            args += [
+                (
+                    self.max_workers,
+                    self.feature_group_name,
+                    self.sagemaker_fs_runtime_client_config,
+                    data_frame[start_index:end_index],
+                    start_index,
+                    timeout,
+                )
+            ]
 
         def init_worker():
             # ignore keyboard interrupts in child processes.
@@ -285,13 +297,21 @@ def init_worker():
         self._processing_pool = ProcessingPool(self.max_processes, init_worker)
         self._processing_pool.restart(force=True)
 
-        f = lambda x: self._run_multi_threaded(*x)  # noqa: E731
+        f = lambda x: IngestionManagerPandas._run_multi_threaded(*x)  # noqa: E731
         self._async_result = self._processing_pool.amap(f, args)
 
         if wait:
             self.wait(timeout=timeout)
 
-    def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None) -> List[int]:
+    @staticmethod
+    def _run_multi_threaded(
+        max_workers: int,
+        feature_group_name: str,
+        sagemaker_fs_runtime_client_config: Config,
+        data_frame: DataFrame,
+        row_offset=0,
+        timeout=None,
+    ) -> List[int]:
         """Start the ingestion process.
 
         Args:
@@ -305,21 +325,23 @@ def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None)
         Returns:
             List of row indices that failed to be ingested.
         """
-        executor = ThreadPoolExecutor(max_workers=self.max_workers)
-        batch_size = math.ceil(data_frame.shape[0] / self.max_workers)
+        executor = ThreadPoolExecutor(max_workers=max_workers)
+        # pylint: disable=I1101
+        batch_size = math.ceil(data_frame.shape[0] / max_workers)
+        # pylint: enable=I1101
 
         futures = {}
-        for i in range(self.max_workers):
+        for i in range(max_workers):
             start_index = min(i * batch_size, data_frame.shape[0])
             end_index = min(i * batch_size + batch_size, data_frame.shape[0])
             futures[
                 executor.submit(
-                    self._ingest_single_batch,
-                    feature_group_name=self.feature_group_name,
+                    IngestionManagerPandas._ingest_single_batch,
+                    feature_group_name=feature_group_name,
                     data_frame=data_frame,
                     start_index=start_index,
                     end_index=end_index,
-                    client_config=self.sagemaker_fs_runtime_client_config,
+                    client_config=sagemaker_fs_runtime_client_config,
                 )
             ] = (start_index + row_offset, end_index + row_offset)
 
diff --git a/src/sagemaker/image_uri_config/blazingtext.json b/src/sagemaker/image_uri_config/blazingtext.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "501404015308",
                 "ap-northeast-2": "306986355934",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
diff --git a/src/sagemaker/image_uri_config/factorization-machines.json b/src/sagemaker/image_uri_config/factorization-machines.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/image-classification.json b/src/sagemaker/image_uri_config/image-classification.json
@@ -10,6 +10,7 @@
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
+                "ap-northeast-3": "867004704886",
                 "ca-central-1": "469771592824",
                 "cn-north-1": "390948362332",
                 "cn-northwest-1": "387376663083",
diff --git a/src/sagemaker/image_uri_config/ipinsights.json b/src/sagemaker/image_uri_config/ipinsights.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/kmeans.json b/src/sagemaker/image_uri_config/kmeans.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/knn.json b/src/sagemaker/image_uri_config/knn.json
@@ -3,10 +3,11 @@
     "versions": {
         "1": {
             "registries": {
-               "af-south-1": "455444449433",
+                "af-south-1": "455444449433",
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/linear-learner.json b/src/sagemaker/image_uri_config/linear-learner.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/model-monitor.json b/src/sagemaker/image_uri_config/model-monitor.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "001633400207",
                 "ap-northeast-1": "574779866223",
                 "ap-northeast-2": "709848358524",
+                "ap-northeast-3": "990339680094",
                 "ap-south-1": "126357580389",
                 "ap-southeast-1": "245545462676",
                 "ap-southeast-2": "563025443158",
diff --git a/src/sagemaker/image_uri_config/ntm.json b/src/sagemaker/image_uri_config/ntm.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/object-detection.json b/src/sagemaker/image_uri_config/object-detection.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "501404015308",
                 "ap-northeast-2": "306986355934",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
diff --git a/src/sagemaker/image_uri_config/object2vec.json b/src/sagemaker/image_uri_config/object2vec.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/pca.json b/src/sagemaker/image_uri_config/pca.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/randomcutforest.json b/src/sagemaker/image_uri_config/randomcutforest.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "351501993468",
                 "ap-northeast-2": "835164637446",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "712309505854",
diff --git a/src/sagemaker/image_uri_config/semantic-segmentation.json b/src/sagemaker/image_uri_config/semantic-segmentation.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "501404015308",
                 "ap-northeast-2": "306986355934",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
diff --git a/src/sagemaker/image_uri_config/seq2seq.json b/src/sagemaker/image_uri_config/seq2seq.json
@@ -7,6 +7,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "501404015308",
                 "ap-northeast-2": "306986355934",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
diff --git a/src/sagemaker/image_uri_config/sklearn.json b/src/sagemaker/image_uri_config/sklearn.json
@@ -9,6 +9,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
@@ -39,6 +40,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
diff --git a/src/sagemaker/image_uri_config/xgboost.json b/src/sagemaker/image_uri_config/xgboost.json
@@ -10,6 +10,7 @@
                 "ap-east-1": "286214385809",
                 "ap-northeast-1": "501404015308",
                 "ap-northeast-2": "306986355934",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "991648021394",
                 "ap-southeast-1": "475088953585",
                 "ap-southeast-2": "544295431143",
@@ -41,6 +42,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
@@ -72,6 +74,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
@@ -103,6 +106,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
@@ -132,6 +136,7 @@
                 "ap-east-1": "651117190479",
                 "ap-northeast-1": "354813040037",
                 "ap-northeast-2": "366743142698",
+                "ap-northeast-3": "867004704886",
                 "ap-south-1": "720646828776",
                 "ap-southeast-1": "121021644041",
                 "ap-southeast-2": "783357654285",
diff --git a/src/sagemaker/inputs.py b/src/sagemaker/inputs.py
@@ -127,7 +127,7 @@ def __init__(self, seed):
 class CreateModelInput(object):
     """A class containing parameters which can be used to create a SageMaker Model
 
-    Attributes:
+    Parameters:
         instance_type (str): type or EC2 instance will be used for model deployment.
         accelerator_type (str): elastic inference accelerator type.
     """
diff --git a/src/sagemaker/model.py b/src/sagemaker/model.py
diff --git a/src/sagemaker/utils.py b/src/sagemaker/utils.py
diff --git a/tests/scripts/run-notebook-test.sh b/tests/scripts/run-notebook-test.sh
diff --git a/tests/unit/sagemaker/image_uris/test_algos.py b/tests/unit/sagemaker/image_uris/test_algos.py
diff --git a/tests/unit/sagemaker/image_uris/test_model_monitor.py b/tests/unit/sagemaker/image_uris/test_model_monitor.py
diff --git a/tests/unit/sagemaker/image_uris/test_sklearn.py b/tests/unit/sagemaker/image_uris/test_sklearn.py
diff --git a/tests/unit/sagemaker/image_uris/test_xgboost.py b/tests/unit/sagemaker/image_uris/test_xgboost.py

Original file line number	Diff line number	Diff line change
`@@ -328,6 +328,7 @@ def create_model(`
`328`	`328`	`predictor_cls=predictor_cls,`
`329`	`329`	`name=name,`
`330`	`330`	`vpc_config=vpc_config,`
	`331`	`+ enable_network_isolation=enable_network_isolation,`
`331`	`332`	`sagemaker_session=sagemaker_session or self.sagemaker_session,`
`332`	`333`	`)`
`333`	`334`	`return pipeline`