Merge branch 'master' into master

Author: chuyang-deng

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## v1.69.0 (2020-07-09)
+
+### Features
+
+ * Add ModelClientConfig Fields for Batch Transform
+
+### Documentation Changes
+
+ * add KFP Processing component
+
 ## v1.68.0 (2020-07-07)
 
 ### Features

VERSION

@@ -1 +1 @@
-1.68.1.dev0
+1.69.1.dev0

doc/frameworks/mxnet/using_mxnet.rst

@@ -321,13 +321,14 @@ If there are other packages you want to use with your script, you can include a
 Both ``requirements.txt`` and your training script should be put in the same folder.
 You must specify this folder in ``source_dir`` argument when creating an MXNet estimator.
 
-The function of installing packages using ``requirements.txt`` is supported for all MXNet versions during training.
+The function of installing packages using ``requirements.txt`` is supported for MXNet versions 1.3.0 and higher during training.
+
 When serving an MXNet model, support for this function varies with MXNet versions.
 For MXNet 1.6.0 or newer, ``requirements.txt`` must be under folder ``code``.
 The SageMaker MXNet Estimator automatically saves ``code`` in ``model.tar.gz`` after training (assuming you set up your script and ``requirements.txt`` correctly as stipulated in the previous paragraph).
 In the case of bringing your own trained model for deployment, you must save ``requirements.txt`` under folder ``code`` in ``model.tar.gz`` yourself or specify it through ``dependencies``.
-For MXNet 1.4.1, ``requirements.txt`` is not supported for inference.
-For MXNet 0.12.1-1.3.0, ``requirements.txt`` must be in ``source_dir``.
+For MXNet 0.12.1-1.2.1, 1.4.0-1.4.1, ``requirements.txt`` is not supported for inference.
+For MXNet 1.3.0, ``requirements.txt`` must be in ``source_dir``.
 
 A ``requirements.txt`` file is a text file that contains a list of items that are installed by using ``pip install``.
 You can also specify the version of an item to install.

setup.py

@@ -34,7 +34,7 @@ def read_version():
 
 # Declare minimal set for installation
 required_packages = [
-    "boto3>=1.13.24",
+    "boto3>=1.14.12",
     "numpy>=1.9.0",
     "protobuf>=3.1",
     "scipy>=0.19.0",

src/sagemaker/analytics.py

@@ -431,6 +431,8 @@ def __init__(
         metric_names=None,
         parameter_names=None,
         sagemaker_session=None,
+        input_artifact_names=None,
+        output_artifact_names=None,
     ):
         """Initialize a ``ExperimentAnalytics`` instance.
 
@@ -450,6 +452,11 @@ def __init__(
             sagemaker_session (sagemaker.session.Session): Session object which manages interactions
                 with Amazon SageMaker APIs and any other AWS services needed. If not specified,
                 one is created using the default AWS configuration chain.
+            input_artifact_names(dict optional):The input artifacts for the experiment. Examples of
+                input artifacts are datasets, algorithms, hyperparameters, source code, and instance
+                types.
+            output_artifact_names(dict optional): The output artifacts for the experiment. Examples
+                of output artifacts are metrics, snapshots, logs, and images.
         """
         sagemaker_session = sagemaker_session or Session()
         self._sage_client = sagemaker_session.sagemaker_client
@@ -463,6 +470,8 @@ def __init__(
         self._sort_order = sort_order
         self._metric_names = metric_names
         self._parameter_names = parameter_names
+        self._input_artifact_names = input_artifact_names
+        self._output_artifact_names = output_artifact_names
         self._trial_components = None
         super(ExperimentAnalytics, self).__init__()
         self.clear_cache()
@@ -516,6 +525,21 @@ def _reshape_metrics(self, metrics):
                     out["{} - {}".format(metric_name, stat_type)] = stat_value
         return out
 
+    def _reshape_artifacts(self, artifacts, _artifact_names):
+        """Reshape trial component input/output artifacts to a pandas column
+        Args:
+            artifacts: trial component input/output artifacts
+        Returns:
+            dict: Key: artifacts name, Value: artifacts value
+        """
+        out = OrderedDict()
+        for name, value in sorted(artifacts.items()):
+            if _artifact_names and (name not in _artifact_names):
+                continue
+            out["{} - {}".format(name, "MediaType")] = value.get("MediaType")
+            out["{} - {}".format(name, "Value")] = value.get("Value")
+        return out
+
     def _reshape(self, trial_component):
         """Reshape trial component data to pandas columns
         Args:
@@ -533,6 +557,16 @@ def _reshape(self, trial_component):
 
         out.update(self._reshape_parameters(trial_component.get("Parameters", [])))
         out.update(self._reshape_metrics(trial_component.get("Metrics", [])))
+        out.update(
+            self._reshape_artifacts(
+                trial_component.get("InputArtifacts", []), self._input_artifact_names
+            )
+        )
+        out.update(
+            self._reshape_artifacts(
+                trial_component.get("OutputArtifacts", []), self._output_artifact_names
+            )
+        )
         return out
 
     def _fetch_dataframe(self):

src/sagemaker/model_monitor/model_monitoring.py

@@ -57,6 +57,7 @@
     "ca-central-1": "536280801234",
     "cn-north-1": "453000072557",
     "cn-northwest-1": "453252182341",
+    "us-gov-west-1": "362178532790",
 }
 
 STATISTICS_JSON_DEFAULT_FILE_NAME = "statistics.json"

src/sagemaker/session.py

@@ -1996,6 +1996,7 @@ def transform(
         experiment_config,
         tags,
         data_processing,
+        model_client_config=None,
     ):
         """Create an Amazon SageMaker transform job.
 
@@ -2020,6 +2021,9 @@ def transform(
             data_processing(dict): A dictionary describing config for combining the input data and
                 transformed data. For more, see
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
+            model_client_config (dict): A dictionary describing the model configuration for the
+                job. Dictionary contains two optional keys,
+                'InvocationsTimeoutInSeconds', and 'InvocationsMaxRetries'.
         """
         transform_request = {
             "TransformJobName": job_name,
@@ -2050,6 +2054,9 @@ def transform(
         if experiment_config and len(experiment_config) > 0:
             transform_request["ExperimentConfig"] = experiment_config
 
+        if model_client_config and len(model_client_config) > 0:
+            transform_request["ModelClientConfig"] = model_client_config
+
         LOGGER.info("Creating transform job with name: %s", job_name)
         LOGGER.debug("Transform request: %s", json.dumps(transform_request, indent=4))
         self.sagemaker_client.create_transform_job(**transform_request)

src/sagemaker/tensorflow/defaults.py

@@ -21,7 +21,7 @@
 LATEST_VERSION = "2.2.0"
 """The latest version of TensorFlow included in the SageMaker pre-built Docker images."""
 
-LATEST_SERVING_VERSION = "2.1.0"
+LATEST_SERVING_VERSION = "2.2.0"
 """The latest version of TensorFlow Serving included in the SageMaker pre-built Docker images."""
 
 LATEST_PY2_VERSION = "2.1.0"

src/sagemaker/transformer.py

@@ -120,6 +120,7 @@ def transform(
         output_filter=None,
         join_source=None,
         experiment_config=None,
+        model_client_config=None,
         wait=False,
         logs=False,
     ):
@@ -172,6 +173,10 @@ def transform(
                 Dictionary contains three optional keys,
                 'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
                 (default: ``None``).
+            model_client_config (dict[str, str]): Model configuration.
+                Dictionary contains two optional keys,
+                'InvocationsTimeoutInSeconds', and 'InvocationsMaxRetries'.
+                (default: ``None``).
             wait (bool): Whether the call should wait until the job completes
                 (default: False).
             logs (bool): Whether to show the logs produced by the job.
@@ -208,6 +213,7 @@ def transform(
             output_filter,
             join_source,
             experiment_config,
+            model_client_config,
         )
 
         if wait:
@@ -342,6 +348,7 @@ def start_new(
         output_filter,
         join_source,
         experiment_config,
+        model_client_config,
     ):
         """
         Args:
@@ -355,6 +362,7 @@ def start_new(
             output_filter:
             join_source:
             experiment_config:
+            model_client_config:
         """
         config = _TransformJob._load_config(
             data, data_type, content_type, compression_type, split_type, transformer
@@ -374,6 +382,7 @@ def start_new(
             output_config=config["output_config"],
             resource_config=config["resource_config"],
             experiment_config=experiment_config,
+            model_client_config=model_client_config,
             tags=transformer.tags,
             data_processing=data_processing,
         )

tests/conftest.py

@@ -28,7 +28,7 @@
 from sagemaker.rl import RLEstimator
 from sagemaker.sklearn.defaults import SKLEARN_VERSION
 from sagemaker.tensorflow import TensorFlow
-from sagemaker.tensorflow.defaults import LATEST_VERSION, LATEST_SERVING_VERSION
+from sagemaker.tensorflow.defaults import LATEST_VERSION
 
 DEFAULT_REGION = "us-west-2"
 CUSTOM_BUCKET_NAME_PREFIX = "sagemaker-custom-bucket"
@@ -336,10 +336,3 @@ def pytest_generate_tests(metafunc):
 @pytest.fixture(scope="module")
 def xgboost_full_version(request):
     return request.config.getoption("--xgboost-full-version")
-
-
-@pytest.fixture(scope="module")
-def tf_serving_version(tf_full_version):
-    if tf_full_version == LATEST_VERSION:
-        return LATEST_SERVING_VERSION
-    return tf_full_version

tests/integ/test_data_capture_config.py

@@ -41,7 +41,7 @@
 
 
 def test_enabling_data_capture_on_endpoint_shows_correct_data_capture_status(
-    sagemaker_session, tf_serving_version
+    sagemaker_session, tf_full_version
 ):
     endpoint_name = unique_name_from_base("sagemaker-tensorflow-serving")
     model_data = sagemaker_session.upload_data(
@@ -52,7 +52,7 @@ def test_enabling_data_capture_on_endpoint_shows_correct_data_capture_status(
         model = Model(
             model_data=model_data,
             role=ROLE,
-            framework_version=tf_serving_version,
+            framework_version=tf_full_version,
             sagemaker_session=sagemaker_session,
         )
         predictor = model.deploy(
@@ -98,7 +98,7 @@ def test_enabling_data_capture_on_endpoint_shows_correct_data_capture_status(
 
 
 def test_disabling_data_capture_on_endpoint_shows_correct_data_capture_status(
-    sagemaker_session, tf_serving_version
+    sagemaker_session, tf_full_version
 ):
     endpoint_name = unique_name_from_base("sagemaker-tensorflow-serving")
     model_data = sagemaker_session.upload_data(
@@ -109,7 +109,7 @@ def test_disabling_data_capture_on_endpoint_shows_correct_data_capture_status(
         model = Model(
             model_data=model_data,
             role=ROLE,
-            framework_version=tf_serving_version,
+            framework_version=tf_full_version,
             sagemaker_session=sagemaker_session,
         )
         destination_s3_uri = os.path.join(
@@ -184,7 +184,7 @@ def test_disabling_data_capture_on_endpoint_shows_correct_data_capture_status(
 
 
 def test_updating_data_capture_on_endpoint_shows_correct_data_capture_status(
-    sagemaker_session, tf_serving_version
+    sagemaker_session, tf_full_version
 ):
     endpoint_name = sagemaker.utils.unique_name_from_base("sagemaker-tensorflow-serving")
     model_data = sagemaker_session.upload_data(
@@ -195,7 +195,7 @@ def test_updating_data_capture_on_endpoint_shows_correct_data_capture_status(
         model = Model(
             model_data=model_data,
             role=ROLE,
-            framework_version=tf_serving_version,
+            framework_version=tf_full_version,
             sagemaker_session=sagemaker_session,
         )
         destination_s3_uri = os.path.join(

tests/integ/test_experiments_analytics.py

@@ -43,6 +43,65 @@ def experiment(sagemaker_session):
         _delete_resources(sm, experiment_name, trials)
 
 
+@contextmanager
+def experiment_with_artifacts(sagemaker_session):
+    sm = sagemaker_session.sagemaker_client
+    trials = {}  # for resource cleanup
+
+    experiment_name = "experiment-" + str(uuid.uuid4())
+    try:
+        sm.create_experiment(ExperimentName=experiment_name)
+
+        # Search returns 10 results by default. Add 20 trials to verify pagination.
+        for i in range(20):
+            trial_name = "trial-" + str(uuid.uuid4())
+            sm.create_trial(TrialName=trial_name, ExperimentName=experiment_name)
+
+            trial_component_name = "tc-" + str(uuid.uuid4())
+            trials[trial_name] = trial_component_name
+
+            sm.create_trial_component(
+                TrialComponentName=trial_component_name, DisplayName="Training"
+            )
+            sm.update_trial_component(
+                TrialComponentName=trial_component_name,
+                Parameters={"hp1": {"NumberValue": i}},
+                InputArtifacts={
+                    "inputArtifacts1": {"MediaType": "text/csv", "Value": "s3:/foo/bar1"}
+                },
+                OutputArtifacts={
+                    "outputArtifacts1": {"MediaType": "text/plain", "Value": "s3:/foo/bar2"}
+                },
+            )
+            sm.associate_trial_component(
+                TrialComponentName=trial_component_name, TrialName=trial_name
+            )
+
+        time.sleep(15)  # wait for search to get updated
+
+        yield experiment_name
+    finally:
+        _delete_resources(sm, experiment_name, trials)
+
+
+@pytest.mark.canary_quick
+def test_experiment_analytics_artifacts(sagemaker_session):
+    with experiment_with_artifacts(sagemaker_session) as experiment_name:
+        analytics = ExperimentAnalytics(
+            experiment_name=experiment_name, sagemaker_session=sagemaker_session
+        )
+
+        assert list(analytics.dataframe().columns) == [
+            "TrialComponentName",
+            "DisplayName",
+            "hp1",
+            "inputArtifacts1 - MediaType",
+            "inputArtifacts1 - Value",
+            "outputArtifacts1 - MediaType",
+            "outputArtifacts1 - Value",
+        ]
+
+
 @pytest.mark.canary_quick
 def test_experiment_analytics(sagemaker_session):
     with experiment(sagemaker_session) as experiment_name:

tests/integ/test_model_monitor.py

@@ -88,7 +88,7 @@
 
 
 @pytest.fixture(scope="module")
-def predictor(sagemaker_session, tf_serving_version):
+def predictor(sagemaker_session, tf_full_version):
     endpoint_name = unique_name_from_base("sagemaker-tensorflow-serving")
     model_data = sagemaker_session.upload_data(
         path=os.path.join(tests.integ.DATA_DIR, "tensorflow-serving-test-model.tar.gz"),
@@ -100,7 +100,7 @@ def predictor(sagemaker_session, tf_serving_version):
         model = Model(
             model_data=model_data,
             role=ROLE,
-            framework_version=tf_serving_version,
+            framework_version=tf_full_version,
             sagemaker_session=sagemaker_session,
         )
         predictor = model.deploy(