Merge branch 'zwei' into remove-serde-parameters

Author: bveeramani

src/sagemaker/image_uri_config/model-monitor.json

@@ -0,0 +1,30 @@
+{
+    "scope": ["monitoring"],
+    "versions": {
+        "": {
+            "registries": {
+                "eu-north-1": "895015795356",
+                "me-south-1": "607024016150",
+                "ap-south-1": "126357580389",
+                "us-east-2": "777275614652",
+                "eu-west-1": "468650794304",
+                "eu-central-1": "048819808253",
+                "sa-east-1": "539772159869",
+                "ap-east-1": "001633400207",
+                "us-east-1": "156813124566",
+                "ap-northeast-2": "709848358524",
+                "eu-west-2": "749857270468",
+                "eu-west-3": "680080141114",
+                "ap-northeast-1": "574779866223",
+                "us-west-2": "159807026194",
+                "us-west-1": "890145073186",
+                "ap-southeast-1": "245545462676",
+                "ap-southeast-2": "563025443158",
+                "ca-central-1": "536280801234",
+                "cn-north-1": "453000072557",
+                "cn-northwest-1": "453252182341"
+            },
+            "repository": "sagemaker-model-monitor-analyzer"
+        }
+    }
+}

src/sagemaker/image_uris.py

@@ -22,7 +22,7 @@
 
 logger = logging.getLogger(__name__)
 
-ECR_URI_TEMPLATE = "{registry}.dkr.{hostname}/{repository}:{tag}"
+ECR_URI_TEMPLATE = "{registry}.dkr.{hostname}/{repository}"
 
 
 def retrieve(
@@ -69,14 +69,17 @@ def retrieve(
     registry = _registry_from_region(region, version_config["registries"])
     hostname = utils._botocore_resolver().construct_endpoint("ecr", region)["hostname"]
 
+    repo = version_config["repository"]
+
     processor = _processor(
         instance_type, config.get("processors") or version_config.get("processors")
     )
     tag = _format_tag(version_config.get("tag_prefix", version), processor, py_version)
 
-    repo = version_config["repository"]
+    if tag:
+        repo += ":{}".format(tag)
 
-    return ECR_URI_TEMPLATE.format(registry=registry, hostname=hostname, repository=repo, tag=tag)
+    return ECR_URI_TEMPLATE.format(registry=registry, hostname=hostname, repository=repo)
 
 
 def _config_for_framework_and_scope(framework, image_scope, accelerator_type=None):

src/sagemaker/model_monitor/model_monitoring.py

@@ -26,38 +26,16 @@
 from six.moves.urllib.parse import urlparse
 from botocore.exceptions import ClientError
 
+from sagemaker import image_uris
 from sagemaker.exceptions import UnexpectedStatusException
 from sagemaker.model_monitor.monitoring_files import Constraints, ConstraintViolations, Statistics
 from sagemaker.network import NetworkConfig
 from sagemaker.processing import Processor, ProcessingInput, ProcessingJob, ProcessingOutput
 from sagemaker.s3 import S3Uploader
 from sagemaker.session import Session
-from sagemaker.utils import name_from_base, retries, get_ecr_image_uri_prefix
-
-_DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS = "{}/sagemaker-model-monitor-analyzer"
-
-_DEFAULT_MONITOR_IMAGE_REGION_ACCOUNT_MAPPING = {
-    "eu-north-1": "895015795356",
-    "me-south-1": "607024016150",
-    "ap-south-1": "126357580389",
-    "us-east-2": "777275614652",
-    "eu-west-1": "468650794304",
-    "eu-central-1": "048819808253",
-    "sa-east-1": "539772159869",
-    "ap-east-1": "001633400207",
-    "us-east-1": "156813124566",
-    "ap-northeast-2": "709848358524",
-    "eu-west-2": "749857270468",
-    "eu-west-3": "680080141114",
-    "ap-northeast-1": "574779866223",
-    "us-west-2": "159807026194",
-    "us-west-1": "890145073186",
-    "ap-southeast-1": "245545462676",
-    "ap-southeast-2": "563025443158",
-    "ca-central-1": "536280801234",
-    "cn-north-1": "453000072557",
-    "cn-northwest-1": "453252182341",
-}
+from sagemaker.utils import name_from_base, retries
+
+DEFAULT_REPOSITORY_NAME = "sagemaker-model-monitor-analyzer"
 
 STATISTICS_JSON_DEFAULT_FILE_NAME = "statistics.json"
 CONSTRAINTS_JSON_DEFAULT_FILE_NAME = "constraints.json"
@@ -89,6 +67,8 @@
 
 _LOGGER = logging.getLogger(__name__)
 
+framework_name = "model-monitor"
+
 
 class ModelMonitor(object):
     """Sets up Amazon SageMaker Monitoring Schedules and baseline suggestions. Use this class when
@@ -1787,9 +1767,7 @@ def _get_default_image_uri(region):
         Returns:
             str: The Default Model Monitoring image uri based on the region.
         """
-        return _DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS.format(
-            get_ecr_image_uri_prefix(_DEFAULT_MONITOR_IMAGE_REGION_ACCOUNT_MAPPING[region], region)
-        )
+        return image_uris.retrieve(framework=framework_name, region=region)
 
 
 class BaseliningJob(ProcessingJob):

src/sagemaker/predictor.py

@@ -20,7 +20,7 @@
 from sagemaker.utils import name_from_base
 
 from sagemaker.model_monitor.model_monitoring import (
-    _DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS,
+    DEFAULT_REPOSITORY_NAME,
     ModelMonitor,
     DefaultModelMonitor,
 )
@@ -348,10 +348,7 @@ def list_monitors(self):
             image_uri = schedule["MonitoringScheduleConfig"]["MonitoringJobDefinition"][
                 "MonitoringAppSpecification"
             ]["ImageUri"]
-            index_after_placeholders = _DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS.rfind("{}")
-            if image_uri.endswith(
-                _DEFAULT_MONITOR_IMAGE_URI_WITH_PLACEHOLDERS[index_after_placeholders + len("{}") :]
-            ):
+            if image_uri.endswith(DEFAULT_REPOSITORY_NAME):
                 monitors.append(
                     DefaultModelMonitor.attach(
                         monitor_schedule_name=schedule_name,

src/sagemaker/rl/estimator.py

@@ -36,6 +36,7 @@
         "0.11.0": {"tensorflow": "1.11", "mxnet": "1.3"},
         "0.11.1": {"tensorflow": "1.12"},
         "0.11": {"tensorflow": "1.12", "mxnet": "1.3"},
+        "1.0.0": {"tensorflow": "1.12"},
     },
     "ray": {
         "0.5.3": {"tensorflow": "1.11"},
@@ -68,7 +69,7 @@ class RLEstimator(Framework):
 
     COACH_LATEST_VERSION_TF = "0.11.1"
     COACH_LATEST_VERSION_MXNET = "0.11.0"
-    RAY_LATEST_VERSION = "0.6.5"
+    RAY_LATEST_VERSION = "0.8.5"
 
     def __init__(
         self,

tests/conftest.py

@@ -22,7 +22,6 @@
 
 from sagemaker import Session, image_uris, utils
 from sagemaker.local import LocalSession
-from sagemaker.rl import RLEstimator
 import tests.integ
 
 DEFAULT_REGION = "us-west-2"
@@ -41,15 +40,20 @@
 
 FRAMEWORKS_FOR_GENERATED_VERSION_FIXTURES = (
     "chainer",
+    "coach_mxnet",
+    "coach_tensorflow",
     "inferentia_mxnet",
     "inferentia_tensorflow",
     "mxnet",
     "neo_mxnet",
     "neo_pytorch",
     "neo_tensorflow",
     "pytorch",
+    "ray_pytorch",
+    "ray_tensorflow",
     "sklearn",
     "tensorflow",
+    "vw",
     "xgboost",
 )
 
@@ -181,46 +185,6 @@ def _tf_py_version(tf_version, request):
     return "py37"
 
 
-@pytest.fixture(scope="module", params=["0.10.1", "0.10.1", "0.11", "0.11.0", "0.11.1"])
-def rl_coach_tf_version(request):
-    return request.param
-
-
-@pytest.fixture(scope="module", params=["0.11", "0.11.0"])
-def rl_coach_mxnet_version(request):
-    return request.param
-
-
-@pytest.fixture(scope="module", params=["0.5", "0.5.3", "0.6", "0.6.5", "0.8.2", "0.8.5"])
-def rl_ray_tf_version(request):
-    return request.param
-
-
-@pytest.fixture(scope="module", params=["0.8.5"])
-def rl_ray_pytorch_version(request):
-    return request.param
-
-
-@pytest.fixture(scope="module", params=["8.7.0"])
-def rl_vw_version(request):
-    return request.param
-
-
-@pytest.fixture(scope="module")
-def rl_coach_mxnet_full_version():
-    return RLEstimator.COACH_LATEST_VERSION_MXNET
-
-
-@pytest.fixture(scope="module")
-def rl_coach_tf_full_version():
-    return RLEstimator.COACH_LATEST_VERSION_TF
-
-
-@pytest.fixture(scope="module")
-def rl_ray_full_version():
-    return RLEstimator.RAY_LATEST_VERSION
-
-
 @pytest.fixture(scope="module")
 def tf_full_version(tensorflow_training_latest_version, tensorflow_inference_latest_version):
     """Fixture for TF tests that test both training and inference.

tests/data/ray_cartpole/train_ray.py

@@ -5,12 +5,12 @@
 from ray.tune.logger import pretty_print
 
 # Based on https://github.com/ray-project/ray/blob/master/doc/source/rllib-training.rst#python-api
-ray.init(log_to_driver=False)
+ray.init(log_to_driver=False, webui_host="127.0.0.1")
 config = ppo.DEFAULT_CONFIG.copy()
 config["num_gpus"] = int(os.environ.get("SM_NUM_GPUS", 0))
 checkpoint_dir = os.environ.get("SM_MODEL_DIR", "/Users/nadzeya/gym")
 config["num_workers"] = 1
-agent = ppo.PPOAgent(config=config, env="CartPole-v0")
+agent = ppo.PPOTrainer(config=config, env="CartPole-v0")
 
 # Can optionally call agent.restore(path) to load a checkpoint.
 

tests/integ/test_rl.py

@@ -24,9 +24,9 @@
 
 
 @pytest.mark.canary_quick
-def test_coach_mxnet(sagemaker_session, rl_coach_mxnet_full_version, cpu_instance_type):
+def test_coach_mxnet(sagemaker_session, coach_mxnet_latest_version, cpu_instance_type):
     estimator = _test_coach(
-        sagemaker_session, RLFramework.MXNET, rl_coach_mxnet_full_version, cpu_instance_type
+        sagemaker_session, RLFramework.MXNET, coach_mxnet_latest_version, cpu_instance_type
     )
     job_name = unique_name_from_base("test-coach-mxnet")
 
@@ -51,9 +51,12 @@ def test_coach_mxnet(sagemaker_session, rl_coach_mxnet_full_version, cpu_instanc
     assert 0 < action[0][1] < 1
 
 
-def test_coach_tf(sagemaker_session, rl_coach_tf_full_version, cpu_instance_type):
+def test_coach_tf(sagemaker_session, coach_tensorflow_latest_version, cpu_instance_type):
     estimator = _test_coach(
-        sagemaker_session, RLFramework.TENSORFLOW, rl_coach_tf_full_version, cpu_instance_type
+        sagemaker_session,
+        RLFramework.TENSORFLOW,
+        coach_tensorflow_latest_version,
+        cpu_instance_type,
     )
     job_name = unique_name_from_base("test-coach-tf")
 
@@ -96,7 +99,7 @@ def _test_coach(sagemaker_session, rl_framework, rl_coach_version, cpu_instance_
 
 
 @pytest.mark.canary_quick
-def test_ray_tf(sagemaker_session, rl_ray_full_version, cpu_instance_type):
+def test_ray_tf(sagemaker_session, ray_tensorflow_latest_version, cpu_instance_type):
     source_dir = os.path.join(DATA_DIR, "ray_cartpole")
     cartpole = "train_ray.py"
 
@@ -105,7 +108,7 @@ def test_ray_tf(sagemaker_session, rl_ray_full_version, cpu_instance_type):
         source_dir=source_dir,
         toolkit=RLToolkit.RAY,
         framework=RLFramework.TENSORFLOW,
-        toolkit_version=rl_ray_full_version,
+        toolkit_version=ray_tensorflow_latest_version,
         sagemaker_session=sagemaker_session,
         role="SageMakerRole",
         instance_type=cpu_instance_type,

tests/unit/sagemaker/image_uris/expected_uris.py

@@ -19,6 +19,7 @@
 }
 DOMAIN = "amazonaws.com"
 IMAGE_URI_FORMAT = "{}.dkr.ecr.{}.{}/{}:{}"
+MONITOR_URI_FORMAT = "{}.dkr.ecr.{}.{}/sagemaker-model-monitor-analyzer"
 REGION = "us-west-2"
 
 
@@ -34,3 +35,8 @@ def framework_uri(repo, fw_version, account, py_version=None, processor="cpu", r
 def algo_uri(algo, account, region, version=1):
     domain = ALTERNATE_DOMAINS.get(region, DOMAIN)
     return IMAGE_URI_FORMAT.format(account, region, domain, algo, version)
+
+
+def monitor_uri(account, region=REGION):
+    domain = ALTERNATE_DOMAINS.get(region, DOMAIN)
+    return MONITOR_URI_FORMAT.format(account, region, domain)

tests/unit/sagemaker/image_uris/test_model_monitor.py

@@ -0,0 +1,48 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+from sagemaker import image_uris
+from tests.unit.sagemaker.image_uris import expected_uris, regions
+
+ACCOUNTS = {
+    "eu-north-1": "895015795356",
+    "me-south-1": "607024016150",
+    "ap-south-1": "126357580389",
+    "us-east-2": "777275614652",
+    "eu-west-1": "468650794304",
+    "eu-central-1": "048819808253",
+    "sa-east-1": "539772159869",
+    "ap-east-1": "001633400207",
+    "us-east-1": "156813124566",
+    "ap-northeast-2": "709848358524",
+    "eu-west-2": "749857270468",
+    "eu-west-3": "680080141114",
+    "ap-northeast-1": "574779866223",
+    "us-west-2": "159807026194",
+    "us-west-1": "890145073186",
+    "ap-southeast-1": "245545462676",
+    "ap-southeast-2": "563025443158",
+    "ca-central-1": "536280801234",
+    "cn-north-1": "453000072557",
+    "cn-northwest-1": "453252182341",
+}
+
+
+def test_model_monitor():
+    for region in regions.regions():
+        if region in ACCOUNTS.keys():
+            uri = image_uris.retrieve("model-monitor", region=region)
+
+            expected = expected_uris.monitor_uri(ACCOUNTS[region], region)
+            assert expected == uri