Skip to content

Commit f2c3812

Browse files
laurenyulaurenyu
authored
Merge branch 'zwei' into deprecate-util-functions
2 parents 43f0caa + 70eb3a5 commit f2c3812

File tree

12 files changed

+141
-103
lines changed

12 files changed

+141
-103
lines changed

‎src/sagemaker/debugger.py

Lines changed: 4 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -23,32 +23,9 @@
2323

2424
import smdebug_rulesconfig as rule_configs # noqa: F401 # pylint: disable=unused-import
2525

26-
from sagemaker.utils import get_ecr_image_uri_prefix
27-
28-
RULES_ECR_REPO_NAME = "sagemaker-debugger-rules"
29-
30-
SAGEMAKER_RULE_CONTAINERS_ACCOUNTS_MAP = {
31-
"eu-north-1": {RULES_ECR_REPO_NAME: "314864569078"},
32-
"me-south-1": {RULES_ECR_REPO_NAME: "986000313247"},
33-
"ap-south-1": {RULES_ECR_REPO_NAME: "904829902805"},
34-
"eu-west-3": {RULES_ECR_REPO_NAME: "447278800020"},
35-
"us-east-2": {RULES_ECR_REPO_NAME: "915447279597"},
36-
"eu-west-1": {RULES_ECR_REPO_NAME: "929884845733"},
37-
"eu-central-1": {RULES_ECR_REPO_NAME: "482524230118"},
38-
"sa-east-1": {RULES_ECR_REPO_NAME: "818342061345"},
39-
"ap-east-1": {RULES_ECR_REPO_NAME: "199566480951"},
40-
"us-east-1": {RULES_ECR_REPO_NAME: "503895931360"},
41-
"ap-northeast-2": {RULES_ECR_REPO_NAME: "578805364391"},
42-
"eu-west-2": {RULES_ECR_REPO_NAME: "250201462417"},
43-
"ap-northeast-1": {RULES_ECR_REPO_NAME: "430734990657"},
44-
"us-west-2": {RULES_ECR_REPO_NAME: "895741380848"},
45-
"us-west-1": {RULES_ECR_REPO_NAME: "685455198987"},
46-
"ap-southeast-1": {RULES_ECR_REPO_NAME: "972752614525"},
47-
"ap-southeast-2": {RULES_ECR_REPO_NAME: "184798709955"},
48-
"ca-central-1": {RULES_ECR_REPO_NAME: "519511493484"},
49-
"cn-north-1": {RULES_ECR_REPO_NAME: "618459771430"},
50-
"cn-northwest-1": {RULES_ECR_REPO_NAME: "658757709296"},
51-
}
26+
from sagemaker import image_uris
27+
28+
framework_name = "debugger"
5229

5330

5431
def get_rule_container_image_uri(region):
@@ -61,9 +38,7 @@ def get_rule_container_image_uri(region):
6138
Returns:
6239
str: Formatted image uri for the given region and the rule container type
6340
"""
64-
registry_id = SAGEMAKER_RULE_CONTAINERS_ACCOUNTS_MAP.get(region).get(RULES_ECR_REPO_NAME)
65-
image_uri_prefix = get_ecr_image_uri_prefix(registry_id, region)
66-
return "{}/{}:latest".format(image_uri_prefix, RULES_ECR_REPO_NAME)
41+
return image_uris.retrieve(framework_name, region)
6742

6843

6944
class Rule(object):

‎src/sagemaker/estimator.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,10 @@
2929
from sagemaker.debugger import DebuggerHookConfig
3030
from sagemaker.debugger import TensorBoardOutputConfig # noqa: F401 # pylint: disable=unused-import
3131
from sagemaker.debugger import get_rule_container_image_uri
32-
from sagemaker.s3 import S3Uploader
32+
from sagemaker.s3 import S3Uploader, parse_s3_url
3333

3434
from sagemaker.fw_utils import (
3535
tar_and_upload_dir,
36-
parse_s3_url,
3736
UploadedCode,
3837
validate_source_dir,
3938
_region_supports_debugger,

‎src/sagemaker/fw_utils.py

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
from collections import namedtuple
2222

2323
import sagemaker.utils
24-
from sagemaker import s3
2524

2625
logger = logging.getLogger("sagemaker")
2726

@@ -244,20 +243,6 @@ def framework_version_from_tag(image_tag):
244243
return None if tag_match is None else tag_match.group(1)
245244

246245

247-
def parse_s3_url(url):
248-
"""Calls the method with the same name in the s3 module.
249-
250-
:func:~sagemaker.s3.parse_s3_url
251-
252-
Args:
253-
url: A URL, expected with an s3 scheme.
254-
255-
Returns: The return value of s3.parse_s3_url, which is a tuple containing:
256-
str: S3 bucket name str: S3 key
257-
"""
258-
return s3.parse_s3_url(url)
259-
260-
261246
def model_code_key_prefix(code_location_key_prefix, model_name, image):
262247
"""Returns the s3 key prefix for uploading code during model deployment
263248
The location returned is a potential concatenation of 2 parts

‎src/sagemaker/image_uri_config/debugger.json

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"scope": ["debugger"],
3+
"versions": {
4+
"latest": {
5+
"registries": {
6+
"ap-east-1": "199566480951",
7+
"ap-northeast-1": "430734990657",
8+
"ap-northeast-2": "578805364391",
9+
"ap-south-1": "904829902805",
10+
"ap-southeast-1": "972752614525",
11+
"ap-southeast-2": "184798709955",
12+
"ca-central-1": "519511493484",
13+
"cn-north-1": "618459771430",
14+
"cn-northwest-1": "658757709296",
15+
"eu-central-1": "482524230118",
16+
"eu-north-1": "314864569078",
17+
"eu-west-1": "929884845733",
18+
"eu-west-2": "250201462417",
19+
"eu-west-3": "447278800020",
20+
"me-south-1": "986000313247",
21+
"sa-east-1": "818342061345",
22+
"us-east-1": "199566480951",
23+
"us-east-2": "915447279597",
24+
"us-west-1": "685455198987",
25+
"us-west-2": "895741380848"
26+
},
27+
"repository": "sagemaker-debugger-rules"
28+
}
29+
}
30+
}

‎src/sagemaker/image_uri_config/model-monitor.json

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,26 @@
33
"versions": {
44
"": {
55
"registries": {
6-
"eu-north-1": "895015795356",
7-
"me-south-1": "607024016150",
8-
"ap-south-1": "126357580389",
9-
"us-east-2": "777275614652",
10-
"eu-west-1": "468650794304",
11-
"eu-central-1": "048819808253",
12-
"sa-east-1": "539772159869",
136
"ap-east-1": "001633400207",
14-
"us-east-1": "156813124566",
15-
"ap-northeast-2": "709848358524",
16-
"eu-west-2": "749857270468",
17-
"eu-west-3": "680080141114",
187
"ap-northeast-1": "574779866223",
19-
"us-west-2": "159807026194",
20-
"us-west-1": "890145073186",
8+
"ap-northeast-2": "709848358524",
9+
"ap-south-1": "126357580389",
2110
"ap-southeast-1": "245545462676",
2211
"ap-southeast-2": "563025443158",
2312
"ca-central-1": "536280801234",
2413
"cn-north-1": "453000072557",
25-
"cn-northwest-1": "453252182341"
14+
"cn-northwest-1": "453252182341",
15+
"eu-central-1": "048819808253",
16+
"eu-north-1": "895015795356",
17+
"eu-west-1": "468650794304",
18+
"eu-west-2": "749857270468",
19+
"eu-west-3": "680080141114",
20+
"me-south-1": "607024016150",
21+
"sa-east-1": "539772159869",
22+
"us-east-1": "156813124566",
23+
"us-east-2": "777275614652",
24+
"us-west-1": "890145073186",
25+
"us-west-2": "159807026194"
2626
},
2727
"repository": "sagemaker-model-monitor-analyzer"
2828
}

‎src/sagemaker/model.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
import os
1919

2020
import sagemaker
21-
from sagemaker import fw_utils, image_uris, local, session, utils, git_utils
22-
from sagemaker.fw_utils import UploadedCode
21+
from sagemaker import fw_utils, image_uris, local, s3, session, utils, git_utils
2322
from sagemaker.transformer import Transformer
2423

2524
LOGGER = logging.getLogger("sagemaker")
@@ -715,7 +714,7 @@ def __init__(
715714
self.git_config = git_config
716715
self.container_log_level = container_log_level
717716
if code_location:
718-
self.bucket, self.key_prefix = fw_utils.parse_s3_url(code_location)
717+
self.bucket, self.key_prefix = s3.parse_s3_url(code_location)
719718
else:
720719
self.bucket, self.key_prefix = None, None
721720
if self.git_config:
@@ -788,7 +787,7 @@ def _upload_code(self, key_prefix, repack=False):
788787
)
789788

790789
self.repacked_model_data = repacked_model_data
791-
self.uploaded_code = UploadedCode(
790+
self.uploaded_code = fw_utils.UploadedCode(
792791
s3_prefix=self.repacked_model_data, script_name=os.path.basename(self.entry_point)
793792
)
794793

‎src/sagemaker/workflow/airflow.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import re
1818

1919
import sagemaker
20-
from sagemaker import fw_utils, job, utils, session, vpc_utils
20+
from sagemaker import fw_utils, job, utils, s3, session, vpc_utils
2121
from sagemaker.amazon import amazon_estimator
2222
from sagemaker.tensorflow import TensorFlow
2323

@@ -33,10 +33,10 @@ def prepare_framework(estimator, s3_operations):
3333
`source_dir` ).
3434
"""
3535
if estimator.code_location is not None:
36-
bucket, key = fw_utils.parse_s3_url(estimator.code_location)
36+
bucket, key = s3.parse_s3_url(estimator.code_location)
3737
key = os.path.join(key, estimator._current_job_name, "source", "sourcedir.tar.gz")
3838
elif estimator.uploaded_code is not None:
39-
bucket, key = fw_utils.parse_s3_url(estimator.uploaded_code.s3_prefix)
39+
bucket, key = s3.parse_s3_url(estimator.uploaded_code.s3_prefix)
4040
else:
4141
bucket = estimator.sagemaker_session._default_bucket
4242
key = os.path.join(estimator._current_job_name, "source", "sourcedir.tar.gz")

‎tests/unit/sagemaker/image_uris/test_debugger.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from __future__ import absolute_import
14+
15+
from sagemaker import image_uris
16+
from tests.unit.sagemaker.image_uris import expected_uris, regions
17+
18+
ACCOUNTS = {
19+
"ap-east-1": "199566480951",
20+
"ap-northeast-1": "430734990657",
21+
"ap-northeast-2": "578805364391",
22+
"ap-south-1": "904829902805",
23+
"ap-southeast-1": "972752614525",
24+
"ap-southeast-2": "184798709955",
25+
"ca-central-1": "519511493484",
26+
"cn-north-1": "618459771430",
27+
"cn-northwest-1": "658757709296",
28+
"eu-central-1": "482524230118",
29+
"eu-north-1": "314864569078",
30+
"eu-west-1": "929884845733",
31+
"eu-west-2": "250201462417",
32+
"eu-west-3": "447278800020",
33+
"me-south-1": "986000313247",
34+
"sa-east-1": "818342061345",
35+
"us-east-1": "199566480951",
36+
"us-east-2": "915447279597",
37+
"us-west-1": "685455198987",
38+
"us-west-2": "895741380848",
39+
}
40+
41+
42+
def test_debugger():
43+
for region in regions.regions():
44+
if region in ACCOUNTS.keys():
45+
uri = image_uris.retrieve("debugger", region=region)
46+
47+
expected = expected_uris.algo_uri(
48+
"sagemaker-debugger-rules", ACCOUNTS[region], region, version="latest"
49+
)
50+
assert expected == uri

‎tests/unit/sagemaker/image_uris/test_model_monitor.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,26 +16,26 @@
1616
from tests.unit.sagemaker.image_uris import expected_uris, regions
1717

1818
ACCOUNTS = {
19-
"eu-north-1": "895015795356",
20-
"me-south-1": "607024016150",
21-
"ap-south-1": "126357580389",
22-
"us-east-2": "777275614652",
23-
"eu-west-1": "468650794304",
24-
"eu-central-1": "048819808253",
25-
"sa-east-1": "539772159869",
2619
"ap-east-1": "001633400207",
27-
"us-east-1": "156813124566",
28-
"ap-northeast-2": "709848358524",
29-
"eu-west-2": "749857270468",
30-
"eu-west-3": "680080141114",
3120
"ap-northeast-1": "574779866223",
32-
"us-west-2": "159807026194",
33-
"us-west-1": "890145073186",
21+
"ap-northeast-2": "709848358524",
22+
"ap-south-1": "126357580389",
3423
"ap-southeast-1": "245545462676",
3524
"ap-southeast-2": "563025443158",
3625
"ca-central-1": "536280801234",
3726
"cn-north-1": "453000072557",
3827
"cn-northwest-1": "453252182341",
28+
"eu-central-1": "048819808253",
29+
"eu-north-1": "895015795356",
30+
"eu-west-1": "468650794304",
31+
"eu-west-2": "749857270468",
32+
"eu-west-3": "680080141114",
33+
"me-south-1": "607024016150",
34+
"sa-east-1": "539772159869",
35+
"us-east-1": "156813124566",
36+
"us-east-2": "777275614652",
37+
"us-west-1": "890145073186",
38+
"us-west-2": "159807026194",
3939
}
4040

4141

‎tests/unit/test_airflow.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ def test_byo_training_config_all_args(sagemaker_session):
168168
@patch("os.path.isfile", MagicMock(return_value=True))
169169
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
170170
@patch(
171-
"sagemaker.fw_utils.parse_s3_url",
171+
"sagemaker.s3.parse_s3_url",
172172
MagicMock(
173173
return_value=["output", "tensorflow-training-{}/source/sourcedir.tar.gz".format(TIME_STAMP)]
174174
),
@@ -468,7 +468,7 @@ def test_amazon_alg_training_config_all_args(sagemaker_session):
468468
@patch("os.path.isfile", MagicMock(return_value=True))
469469
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
470470
@patch(
471-
"sagemaker.fw_utils.parse_s3_url",
471+
"sagemaker.s3.parse_s3_url",
472472
MagicMock(
473473
return_value=[
474474
"output",
@@ -610,7 +610,7 @@ def test_framework_tuning_config(retrieve_image_uri, sagemaker_session):
610610
@patch("os.path.isfile", MagicMock(return_value=True))
611611
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
612612
@patch(
613-
"sagemaker.fw_utils.parse_s3_url",
613+
"sagemaker.s3.parse_s3_url",
614614
MagicMock(
615615
return_value=[
616616
"output",
@@ -1020,7 +1020,7 @@ def test_amazon_alg_model_config(sagemaker_session):
10201020
@patch("os.path.isfile", MagicMock(return_value=True))
10211021
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
10221022
@patch(
1023-
"sagemaker.fw_utils.parse_s3_url",
1023+
"sagemaker.s3.parse_s3_url",
10241024
MagicMock(
10251025
return_value=[
10261026
"output",
@@ -1185,7 +1185,7 @@ def test_transform_config(sagemaker_session):
11851185
@patch("os.path.isfile", MagicMock(return_value=True))
11861186
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
11871187
@patch(
1188-
"sagemaker.fw_utils.parse_s3_url",
1188+
"sagemaker.s3.parse_s3_url",
11891189
MagicMock(
11901190
return_value=[
11911191
"output",
@@ -1436,7 +1436,7 @@ def test_deploy_amazon_alg_model_config(sagemaker_session):
14361436
@patch("os.path.isfile", MagicMock(return_value=True))
14371437
@patch("sagemaker.estimator.tar_and_upload_dir", MagicMock())
14381438
@patch(
1439-
"sagemaker.fw_utils.parse_s3_url",
1439+
"sagemaker.s3.parse_s3_url",
14401440
MagicMock(
14411441
return_value=[
14421442
"output",

‎tests/unit/test_fw_utils.py

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -411,18 +411,6 @@ def test_framework_version_from_tag_other():
411411
assert version is None
412412

413413

414-
def test_parse_s3_url():
415-
bucket, key_prefix = fw_utils.parse_s3_url("s3://bucket/code_location")
416-
assert "bucket" == bucket
417-
assert "code_location" == key_prefix
418-
419-
420-
def test_parse_s3_url_fail():
421-
with pytest.raises(ValueError) as error:
422-
fw_utils.parse_s3_url("t3://code_location")
423-
assert "Expecting 's3' scheme" in str(error)
424-
425-
426414
def test_model_code_key_prefix_with_all_values_present():
427415
key_prefix = fw_utils.model_code_key_prefix("prefix", "model_name", "image_uri")
428416
assert key_prefix == "prefix/model_name"

0 commit comments

Comments
 (0)