Skip to content

Commit 0acecda

Browse files
icywang86ruiicywang86rui
authored
Merge branch 'master' into santon/fix-2165
2 parents 183094e + 9f399ce commit 0acecda

File tree

8 files changed

+738
-36
lines changed

8 files changed

+738
-36
lines changed

‎CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Changelog
22

3+
## v2.45.0 (2021-06-07)
4+
5+
### Features
6+
7+
* Add support for Callback steps in model building pipelines
8+
39
## v2.44.0 (2021-06-01)
410

511
### Features

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.44.1.dev0
1+
2.45.1.dev0

‎src/sagemaker/image_uri_config/huggingface.json

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
"processors": ["gpu"],
44
"version_aliases": {
55
"4.4": "4.4.2",
6-
"4.5": "4.5.0"
6+
"4.5": "4.5.0",
7+
"4.6": "4.6.1"
78
},
89
"versions": {
910
"4.4.2": {
@@ -137,6 +138,103 @@
137138
},
138139
"repository": "huggingface-tensorflow-training"
139140
}
141+
},
142+
"4.6.1": {
143+
"version_aliases": {
144+
"pytorch1.6": "pytorch1.6.0",
145+
"pytorch1.7": "pytorch1.7.1",
146+
"tensorflow2.4": "tensorflow2.4.1"
147+
},
148+
"pytorch1.6.0": {
149+
"py_versions": ["py36"],
150+
"registries": {
151+
"af-south-1": "626614931356",
152+
"ap-east-1": "871362719292",
153+
"ap-northeast-1": "763104351884",
154+
"ap-northeast-2": "763104351884",
155+
"ap-south-1": "763104351884",
156+
"ap-southeast-1": "763104351884",
157+
"ap-southeast-2": "763104351884",
158+
"ca-central-1": "763104351884",
159+
"cn-north-1": "727897471807",
160+
"cn-northwest-1": "727897471807",
161+
"eu-central-1": "763104351884",
162+
"eu-north-1": "763104351884",
163+
"eu-west-1": "763104351884",
164+
"eu-west-2": "763104351884",
165+
"eu-west-3": "763104351884",
166+
"eu-south-1": "692866216735",
167+
"me-south-1": "217643126080",
168+
"sa-east-1": "763104351884",
169+
"us-east-1": "763104351884",
170+
"us-east-2": "763104351884",
171+
"us-gov-west-1": "442386744353",
172+
"us-iso-east-1": "886529160074",
173+
"us-west-1": "763104351884",
174+
"us-west-2": "763104351884"
175+
},
176+
"repository": "huggingface-pytorch-training"
177+
},
178+
"pytorch1.7.1": {
179+
"py_versions": ["py36"],
180+
"registries": {
181+
"af-south-1": "626614931356",
182+
"ap-east-1": "871362719292",
183+
"ap-northeast-1": "763104351884",
184+
"ap-northeast-2": "763104351884",
185+
"ap-south-1": "763104351884",
186+
"ap-southeast-1": "763104351884",
187+
"ap-southeast-2": "763104351884",
188+
"ca-central-1": "763104351884",
189+
"cn-north-1": "727897471807",
190+
"cn-northwest-1": "727897471807",
191+
"eu-central-1": "763104351884",
192+
"eu-north-1": "763104351884",
193+
"eu-west-1": "763104351884",
194+
"eu-west-2": "763104351884",
195+
"eu-west-3": "763104351884",
196+
"eu-south-1": "692866216735",
197+
"me-south-1": "217643126080",
198+
"sa-east-1": "763104351884",
199+
"us-east-1": "763104351884",
200+
"us-east-2": "763104351884",
201+
"us-gov-west-1": "442386744353",
202+
"us-iso-east-1": "886529160074",
203+
"us-west-1": "763104351884",
204+
"us-west-2": "763104351884"
205+
},
206+
"repository": "huggingface-pytorch-training"
207+
},
208+
"tensorflow2.4.1": {
209+
"py_versions": ["py37"],
210+
"registries": {
211+
"af-south-1": "626614931356",
212+
"ap-east-1": "871362719292",
213+
"ap-northeast-1": "763104351884",
214+
"ap-northeast-2": "763104351884",
215+
"ap-south-1": "763104351884",
216+
"ap-southeast-1": "763104351884",
217+
"ap-southeast-2": "763104351884",
218+
"ca-central-1": "763104351884",
219+
"cn-north-1": "727897471807",
220+
"cn-northwest-1": "727897471807",
221+
"eu-central-1": "763104351884",
222+
"eu-north-1": "763104351884",
223+
"eu-south-1": "692866216735",
224+
"eu-west-1": "763104351884",
225+
"eu-west-2": "763104351884",
226+
"eu-west-3": "763104351884",
227+
"me-south-1": "217643126080",
228+
"sa-east-1": "763104351884",
229+
"us-east-1": "763104351884",
230+
"us-east-2": "763104351884",
231+
"us-gov-west-1": "442386744353",
232+
"us-iso-east-1": "886529160074",
233+
"us-west-1": "763104351884",
234+
"us-west-2": "763104351884"
235+
},
236+
"repository": "huggingface-tensorflow-training"
237+
}
140238
}
141239
}
142240
}

‎src/sagemaker/session.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3515,7 +3515,11 @@ def get_caller_identity_arn(self):
35153515
user_profile_desc = self.sagemaker_client.describe_user_profile(
35163516
DomainId=domain_id, UserProfileName=user_profile_name
35173517
)
3518-
return user_profile_desc["UserSettings"]["ExecutionRole"]
3518+
if user_profile_desc.get("UserSettings") is not None:
3519+
return user_profile_desc["UserSettings"]["ExecutionRole"]
3520+
3521+
domain_desc = self.sagemaker_client.describe_domain(DomainId=domain_id)
3522+
return domain_desc["DefaultUserSettings"]["ExecutionRole"]
35193523
except ClientError:
35203524
LOGGER.debug(
35213525
"Couldn't call 'describe_notebook_instance' to get the Role "

‎tests/conftest.py

Lines changed: 31 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -190,36 +190,9 @@ def pytorch_inference_py_version(pytorch_inference_version, request):
190190
return "py3"
191191

192192

193-
def _huggingface_base_fm_version(huggingface_vesion, base_fw):
194-
config = image_uris.config_for_framework("huggingface")
195-
training_config = config.get("training")
196-
original_version = huggingface_vesion
197-
if "version_aliases" in training_config:
198-
huggingface_vesion = training_config.get("version_aliases").get(
199-
huggingface_vesion, huggingface_vesion
200-
)
201-
version_config = training_config.get("versions").get(huggingface_vesion)
202-
for key in list(version_config.keys()):
203-
if key.startswith(base_fw):
204-
base_fw_version = key[len(base_fw) :]
205-
if len(original_version.split(".")) == 2:
206-
base_fw_version = ".".join(base_fw_version.split(".")[:-1])
207-
return base_fw_version
208-
209-
210193
@pytest.fixture(scope="module")
211194
def huggingface_pytorch_version(huggingface_training_version):
212-
return _huggingface_base_fm_version(huggingface_training_version, "pytorch")
213-
214-
215-
@pytest.fixture(scope="module")
216-
def huggingface_pytorch_latest_version(huggingface_training_latest_version):
217-
return _huggingface_base_fm_version(huggingface_training_latest_version, "pytorch")
218-
219-
220-
@pytest.fixture(scope="module")
221-
def huggingface_tensorflow_latest_version(huggingface_training_latest_version):
222-
return _huggingface_base_fm_version(huggingface_training_latest_version, "tensorflow")
195+
return _huggingface_base_fm_version(huggingface_training_version, "pytorch")[0]
223196

224197

225198
@pytest.fixture(scope="module")
@@ -395,6 +368,32 @@ def _generate_all_framework_version_fixtures(metafunc):
395368
)
396369

397370

371+
def _huggingface_base_fm_version(huggingface_vesion, base_fw):
372+
config = image_uris.config_for_framework("huggingface")
373+
training_config = config.get("training")
374+
original_version = huggingface_vesion
375+
if "version_aliases" in training_config:
376+
huggingface_vesion = training_config.get("version_aliases").get(
377+
huggingface_vesion, huggingface_vesion
378+
)
379+
version_config = training_config.get("versions").get(huggingface_vesion)
380+
versions = list()
381+
for key in list(version_config.keys()):
382+
if key.startswith(base_fw):
383+
base_fw_version = key[len(base_fw) :]
384+
if len(original_version.split(".")) == 2:
385+
base_fw_version = ".".join(base_fw_version.split(".")[:-1])
386+
versions.append(base_fw_version)
387+
return versions
388+
389+
390+
def _generate_huggingface_base_fw_latest_versions(metafunc, huggingface_version, base_fw):
391+
versions = _huggingface_base_fm_version(huggingface_version, base_fw)
392+
fixture_name = f"huggingface_{base_fw}_latest_version"
393+
if fixture_name in metafunc.fixturenames:
394+
metafunc.parametrize(fixture_name, versions, scope="session")
395+
396+
398397
def _parametrize_framework_version_fixtures(metafunc, fixture_prefix, config):
399398
fixture_name = "{}_version".format(fixture_prefix)
400399
if fixture_name in metafunc.fixturenames:
@@ -407,6 +406,10 @@ def _parametrize_framework_version_fixtures(metafunc, fixture_prefix, config):
407406
if fixture_name in metafunc.fixturenames:
408407
metafunc.parametrize(fixture_name, (latest_version,), scope="session")
409408

409+
if "huggingface" in fixture_prefix:
410+
_generate_huggingface_base_fw_latest_versions(metafunc, latest_version, "pytorch")
411+
_generate_huggingface_base_fw_latest_versions(metafunc, latest_version, "tensorflow")
412+
410413
fixture_name = "{}_latest_py_version".format(fixture_prefix)
411414
if fixture_name in metafunc.fixturenames:
412415
config = config["versions"]

0 commit comments

Comments
 (0)