Skip to content

Commit b1a2c23

Browse files
chuyang-dengChuyang Deng
chuyang-deng
and
Chuyang Deng
authored
feature: support PyTorch 1.6 training (#1799)
* feature: support PyTorch 1.6 training * use 'pytorch_inference_latest_version' for tests that deploys pytorch model * resolve flake8 errors * replace all pytorch deploy to use latest inference version Co-authored-by: Chuyang Deng <[email protected]>
1 parent 3980a01 commit b1a2c23

File tree

5 files changed

+164
-62
lines changed

5 files changed

+164
-62
lines changed

‎src/sagemaker/image_uri_config/pytorch.json

Lines changed: 126 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,49 @@
11
{
2-
"training": {
3-
"processors": ["cpu", "gpu"],
2+
"eia": {
3+
"processors": [
4+
"cpu"
5+
],
6+
"version_aliases": {
7+
"1.3": "1.3.1"
8+
},
9+
"versions": {
10+
"1.3.1": {
11+
"py_versions": [
12+
"py3"
13+
],
14+
"registries": {
15+
"ap-east-1": "871362719292",
16+
"ap-northeast-1": "763104351884",
17+
"ap-northeast-2": "763104351884",
18+
"ap-south-1": "763104351884",
19+
"ap-southeast-1": "763104351884",
20+
"ap-southeast-2": "763104351884",
21+
"ca-central-1": "763104351884",
22+
"cn-north-1": "727897471807",
23+
"cn-northwest-1": "727897471807",
24+
"eu-central-1": "763104351884",
25+
"eu-north-1": "763104351884",
26+
"eu-west-1": "763104351884",
27+
"eu-west-2": "763104351884",
28+
"eu-west-3": "763104351884",
29+
"me-south-1": "217643126080",
30+
"sa-east-1": "763104351884",
31+
"us-east-1": "763104351884",
32+
"us-east-2": "763104351884",
33+
"us-gov-west-1": "442386744353",
34+
"us-iso-east-1": "886529160074",
35+
"us-west-1": "763104351884",
36+
"us-west-2": "763104351884"
37+
},
38+
"repository": "pytorch-inference-eia"
39+
}
40+
}
41+
},
42+
"inference": {
43+
"processors": [
44+
"cpu",
45+
"gpu"
46+
],
447
"version_aliases": {
548
"0.4": "0.4.0",
649
"1.0": "1.0.0",
@@ -12,6 +55,10 @@
1255
},
1356
"versions": {
1457
"0.4.0": {
58+
"py_versions": [
59+
"py2",
60+
"py3"
61+
],
1562
"registries": {
1663
"ap-east-1": "057415533634",
1764
"ap-northeast-1": "520713654638",
@@ -36,10 +83,13 @@
3683
"us-west-1": "520713654638",
3784
"us-west-2": "520713654638"
3885
},
39-
"repository": "sagemaker-pytorch",
40-
"py_versions": ["py2", "py3"]
86+
"repository": "sagemaker-pytorch"
4187
},
4288
"1.0.0": {
89+
"py_versions": [
90+
"py2",
91+
"py3"
92+
],
4393
"registries": {
4494
"ap-east-1": "057415533634",
4595
"ap-northeast-1": "520713654638",
@@ -64,10 +114,13 @@
64114
"us-west-1": "520713654638",
65115
"us-west-2": "520713654638"
66116
},
67-
"repository": "sagemaker-pytorch",
68-
"py_versions": ["py2", "py3"]
117+
"repository": "sagemaker-pytorch"
69118
},
70119
"1.1.0": {
120+
"py_versions": [
121+
"py2",
122+
"py3"
123+
],
71124
"registries": {
72125
"ap-east-1": "057415533634",
73126
"ap-northeast-1": "520713654638",
@@ -92,10 +145,13 @@
92145
"us-west-1": "520713654638",
93146
"us-west-2": "520713654638"
94147
},
95-
"repository": "sagemaker-pytorch",
96-
"py_versions": ["py2", "py3"]
148+
"repository": "sagemaker-pytorch"
97149
},
98150
"1.2.0": {
151+
"py_versions": [
152+
"py2",
153+
"py3"
154+
],
99155
"registries": {
100156
"ap-east-1": "871362719292",
101157
"ap-northeast-1": "763104351884",
@@ -120,10 +176,13 @@
120176
"us-west-1": "763104351884",
121177
"us-west-2": "763104351884"
122178
},
123-
"repository": "pytorch-training",
124-
"py_versions": ["py2", "py3"]
179+
"repository": "pytorch-inference"
125180
},
126181
"1.3.1": {
182+
"py_versions": [
183+
"py2",
184+
"py3"
185+
],
127186
"registries": {
128187
"ap-east-1": "871362719292",
129188
"ap-northeast-1": "763104351884",
@@ -148,10 +207,12 @@
148207
"us-west-1": "763104351884",
149208
"us-west-2": "763104351884"
150209
},
151-
"repository": "pytorch-training",
152-
"py_versions": ["py2", "py3"]
210+
"repository": "pytorch-inference"
153211
},
154212
"1.4.0": {
213+
"py_versions": [
214+
"py3"
215+
],
155216
"registries": {
156217
"ap-east-1": "871362719292",
157218
"ap-northeast-1": "763104351884",
@@ -176,10 +237,12 @@
176237
"us-west-1": "763104351884",
177238
"us-west-2": "763104351884"
178239
},
179-
"repository": "pytorch-training",
180-
"py_versions": ["py2", "py3"]
240+
"repository": "pytorch-inference"
181241
},
182242
"1.5.0": {
243+
"py_versions": [
244+
"py3"
245+
],
183246
"registries": {
184247
"ap-east-1": "871362719292",
185248
"ap-northeast-1": "763104351884",
@@ -204,24 +267,31 @@
204267
"us-west-1": "763104351884",
205268
"us-west-2": "763104351884"
206269
},
207-
"repository": "pytorch-training",
208-
"py_versions": ["py3"]
270+
"repository": "pytorch-inference"
209271
}
210272
}
211273
},
212-
"inference": {
213-
"processors": ["cpu", "gpu"],
274+
"training": {
275+
"processors": [
276+
"cpu",
277+
"gpu"
278+
],
214279
"version_aliases": {
215280
"0.4": "0.4.0",
216281
"1.0": "1.0.0",
217282
"1.1": "1.1.0",
218283
"1.2": "1.2.0",
219284
"1.3": "1.3.1",
220285
"1.4": "1.4.0",
221-
"1.5": "1.5.0"
286+
"1.5": "1.5.0",
287+
"1.6": "1.6.0"
222288
},
223289
"versions": {
224290
"0.4.0": {
291+
"py_versions": [
292+
"py2",
293+
"py3"
294+
],
225295
"registries": {
226296
"ap-east-1": "057415533634",
227297
"ap-northeast-1": "520713654638",
@@ -246,10 +316,13 @@
246316
"us-west-1": "520713654638",
247317
"us-west-2": "520713654638"
248318
},
249-
"repository": "sagemaker-pytorch",
250-
"py_versions": ["py2", "py3"]
319+
"repository": "sagemaker-pytorch"
251320
},
252321
"1.0.0": {
322+
"py_versions": [
323+
"py2",
324+
"py3"
325+
],
253326
"registries": {
254327
"ap-east-1": "057415533634",
255328
"ap-northeast-1": "520713654638",
@@ -274,10 +347,13 @@
274347
"us-west-1": "520713654638",
275348
"us-west-2": "520713654638"
276349
},
277-
"repository": "sagemaker-pytorch",
278-
"py_versions": ["py2", "py3"]
350+
"repository": "sagemaker-pytorch"
279351
},
280352
"1.1.0": {
353+
"py_versions": [
354+
"py2",
355+
"py3"
356+
],
281357
"registries": {
282358
"ap-east-1": "057415533634",
283359
"ap-northeast-1": "520713654638",
@@ -302,10 +378,13 @@
302378
"us-west-1": "520713654638",
303379
"us-west-2": "520713654638"
304380
},
305-
"repository": "sagemaker-pytorch",
306-
"py_versions": ["py2", "py3"]
381+
"repository": "sagemaker-pytorch"
307382
},
308383
"1.2.0": {
384+
"py_versions": [
385+
"py2",
386+
"py3"
387+
],
309388
"registries": {
310389
"ap-east-1": "871362719292",
311390
"ap-northeast-1": "763104351884",
@@ -330,10 +409,13 @@
330409
"us-west-1": "763104351884",
331410
"us-west-2": "763104351884"
332411
},
333-
"repository": "pytorch-inference",
334-
"py_versions": ["py2", "py3"]
412+
"repository": "pytorch-training"
335413
},
336414
"1.3.1": {
415+
"py_versions": [
416+
"py2",
417+
"py3"
418+
],
337419
"registries": {
338420
"ap-east-1": "871362719292",
339421
"ap-northeast-1": "763104351884",
@@ -358,10 +440,13 @@
358440
"us-west-1": "763104351884",
359441
"us-west-2": "763104351884"
360442
},
361-
"repository": "pytorch-inference",
362-
"py_versions": ["py2", "py3"]
443+
"repository": "pytorch-training"
363444
},
364445
"1.4.0": {
446+
"py_versions": [
447+
"py2",
448+
"py3"
449+
],
365450
"registries": {
366451
"ap-east-1": "871362719292",
367452
"ap-northeast-1": "763104351884",
@@ -386,10 +471,12 @@
386471
"us-west-1": "763104351884",
387472
"us-west-2": "763104351884"
388473
},
389-
"repository": "pytorch-inference",
390-
"py_versions": ["py3"]
474+
"repository": "pytorch-training"
391475
},
392476
"1.5.0": {
477+
"py_versions": [
478+
"py3"
479+
],
393480
"registries": {
394481
"ap-east-1": "871362719292",
395482
"ap-northeast-1": "763104351884",
@@ -414,18 +501,12 @@
414501
"us-west-1": "763104351884",
415502
"us-west-2": "763104351884"
416503
},
417-
"repository": "pytorch-inference",
418-
"py_versions": ["py3"]
419-
}
420-
}
421-
},
422-
"eia": {
423-
"processors": ["cpu"],
424-
"version_aliases": {
425-
"1.3": "1.3.1"
426-
},
427-
"versions": {
428-
"1.3.1": {
504+
"repository": "pytorch-training"
505+
},
506+
"1.6.0": {
507+
"py_versions": [
508+
"py3"
509+
],
429510
"registries": {
430511
"ap-east-1": "871362719292",
431512
"ap-northeast-1": "763104351884",
@@ -450,9 +531,8 @@
450531
"us-west-1": "763104351884",
451532
"us-west-2": "763104351884"
452533
},
453-
"repository": "pytorch-inference-eia",
454-
"py_versions": ["py3"]
534+
"repository": "pytorch-training"
455535
}
456536
}
457537
}
458-
}
538+
}

‎tests/integ/test_airflow_config.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -574,15 +574,15 @@ def test_xgboost_airflow_config_uploads_data_source_to_s3(
574574
def test_pytorch_airflow_config_uploads_data_source_to_s3_when_inputs_not_provided(
575575
sagemaker_session,
576576
cpu_instance_type,
577-
pytorch_training_latest_version,
578-
pytorch_training_latest_py_version,
577+
pytorch_inference_latest_version,
578+
pytorch_inference_latest_py_version,
579579
):
580580
with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
581581
estimator = PyTorch(
582582
entry_point=PYTORCH_MNIST_SCRIPT,
583583
role=ROLE,
584-
framework_version=pytorch_training_latest_version,
585-
py_version=pytorch_training_latest_py_version,
584+
framework_version=pytorch_inference_latest_version,
585+
py_version=pytorch_inference_latest_py_version,
586586
instance_count=2,
587587
instance_type=cpu_instance_type,
588588
hyperparameters={"epochs": 6, "backend": "gloo"},

‎tests/integ/test_git.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151

5252
@pytest.mark.local_mode
5353
def test_github(
54-
sagemaker_local_session, pytorch_training_latest_version, pytorch_training_latest_py_version
54+
sagemaker_local_session, pytorch_inference_latest_version, pytorch_inference_latest_py_version
5555
):
5656
script_path = "mnist.py"
5757
git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT}
@@ -60,8 +60,8 @@ def test_github(
6060
entry_point=script_path,
6161
role="SageMakerRole",
6262
source_dir="pytorch",
63-
framework_version=pytorch_training_latest_version,
64-
py_version=pytorch_training_latest_py_version,
63+
framework_version=pytorch_inference_latest_version,
64+
py_version=pytorch_inference_latest_py_version,
6565
instance_count=1,
6666
instance_type="local",
6767
sagemaker_session=sagemaker_local_session,

0 commit comments

Comments
 (0)