Skip to content

Commit a1b8ebb

Browse files
mvsuspmvsusp
authored
Merge pull request #2 from aws/mvs-py2
Add PY2 support
2 parents ddf7475 + 05eb3de commit a1b8ebb

File tree

6 files changed

+54
-25
lines changed

6 files changed

+54
-25
lines changed

‎buildspec.yml

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: 0.2
33
env:
44
variables:
55
FRAMEWORK_VERSION: '1.1.0'
6-
CPU_PY_VERSION: '3'
6+
CPU_PY_VERSION: '2'
77
CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
88
GPU_PY_VERSION: '3'
99
GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
@@ -35,34 +35,44 @@ phases:
3535

3636
# build cpu base image
3737
- base_dir="docker/$FRAMEWORK_VERSION/base"
38+
- cpu_py2_base_tag="$FRAMEWORK_VERSION-cpu-py2"
3839
- cpu_py3_base_tag="$FRAMEWORK_VERSION-cpu-py3"
3940
- cpu_dockerfile="Dockerfile.cpu"
4041
- cd $base_dir
41-
- docker build -t $LOCAL_BASE_REPO:$cpu_py3_base_tag -f $cpu_dockerfile .
42+
- docker build -t $LOCAL_BASE_REPO:$cpu_py2_base_tag -f $cpu_dockerfile --build-arg py_version=2 .
43+
- docker build -t $LOCAL_BASE_REPO:$cpu_py3_base_tag -f $cpu_dockerfile --build-arg py_version=3 .
4244
- cd ../../../
4345

4446
# build gpu base image
47+
- gpu_py2_base_tag="$FRAMEWORK_VERSION-gpu-py2"
4548
- gpu_py3_base_tag="$FRAMEWORK_VERSION-gpu-py3"
4649
- gpu_dockerfile="Dockerfile.gpu"
4750
- cd $base_dir
48-
- docker build -t $LOCAL_BASE_REPO:$gpu_py3_base_tag -f $gpu_dockerfile .
51+
- docker build -t $LOCAL_BASE_REPO:$gpu_py2_base_tag -f $gpu_dockerfile --build-arg py_version=2 .
52+
- docker build -t $LOCAL_BASE_REPO:$gpu_py3_base_tag -f $gpu_dockerfile --build-arg py_version=3 .
4953
- cd ../../../
5054

5155
# create wheel
5256
- python3 setup.py bdist_wheel
5357

5458
# build cpu image
5559
- build_dir="docker/$FRAMEWORK_VERSION/final"
60+
- CPU_PY2_TAG="$FRAMEWORK_VERSION-cpu-py2-$BUILD_ID"
5661
- CPU_PY3_TAG="$FRAMEWORK_VERSION-cpu-py3-$BUILD_ID"
57-
- docker build -f "$build_dir/$cpu_dockerfile" -t $PREPROD_IMAGE:$CPU_PY3_TAG .
62+
- docker build -f "$build_dir/$cpu_dockerfile" --build-arg py_version=2 -t $PREPROD_IMAGE:$CPU_PY2_TAG .
63+
- docker build -f "$build_dir/$cpu_dockerfile" --build-arg py_version=3 -t $PREPROD_IMAGE:$CPU_PY3_TAG .
5864

5965
# build gpu image
66+
- GPU_PY2_TAG="$FRAMEWORK_VERSION-gpu-py2-$BUILD_ID"
6067
- GPU_PY3_TAG="$FRAMEWORK_VERSION-gpu-py3-$BUILD_ID"
61-
- docker build -f "$build_dir/$gpu_dockerfile" -t $PREPROD_IMAGE:$GPU_PY3_TAG .
68+
- docker build -f "$build_dir/$gpu_dockerfile" --build-arg py_version=2 -t $PREPROD_IMAGE:$GPU_PY2_TAG .
69+
- docker build -f "$build_dir/$gpu_dockerfile" --build-arg py_version=3 -t $PREPROD_IMAGE:$GPU_PY3_TAG .
6270

6371
# push images to ecr
6472
- $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
73+
- docker push $PREPROD_IMAGE:$CPU_PY2_TAG
6574
- docker push $PREPROD_IMAGE:$CPU_PY3_TAG
75+
- docker push $PREPROD_IMAGE:$GPU_PY2_TAG
6676
- docker push $PREPROD_IMAGE:$GPU_PY3_TAG
6777

6878
# launch remote gpu instance
@@ -74,7 +84,8 @@ phases:
7484
# run cpu integration tests
7585
- |
7686
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
77-
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --processor cpu --tag $CPU_PY3_TAG
87+
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --tag $CPU_PY3_TAG
88+
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --tag $CPU_PY2_TAG
7889
else
7990
echo "skipping cpu integration tests"
8091
fi
@@ -83,7 +94,8 @@ phases:
8394
- |
8495
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
8596
printf "$SETUP_CMDS" > $SETUP_FILE
86-
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GPU_PY3_TAG"
97+
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --tag $GPU_PY3_TAG"
98+
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --tag $GPU_PY2_TAG"
8799
remote-test --github-repo $GITHUB_REPO --test-cmd "$py3_cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM"
88100
else
89101
echo "skipping gpu integration tests"
@@ -92,15 +104,17 @@ phases:
92104
# run cpu sagemaker tests
93105
- |
94106
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
95-
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG
107+
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG
108+
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG
96109
else
97110
echo "skipping cpu sagemaker tests"
98111
fi
99112
100113
# run gpu sagemaker tests
101114
- |
102115
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
103-
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG
116+
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG
117+
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GPU_PY2_TAG
104118
else
105119
echo "skipping gpu sagemaker tests"
106120
fi
@@ -111,5 +125,7 @@ phases:
111125
- cleanup-key-pairs
112126

113127
# remove ecr image
128+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_PY2_TAG
114129
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_PY3_TAG
130+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_PY2_TAG
115131
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_PY3_TAG

‎docker/1.1.0/final/Dockerfile.cpu

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM pytorch-base:1.1.0-cpu-py3
1+
ARG py_version
2+
FROM pytorch-base:1.1.0-cpu-py$py_version
23

34
LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
45

‎docker/1.1.0/final/Dockerfile.gpu

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM pytorch-base:1.1.0-gpu-py3
1+
ARG py_version
2+
FROM pytorch-base:1.1.0-gpu-py$py_version
23

34
LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
45

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ def read(fname):
4949
install_requires=['numpy==1.16.4', 'Pillow==6.0.0', 'retrying==1.3.3', 'sagemaker-containers==2.5.4',
5050
'six==1.12.0', 'torch==1.1.0'],
5151
extras_require={
52-
'test': ['boto3==1.9.169', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.0.2',
52+
'test': ['boto3==1.9.169', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.1.1',
5353
'mock==2.0.0', 'pytest==4.4.0', 'pytest-cov==2.7.1', 'pytest-xdist==1.28.0', 'PyYAML==3.10',
5454
'sagemaker==1.28.1', 'torchvision==0.3.0', 'tox==3.7.0']
5555
},

‎test/conftest.py

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@
3737

3838
dir_path = os.path.dirname(os.path.realpath(__file__))
3939

40-
NO_P2_REGIONS = ['ap-northeast-3', 'ap-southeast-2', 'ca-central-1', 'eu-central-1',
41-
'eu-west-2', 'us-west-1']
42-
NO_P3_REGIONS = ['ap-northeast-3', 'ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
43-
'eu-central-1', 'eu-west-2', 'us-west-1']
40+
NO_P2_REGIONS = ['ap-east-1', 'ap-northeast-3', 'ap-southeast-2', 'ca-central-1', 'eu-central-1', 'eu-north-1',
41+
'eu-west-2', 'eu-west-3', 'us-west-1', 'sa-east-1']
42+
NO_P3_REGIONS = ['ap-east-1', 'ap-northeast-3', 'ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
43+
'eu-central-1', 'eu-north-1', 'eu-west-2', 'eu-west-3', 'sa-east-1', 'us-west-1']
4444

4545

4646
def pytest_addoption(parser):
@@ -51,8 +51,9 @@ def pytest_addoption(parser):
5151
parser.addoption('--docker-base-name', default='pytorch')
5252
parser.addoption('--region', default='us-west-2')
5353
parser.addoption('--framework-version', default=PyTorch.LATEST_VERSION)
54+
parser.addoption('--py-version', choices=['2', '3'], default=str(sys.version_info.major))
5455
parser.addoption('--processor', choices=['gpu', 'cpu'], default='cpu')
55-
# If not specified, will default to {framework-version}-{processor}-py3
56+
# If not specified, will default to {framework-version}-{processor}-py{py-version}
5657
parser.addoption('--tag', default=None)
5758

5859

@@ -71,15 +72,20 @@ def fixture_framework_version(request):
7172
return request.config.getoption('--framework-version')
7273

7374

75+
@pytest.fixture(scope='session', name='py_version')
76+
def fixture_py_version(request):
77+
return 'py{}'.format(int(request.config.getoption('--py-version')))
78+
79+
7480
@pytest.fixture(scope='session', name='processor')
7581
def fixture_processor(request):
7682
return request.config.getoption('--processor')
7783

7884

7985
@pytest.fixture(scope='session', name='tag')
80-
def fixture_tag(request, framework_version, processor):
86+
def fixture_tag(request, framework_version, processor, py_version):
8187
provided_tag = request.config.getoption('--tag')
82-
default_tag = '{}-{}-py3'.format(framework_version, processor)
88+
default_tag = '{}-{}-{}'.format(framework_version, processor, py_version)
8389
return provided_tag if provided_tag else default_tag
8490

8591

@@ -107,11 +113,12 @@ def fixture_use_gpu(processor):
107113

108114

109115
@pytest.fixture(scope='session', name='build_base_image', autouse=True)
110-
def fixture_build_base_image(request, framework_version, processor, tag, docker_base_name):
116+
def fixture_build_base_image(request, framework_version, py_version, processor, tag, docker_base_name):
111117
build_base_image = request.config.getoption('--build-base-image')
112118
if build_base_image:
113119
return image_utils.build_base_image(framework_name=docker_base_name,
114120
framework_version=framework_version,
121+
py_version=py_version,
115122
base_image_tag=tag,
116123
processor=processor,
117124
cwd=os.path.join(dir_path, '..'))
@@ -120,11 +127,12 @@ def fixture_build_base_image(request, framework_version, processor, tag, docker_
120127

121128

122129
@pytest.fixture(scope='session', name='build_image', autouse=True)
123-
def fixture_build_image(request, framework_version, processor, tag, docker_base_name):
130+
def fixture_build_image(request, framework_version, py_version, processor, tag, docker_base_name):
124131
build_image = request.config.getoption('--build-image')
125132
if build_image:
126133
return image_utils.build_image(framework_name=docker_base_name,
127134
framework_version=framework_version,
135+
py_version=py_version,
128136
processor=processor,
129137
tag=tag,
130138
cwd=os.path.join(dir_path, '..'))

‎test/utils/image_utils.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,21 @@
2020
END_COLOR = '\033[0m'
2121

2222

23-
def build_base_image(framework_name, framework_version, processor, base_image_tag, cwd='.'):
23+
def build_base_image(framework_name, framework_version, py_version,
24+
processor, base_image_tag, cwd='.'):
2425
base_image_uri = get_base_image_uri(framework_name, base_image_tag)
2526

2627
dockerfile_location = os.path.join('docker', framework_version, 'base',
2728
'Dockerfile.{}'.format(processor))
2829

2930
subprocess.check_call(['docker', 'build', '-t', base_image_uri,
30-
'-f', dockerfile_location, cwd], cwd=cwd)
31+
'-f', dockerfile_location, '--build-arg',
32+
'py_version={}'.format(py_version[-1]), cwd], cwd=cwd)
3133
print('created image {}'.format(base_image_uri))
3234
return base_image_uri
3335

3436

35-
def build_image(framework_name, framework_version, processor, tag, cwd='.'):
37+
def build_image(framework_name, framework_version, py_version, processor, tag, cwd='.'):
3638
_check_call('python setup.py bdist_wheel')
3739

3840
image_uri = get_image_uri(framework_name, tag)
@@ -41,7 +43,8 @@ def build_image(framework_name, framework_version, processor, tag, cwd='.'):
4143
'Dockerfile.{}'.format(processor))
4244

4345
subprocess.check_call(
44-
['docker', 'build', '-t', image_uri, '-f', dockerfile_location, cwd], cwd=cwd)
46+
['docker', 'build', '-t', image_uri, '-f', dockerfile_location, '--build-arg',
47+
'py_version={}'.format(py_version[-1]), cwd], cwd=cwd)
4548
print('created image {}'.format(image_uri))
4649
return image_uri
4750

0 commit comments

Comments
 (0)