Skip to content

Commit 4610af3

Browse files
icywang86ruiicywang86rui
authored
Add release build (#191)
* Run SM tests in parallel in PR build * Add release build spec and build scripts
1 parent fb1fbdf commit 4610af3

File tree

8 files changed

+237
-12
lines changed

8 files changed

+237
-12
lines changed

‎MANIFEST.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
include VERSION
12
include LICENSE
23
include README.md

‎VERSION

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
2.0.1.dev0

‎buildspec-release.yml

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
version: 0.2
2+
3+
env:
4+
variables:
5+
FRAMEWORK_VERSION: '1.13.1'
6+
GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
7+
SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
8+
9+
phases:
10+
pre_build:
11+
commands:
12+
- start-dockerd
13+
- ACCOUNT=$(aws sts get-caller-identity --query 'Account' --output text)
14+
15+
build:
16+
commands:
17+
# prepare the release (update versions, changelog etc.)
18+
- git-release --prepare
19+
20+
# run linter
21+
- tox -e flake8
22+
23+
# run unit tests
24+
- tox -e py36,py27 test/unit
25+
26+
# Create pip archive
27+
- build_dir="docker/$FRAMEWORK_VERSION"
28+
- python3 setup.py sdist
29+
- tar_name=$(ls dist)
30+
- cp dist/$tar_name $build_dir
31+
32+
# Build all images
33+
- python3 scripts/build_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
34+
35+
# run local cpu integ tests
36+
- $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
37+
- tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --framework-version $FRAMEWORK_VERSION --processor cpu
38+
- tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version $FRAMEWORK_VERSION --processor cpu
39+
40+
# launch remote gpu instance
41+
- prefix='ml.'
42+
- instance_type=${GPU_INSTANCE_TYPE#"$prefix"}
43+
- create-key-pair
44+
- launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
45+
46+
- printf "$SETUP_CMDS" > $SETUP_FILE
47+
- py2_cmd="tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version $FRAMEWORK_VERSION --processor gpu"
48+
- remote-test --github-repo $GITHUB_REPO --test-cmd "$py2_cmd" --setup-file $SETUP_FILE
49+
- py3_cmd="tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --framework-version $FRAMEWORK_VERSION --processor gpu"
50+
- remote-test --github-repo $GITHUB_REPO --test-cmd "$py3_cmd" --setup-file $SETUP_FILE
51+
52+
# Publish all images
53+
- python3 scripts/publish_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
54+
55+
- |
56+
echo '[{
57+
"repository": "sagemaker-tensorflow-scriptmode",
58+
"tags": [{
59+
"source": "1.13.1-cpu-py2",
60+
"dest": ["1.13.1-cpu-py2", "1.13-cpu-py2", "1.13.1-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
61+
},{
62+
"source": "1.13.1-cpu-py3",
63+
"dest": ["1.13.1-cpu-py3", "1.13-cpu-py3", "1.13.1-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
64+
},{
65+
"source": "1.13.1-gpu-py2",
66+
"dest": ["1.13.1-gpu-py2", "1.13-gpu-py2", "1.13.1-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
67+
},{
68+
"source": "1.13.1-gpu-py3",
69+
"dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
70+
}],
71+
"test": [
72+
"tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu",
73+
"tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor cpu",
74+
"tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu",
75+
"tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor gpu"
76+
]
77+
}]' > deployments.json
78+
79+
# publish the release to github
80+
- git-release --publish
81+
82+
finally:
83+
# shut down remote gpu instance
84+
- cleanup-gpu-instances
85+
- cleanup-key-pairs
86+
87+
artifacts:
88+
files:
89+
- deployments.json
90+
name: ARTIFACT_1

‎buildspec.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ version: 0.2
22

33
env:
44
variables:
5-
FRAMEWORK_VERSION: '1.12.0'
6-
CPU_FRAMEWORK_BINARY: 'https://s3.amazonaws.com/inferno-dlami/tensorflow/1.12.0/with_intel_patch_mkl_on_off/tensorflow-1.12.0-cp27-cp27mu-linux_x86_64.whl'
7-
CPU_PY_VERSION: '2'
5+
FRAMEWORK_VERSION: '1.13.1'
6+
CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
7+
CPU_PY_VERSION: '3'
88
CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
9-
GPU_FRAMEWORK_BINARY: 'https://s3.amazonaws.com/inferno-dlami/tensorflow/1.12.0/gpu/tensorflow-1.12.0-cp36-cp36m-linux_x86_64.whl'
9+
GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
1010
GPU_PY_VERSION: '3'
1111
GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
1212
ECR_REPO: 'sagemaker-test'
@@ -49,7 +49,7 @@ phases:
4949
- CPU_TAG="$FRAMEWORK_VERSION-cpu-py$CPU_PY_VERSION-$build_id"
5050

5151
- cd $build_dir
52-
- docker build -f $cpu_dockerfile --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG .
52+
- docker build -f $cpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG .
5353
- cd ../../
5454

5555
# build gpu image
@@ -62,7 +62,7 @@ phases:
6262
- GPU_TAG="$FRAMEWORK_VERSION-gpu-py$GPU_PY_VERSION-$build_id"
6363

6464
- cd $build_dir
65-
- docker build -f $gpu_dockerfile --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG .
65+
- docker build -f $gpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG .
6666
- cd ../../
6767

6868
# push images to ecr
@@ -97,15 +97,15 @@ phases:
9797
# run cpu sagemaker tests
9898
- |
9999
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
100-
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --instance-type $CPU_INSTANCE_TYPE
100+
pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --instance-type $CPU_INSTANCE_TYPE
101101
else
102102
echo "skipping cpu sagemaker tests"
103103
fi
104104
105105
# run gpu sagemaker tests
106106
- |
107107
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
108-
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --instance-type $GPU_INSTANCE_TYPE
108+
pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --instance-type $GPU_INSTANCE_TYPE
109109
else
110110
echo "skipping gpu sagemaker tests"
111111
fi

‎scripts/build_all.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from __future__ import absolute_import
14+
15+
import argparse
16+
import os
17+
import subprocess
18+
19+
VERSION = '1.13.1'
20+
REPO = 'sagemaker-tensorflow-scriptmode'
21+
PY2_CPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp27-cp27mu-linux_x86_64.whl' # noqa
22+
PY3_CPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' # noqa
23+
PY2_GPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp27-cp27mu-linux_x86_64.whl' # noqa
24+
PY3_GPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' # noqa
25+
DEV_ACCOUNT = '142577830533'
26+
REGION = 'us-west-2'
27+
28+
29+
def _parse_args():
30+
31+
parser = argparse.ArgumentParser()
32+
33+
parser.add_argument('--account', type=str, default=DEV_ACCOUNT)
34+
parser.add_argument('--region', type=str, default=REGION)
35+
parser.add_argument('--version', type=str, default=VERSION)
36+
parser.add_argument('--py2-cpu-binary', type=str, default=PY2_CPU_BINARY)
37+
parser.add_argument('--py3-cpu-binary', type=str, default=PY3_CPU_BINARY)
38+
parser.add_argument('--py2-gpu-binary', type=str, default=PY2_GPU_BINARY)
39+
parser.add_argument('--py3-gpu-binary', type=str, default=PY3_GPU_BINARY)
40+
parser.add_argument('--repo', type=str, default=REPO)
41+
42+
return parser.parse_args()
43+
44+
45+
args = _parse_args()
46+
binaries = {
47+
'py2-cpu': args.py2_cpu_binary,
48+
'py3-cpu': args.py3_cpu_binary,
49+
'py2-gpu': args.py2_gpu_binary,
50+
'py3-gpu': args.py3_gpu_binary
51+
}
52+
build_dir = os.path.join('docker', args.version)
53+
54+
# Run docker-login so we can pull the cached image
55+
login_cmd = subprocess.check_output(
56+
'aws ecr get-login --no-include-email --registry-id {}'.format(args.account).split())
57+
print('Executing docker login command: '.format(login_cmd))
58+
subprocess.check_call(login_cmd.split())
59+
60+
for arch in ['cpu', 'gpu']:
61+
for py_version in ['2', '3']:
62+
binary_url = binaries['py{}-{}'.format(py_version, arch)]
63+
binary_file = os.path.basename(binary_url)
64+
cmd = 'wget -O {}/{} {}'.format(build_dir, binary_file, binary_url)
65+
print('Downloading binary file {}: '.format(cmd))
66+
subprocess.check_call(cmd.split())
67+
tag = '{}-{}-py{}'.format(args.version, arch, py_version)
68+
prev_image_uri = '{}.dkr.ecr.{}.amazonaws.com/{}:{}'.format(args.account, args.region, args.repo, tag)
69+
dockerfile = os.path.join(build_dir, 'Dockerfile.{}'.format(arch))
70+
build_cmd = 'docker build -f {} --cache-from {} ' \
71+
'--build-arg py_version={} --build-arg framework_installable={} ' \
72+
'-t {}:{} {}'.format(dockerfile, prev_image_uri, py_version, binary_file, args.repo, tag, build_dir)
73+
print('Building docker image: {}'.format(build_cmd))
74+
subprocess.check_call(build_cmd.split())
75+
print('Deleting binary file {}'.format(binary_file))
76+
subprocess.check_call('rm {}'.format(os.path.join(build_dir, binary_file)).split())

‎scripts/publish_all.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from __future__ import absolute_import
14+
15+
import argparse
16+
import subprocess
17+
18+
DEV_ACCOUNT = '142577830533'
19+
VERSION = '1.13.1'
20+
REGION = 'us-west-2'
21+
REPO = 'sagemaker-tensorflow-scriptmode'
22+
23+
24+
def _parse_args():
25+
26+
parser = argparse.ArgumentParser()
27+
28+
parser.add_argument('--account', type=str, default=DEV_ACCOUNT)
29+
parser.add_argument('--version', type=str, default=VERSION)
30+
parser.add_argument('--repo', type=str, default=REPO)
31+
parser.add_argument('--region', type=str, default=REGION)
32+
33+
return parser.parse_args()
34+
35+
36+
args = _parse_args()
37+
38+
for arch in ['cpu', 'gpu']:
39+
for py_version in ['2', '3']:
40+
source = '{}:{}-{}-py{}'.format(args.repo, args.version, arch, py_version)
41+
dest = '{}.dkr.ecr.{}.amazonaws.com/{}'.format(args.account, args.region, source)
42+
tag_cmd = 'docker tag {} {}'.format(source, dest)
43+
print('Tagging image: {}'.format(tag_cmd))
44+
subprocess.check_call(tag_cmd.split())
45+
login_cmd = subprocess.check_output(
46+
'aws ecr get-login --no-include-email --registry-id {} --region {}'
47+
.format(args.account, args.region).split())
48+
print('Executing docker login command: {}'.format(login_cmd))
49+
subprocess.check_call(login_cmd.split())
50+
push_cmd = 'docker push {}'.format(dest)
51+
print('Pushing image: {}'.format(push_cmd))
52+
subprocess.check_call(push_cmd.split())

‎setup.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,13 @@ def read(fname):
2424
return open(os.path.join(os.path.dirname(__file__), fname)).read()
2525

2626

27+
def read_version():
28+
return read('VERSION').strip()
29+
30+
2731
setup(
2832
name='sagemaker_tensorflow_container',
29-
version='2.0.0',
33+
version=read_version(),
3034
description='Open source library for creating '
3135
'TensorFlow containers to run on Amazon SageMaker.',
3236

‎tox.ini

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,18 @@ require-code = True
5050
# TEAMCITY_VERSION environment variable exists during build on Teamcity. teamcity-messages uses it in order to enable
5151
# reporting to TeamCity.
5252
passenv =
53-
TEAMCITY_VERSION
5453
AWS_ACCESS_KEY_ID
5554
AWS_SECRET_ACCESS_KEY
5655
AWS_SESSION_TOKEN
56+
AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
57+
AWS_DEFAULT_REGION
5758
# {posargs} can be passed in by additional arguments specified when invoking tox.
5859
# Can be used to specify which tests to run, e.g.: tox -- -s
5960
commands =
6061
coverage run --rcfile .coveragerc_{envname} --source sagemaker_tensorflow_container -m py.test {posargs}
6162
{env:IGNORE_COVERAGE:} coverage report --include *sagemaker_tensorflow_container* --show-missing
62-
deps = .[test]
63-
sagemaker-containers
63+
deps = sagemaker-containers
64+
extras = test
6465

6566
[testenv:flake8]
6667
basepython = python

0 commit comments

Comments
 (0)