Skip to content

Add PY2 support #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 23, 2019
Merged

Add PY2 support #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4092d69
Add PY2 support
mvsusp Aug 22, 2019
eb4c566
Add PY2 support
mvsusp Aug 22, 2019
05eb3de
Add PY2 support
mvsusp Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 25 additions & 9 deletions buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ version: 0.2
env:
variables:
FRAMEWORK_VERSION: '1.1.0'
CPU_PY_VERSION: '3'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this removed?

CPU_PY_VERSION: '2'
CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
GPU_PY_VERSION: '3'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GPU_PY_VERSION: '2'?

GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
Expand Down Expand Up @@ -35,34 +35,44 @@ phases:

# build cpu base image
- base_dir="docker/$FRAMEWORK_VERSION/base"
- cpu_py2_base_tag="$FRAMEWORK_VERSION-cpu-py2"
- cpu_py3_base_tag="$FRAMEWORK_VERSION-cpu-py3"
- cpu_dockerfile="Dockerfile.cpu"
- cd $base_dir
- docker build -t $LOCAL_BASE_REPO:$cpu_py3_base_tag -f $cpu_dockerfile .
- docker build -t $LOCAL_BASE_REPO:$cpu_py2_base_tag -f $cpu_dockerfile --build-arg py_version=2 .
- docker build -t $LOCAL_BASE_REPO:$cpu_py3_base_tag -f $cpu_dockerfile --build-arg py_version=3 .
- cd ../../../

# build gpu base image
- gpu_py2_base_tag="$FRAMEWORK_VERSION-gpu-py2"
- gpu_py3_base_tag="$FRAMEWORK_VERSION-gpu-py3"
- gpu_dockerfile="Dockerfile.gpu"
- cd $base_dir
- docker build -t $LOCAL_BASE_REPO:$gpu_py3_base_tag -f $gpu_dockerfile .
- docker build -t $LOCAL_BASE_REPO:$gpu_py2_base_tag -f $gpu_dockerfile --build-arg py_version=2 .
- docker build -t $LOCAL_BASE_REPO:$gpu_py3_base_tag -f $gpu_dockerfile --build-arg py_version=3 .
- cd ../../../

# create wheel
- python3 setup.py bdist_wheel

# build cpu image
- build_dir="docker/$FRAMEWORK_VERSION/final"
- CPU_PY2_TAG="$FRAMEWORK_VERSION-cpu-py2-$BUILD_ID"
- CPU_PY3_TAG="$FRAMEWORK_VERSION-cpu-py3-$BUILD_ID"
- docker build -f "$build_dir/$cpu_dockerfile" -t $PREPROD_IMAGE:$CPU_PY3_TAG .
- docker build -f "$build_dir/$cpu_dockerfile" --build-arg py_version=2 -t $PREPROD_IMAGE:$CPU_PY2_TAG .
- docker build -f "$build_dir/$cpu_dockerfile" --build-arg py_version=3 -t $PREPROD_IMAGE:$CPU_PY3_TAG .

# build gpu image
- GPU_PY2_TAG="$FRAMEWORK_VERSION-gpu-py2-$BUILD_ID"
- GPU_PY3_TAG="$FRAMEWORK_VERSION-gpu-py3-$BUILD_ID"
- docker build -f "$build_dir/$gpu_dockerfile" -t $PREPROD_IMAGE:$GPU_PY3_TAG .
- docker build -f "$build_dir/$gpu_dockerfile" --build-arg py_version=2 -t $PREPROD_IMAGE:$GPU_PY2_TAG .
- docker build -f "$build_dir/$gpu_dockerfile" --build-arg py_version=3 -t $PREPROD_IMAGE:$GPU_PY3_TAG .

# push images to ecr
- $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
- docker push $PREPROD_IMAGE:$CPU_PY2_TAG
- docker push $PREPROD_IMAGE:$CPU_PY3_TAG
- docker push $PREPROD_IMAGE:$GPU_PY2_TAG
- docker push $PREPROD_IMAGE:$GPU_PY3_TAG

# launch remote gpu instance
Expand All @@ -74,7 +84,8 @@ phases:
# run cpu integration tests
- |
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --processor cpu --tag $CPU_PY3_TAG
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --tag $CPU_PY3_TAG
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --tag $CPU_PY2_TAG
else
echo "skipping cpu integration tests"
fi
Expand All @@ -83,7 +94,8 @@ phases:
- |
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
printf "$SETUP_CMDS" > $SETUP_FILE
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GPU_PY3_TAG"
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --tag $GPU_PY3_TAG"
py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --tag $GPU_PY2_TAG"
remote-test --github-repo $GITHUB_REPO --test-cmd "$py3_cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM"
else
echo "skipping gpu integration tests"
Expand All @@ -92,15 +104,17 @@ phases:
# run cpu sagemaker tests
- |
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG
else
echo "skipping cpu sagemaker tests"
fi

# run gpu sagemaker tests
- |
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG
pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GPU_PY2_TAG
else
echo "skipping gpu sagemaker tests"
fi
Expand All @@ -111,5 +125,7 @@ phases:
- cleanup-key-pairs

# remove ecr image
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_PY2_TAG
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_PY3_TAG
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_PY2_TAG
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_PY3_TAG
3 changes: 2 additions & 1 deletion docker/1.1.0/final/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
FROM pytorch-base:1.1.0-cpu-py3
ARG py_version
FROM pytorch-base:1.1.0-cpu-py$py_version

LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true

Expand Down
3 changes: 2 additions & 1 deletion docker/1.1.0/final/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
FROM pytorch-base:1.1.0-gpu-py3
ARG py_version
FROM pytorch-base:1.1.0-gpu-py$py_version

LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true

Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def read(fname):
install_requires=['numpy==1.16.4', 'Pillow==6.0.0', 'retrying==1.3.3', 'sagemaker-containers==2.5.4',
'six==1.12.0', 'torch==1.1.0'],
extras_require={
'test': ['boto3==1.9.169', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.0.2',
'test': ['boto3==1.9.169', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.1.1',
'mock==2.0.0', 'pytest==4.4.0', 'pytest-cov==2.7.1', 'pytest-xdist==1.28.0', 'PyYAML==3.10',
'sagemaker==1.28.1', 'torchvision==0.3.0', 'tox==3.7.0']
},
Expand Down
26 changes: 17 additions & 9 deletions test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@

dir_path = os.path.dirname(os.path.realpath(__file__))

NO_P2_REGIONS = ['ap-northeast-3', 'ap-southeast-2', 'ca-central-1', 'eu-central-1',
'eu-west-2', 'us-west-1']
NO_P3_REGIONS = ['ap-northeast-3', 'ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
'eu-central-1', 'eu-west-2', 'us-west-1']
NO_P2_REGIONS = ['ap-east-1', 'ap-northeast-3', 'ap-southeast-2', 'ca-central-1', 'eu-central-1', 'eu-north-1',
'eu-west-2', 'eu-west-3', 'us-west-1', 'sa-east-1']
NO_P3_REGIONS = ['ap-east-1', 'ap-northeast-3', 'ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
'eu-central-1', 'eu-north-1', 'eu-west-2', 'eu-west-3', 'sa-east-1', 'us-west-1']


def pytest_addoption(parser):
Expand All @@ -51,8 +51,9 @@ def pytest_addoption(parser):
parser.addoption('--docker-base-name', default='pytorch')
parser.addoption('--region', default='us-west-2')
parser.addoption('--framework-version', default=PyTorch.LATEST_VERSION)
parser.addoption('--py-version', choices=['2', '3'], default=str(sys.version_info.major))
parser.addoption('--processor', choices=['gpu', 'cpu'], default='cpu')
# If not specified, will default to {framework-version}-{processor}-py3
# If not specified, will default to {framework-version}-{processor}-py{py-version}
parser.addoption('--tag', default=None)


Expand All @@ -71,15 +72,20 @@ def fixture_framework_version(request):
return request.config.getoption('--framework-version')


@pytest.fixture(scope='session', name='py_version')
def fixture_py_version(request):
return 'py{}'.format(int(request.config.getoption('--py-version')))


@pytest.fixture(scope='session', name='processor')
def fixture_processor(request):
return request.config.getoption('--processor')


@pytest.fixture(scope='session', name='tag')
def fixture_tag(request, framework_version, processor):
def fixture_tag(request, framework_version, processor, py_version):
provided_tag = request.config.getoption('--tag')
default_tag = '{}-{}-py3'.format(framework_version, processor)
default_tag = '{}-{}-{}'.format(framework_version, processor, py_version)
return provided_tag if provided_tag else default_tag


Expand Down Expand Up @@ -107,11 +113,12 @@ def fixture_use_gpu(processor):


@pytest.fixture(scope='session', name='build_base_image', autouse=True)
def fixture_build_base_image(request, framework_version, processor, tag, docker_base_name):
def fixture_build_base_image(request, framework_version, py_version, processor, tag, docker_base_name):
build_base_image = request.config.getoption('--build-base-image')
if build_base_image:
return image_utils.build_base_image(framework_name=docker_base_name,
framework_version=framework_version,
py_version=py_version,
base_image_tag=tag,
processor=processor,
cwd=os.path.join(dir_path, '..'))
Expand All @@ -120,11 +127,12 @@ def fixture_build_base_image(request, framework_version, processor, tag, docker_


@pytest.fixture(scope='session', name='build_image', autouse=True)
def fixture_build_image(request, framework_version, processor, tag, docker_base_name):
def fixture_build_image(request, framework_version, py_version, processor, tag, docker_base_name):
build_image = request.config.getoption('--build-image')
if build_image:
return image_utils.build_image(framework_name=docker_base_name,
framework_version=framework_version,
py_version=py_version,
processor=processor,
tag=tag,
cwd=os.path.join(dir_path, '..'))
Expand Down
11 changes: 7 additions & 4 deletions test/utils/image_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,21 @@
END_COLOR = '\033[0m'


def build_base_image(framework_name, framework_version, processor, base_image_tag, cwd='.'):
def build_base_image(framework_name, framework_version, py_version,
processor, base_image_tag, cwd='.'):
base_image_uri = get_base_image_uri(framework_name, base_image_tag)

dockerfile_location = os.path.join('docker', framework_version, 'base',
'Dockerfile.{}'.format(processor))

subprocess.check_call(['docker', 'build', '-t', base_image_uri,
'-f', dockerfile_location, cwd], cwd=cwd)
'-f', dockerfile_location, '--build-arg',
'py_version={}'.format(py_version[-1]), cwd], cwd=cwd)
print('created image {}'.format(base_image_uri))
return base_image_uri


def build_image(framework_name, framework_version, processor, tag, cwd='.'):
def build_image(framework_name, framework_version, py_version, processor, tag, cwd='.'):
_check_call('python setup.py bdist_wheel')

image_uri = get_image_uri(framework_name, tag)
Expand All @@ -41,7 +43,8 @@ def build_image(framework_name, framework_version, processor, tag, cwd='.'):
'Dockerfile.{}'.format(processor))

subprocess.check_call(
['docker', 'build', '-t', image_uri, '-f', dockerfile_location, cwd], cwd=cwd)
['docker', 'build', '-t', image_uri, '-f', dockerfile_location, '--build-arg',
'py_version={}'.format(py_version[-1]), cwd], cwd=cwd)
print('created image {}'.format(image_uri))
return image_uri

Expand Down