Add release build

MANIFEST.in

@@ -1,2 +1,3 @@
+include VERSION
 include LICENSE
 include README.md

VERSION

@@ -0,0 +1 @@
+2.0.1.dev0

buildspec-release.yml

@@ -0,0 +1,90 @@
+version: 0.2
+
+env:
+  variables:
+    FRAMEWORK_VERSION: '1.13.1'
+    GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
+    SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
+
+phases:
+  pre_build:
+    commands:
+      - start-dockerd
+      - ACCOUNT=$(aws sts get-caller-identity --query 'Account' --output text)
+
+  build:
+    commands:
+      # prepare the release (update versions, changelog etc.)
+      - git-release --prepare
+
+      # run linter
+      - tox -e flake8
+
+      # run unit tests
+      - tox -e py36,py27 test/unit
+
+      # Create pip archive
+      - build_dir="docker/$FRAMEWORK_VERSION"
+      - python3 setup.py sdist
+      - tar_name=$(ls dist)
+      - cp dist/$tar_name $build_dir
+
+      # Build all images
+      - python3 scripts/build_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
+
+      # run local cpu integ tests
+      - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
+      - tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --framework-version $FRAMEWORK_VERSION --processor cpu
+      - tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version $FRAMEWORK_VERSION --processor cpu
+
+      # launch remote gpu instance
+      - prefix='ml.'
+      - instance_type=${GPU_INSTANCE_TYPE#"$prefix"}
+      - create-key-pair
+      - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
+
+      - printf "$SETUP_CMDS" > $SETUP_FILE
+      - py2_cmd="tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version $FRAMEWORK_VERSION --processor gpu"
+      - remote-test --github-repo $GITHUB_REPO --test-cmd "$py2_cmd" --setup-file $SETUP_FILE
+      - py3_cmd="tox -e py36 -- test/integration/local --docker-base-name sagemaker-tensorflow-scriptmode --framework-version $FRAMEWORK_VERSION --processor gpu"
+      - remote-test --github-repo $GITHUB_REPO --test-cmd "$py3_cmd" --setup-file $SETUP_FILE
+
+      # Publish all images
+      - python3 scripts/publish_all.py --account $ACCOUNT --region $AWS_DEFAULT_REGION
+
+      - |
+        echo '[{
+          "repository": "sagemaker-tensorflow-scriptmode",
+          "tags": [{
+            "source": "1.13.1-cpu-py2",
+            "dest": ["1.13.1-cpu-py2", "1.13-cpu-py2", "1.13.1-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
+          },{
+            "source": "1.13.1-cpu-py3",
+            "dest": ["1.13.1-cpu-py3", "1.13-cpu-py3", "1.13.1-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
+          },{
+            "source": "1.13.1-gpu-py2",
+            "dest": ["1.13.1-gpu-py2", "1.13-gpu-py2", "1.13.1-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
+          },{
+            "source": "1.13.1-gpu-py3",
+            "dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
+          }],
+          "test": [
+            "tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu",
+            "tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor cpu",
+            "tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu",
+            "tox -e py36 -- test/integration/sagemaker -n auto --region {region} --account-id 520713654638 --docker-base-name sagemaker-tensorflow-scriptmode --py-version 2 --framework-version 1.13.1 --processor gpu"
+          ]
+        }]' > deployments.json
+
+      # publish the release to github
+      - git-release --publish
+
+    finally:
+      # shut down remote gpu instance
+      - cleanup-gpu-instances
+      - cleanup-key-pairs
+
+artifacts:
+  files:
+    - deployments.json
+  name: ARTIFACT_1

buildspec.yml

@@ -2,11 +2,11 @@ version: 0.2
 
 env:
   variables:
-    FRAMEWORK_VERSION: '1.12.0'
-    CPU_FRAMEWORK_BINARY: 'https://s3.amazonaws.com/inferno-dlami/tensorflow/1.12.0/with_intel_patch_mkl_on_off/tensorflow-1.12.0-cp27-cp27mu-linux_x86_64.whl'
-    CPU_PY_VERSION: '2'
+    FRAMEWORK_VERSION: '1.13.1'
+    CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
+    CPU_PY_VERSION: '3'
     CPU_INSTANCE_TYPE: 'ml.c4.xlarge'
-    GPU_FRAMEWORK_BINARY: 'https://s3.amazonaws.com/inferno-dlami/tensorflow/1.12.0/gpu/tensorflow-1.12.0-cp36-cp36m-linux_x86_64.whl'
+    GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
     GPU_PY_VERSION: '3'
     GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
     ECR_REPO: 'sagemaker-test'
@@ -49,7 +49,7 @@ phases:
       - CPU_TAG="$FRAMEWORK_VERSION-cpu-py$CPU_PY_VERSION-$build_id"
 
       - cd $build_dir
-      - docker build -f $cpu_dockerfile --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG .
+      - docker build -f $cpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG .
       - cd ../../
 
       # build gpu image
@@ -62,7 +62,7 @@ phases:
       - GPU_TAG="$FRAMEWORK_VERSION-gpu-py$GPU_PY_VERSION-$build_id"
 
       - cd $build_dir
-      - docker build -f $gpu_dockerfile --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG .
+      - docker build -f $gpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG .
       - cd ../../
 
       # push images to ecr
@@ -97,15 +97,15 @@ phases:
       # run cpu sagemaker tests
       - |
         if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
-          pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --instance-type $CPU_INSTANCE_TYPE
+          pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --instance-type $CPU_INSTANCE_TYPE
         else
           echo "skipping cpu sagemaker tests"
         fi
 
       # run gpu sagemaker tests
       - |
         if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
-          pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --instance-type $GPU_INSTANCE_TYPE
+          pytest test/integration/sagemaker -n auto --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --instance-type $GPU_INSTANCE_TYPE
         else
           echo "skipping gpu sagemaker tests"
         fi

scripts/build_all.py

@@ -0,0 +1,76 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import argparse
+import os
+import subprocess
+
+VERSION = '1.13.1'
+REPO = 'sagemaker-tensorflow-scriptmode'
+PY2_CPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp27-cp27mu-linux_x86_64.whl' # noqa
+PY3_CPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' # noqa
+PY2_GPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp27-cp27mu-linux_x86_64.whl' # noqa
+PY3_GPU_BINARY = 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' # noqa
+DEV_ACCOUNT = '142577830533'
+REGION = 'us-west-2'
+
+
+def _parse_args():
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--account', type=str, default=DEV_ACCOUNT)
+    parser.add_argument('--region', type=str, default=REGION)
+    parser.add_argument('--version', type=str, default=VERSION)
+    parser.add_argument('--py2-cpu-binary', type=str, default=PY2_CPU_BINARY)
+    parser.add_argument('--py3-cpu-binary', type=str, default=PY3_CPU_BINARY)
+    parser.add_argument('--py2-gpu-binary', type=str, default=PY2_GPU_BINARY)
+    parser.add_argument('--py3-gpu-binary', type=str, default=PY3_GPU_BINARY)
+    parser.add_argument('--repo', type=str, default=REPO)
+
+    return parser.parse_args()
+
+
+args = _parse_args()
+binaries = {
+    'py2-cpu': args.py2_cpu_binary,
+    'py3-cpu': args.py3_cpu_binary,
+    'py2-gpu': args.py2_gpu_binary,
+    'py3-gpu': args.py3_gpu_binary
+}
+build_dir = os.path.join('docker', args.version)
+
+# Run docker-login so we can pull the cached image
+login_cmd = subprocess.check_output(
+    'aws ecr get-login --no-include-email --registry-id {}'.format(args.account).split())
+print('Executing docker login command: '.format(login_cmd))
+subprocess.check_call(login_cmd.split())
+
+for arch in ['cpu', 'gpu']:
+    for py_version in ['2', '3']:
+        binary_url = binaries['py{}-{}'.format(py_version, arch)]
+        binary_file = os.path.basename(binary_url)
+        cmd = 'wget -O {}/{} {}'.format(build_dir, binary_file, binary_url)
+        print('Downloading binary file {}: '.format(cmd))
+        subprocess.check_call(cmd.split())
+        tag = '{}-{}-py{}'.format(args.version, arch, py_version)
+        prev_image_uri = '{}.dkr.ecr.{}.amazonaws.com/{}:{}'.format(args.account, args.region, args.repo, tag)
+        dockerfile = os.path.join(build_dir, 'Dockerfile.{}'.format(arch))
+        build_cmd = 'docker build -f {} --cache-from {} ' \
+                    '--build-arg py_version={} --build-arg framework_installable={} ' \
+                    '-t {}:{} {}'.format(dockerfile, prev_image_uri, py_version, binary_file, args.repo, tag, build_dir)
+        print('Building docker image: {}'.format(build_cmd))
+        subprocess.check_call(build_cmd.split())
+        print('Deleting binary file {}'.format(binary_file))
+        subprocess.check_call('rm {}'.format(os.path.join(build_dir, binary_file)).split())

scripts/publish_all.py

@@ -0,0 +1,52 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import argparse
+import subprocess
+
+DEV_ACCOUNT = '142577830533'
+VERSION = '1.13.1'
+REGION = 'us-west-2'
+REPO = 'sagemaker-tensorflow-scriptmode'
+
+
+def _parse_args():
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--account', type=str, default=DEV_ACCOUNT)
+    parser.add_argument('--version', type=str, default=VERSION)
+    parser.add_argument('--repo', type=str, default=REPO)
+    parser.add_argument('--region', type=str, default=REGION)
+
+    return parser.parse_args()
+
+
+args = _parse_args()
+
+for arch in ['cpu', 'gpu']:
+    for py_version in ['2', '3']:
+        source = '{}:{}-{}-py{}'.format(args.repo, args.version, arch, py_version)
+        dest = '{}.dkr.ecr.{}.amazonaws.com/{}'.format(args.account, args.region, source)
+        tag_cmd = 'docker tag {} {}'.format(source, dest)
+        print('Tagging image: {}'.format(tag_cmd))
+        subprocess.check_call(tag_cmd.split())
+        login_cmd = subprocess.check_output(
+            'aws ecr get-login --no-include-email --registry-id {} --region {}'
+            .format(args.account, args.region).split())
+        print('Executing docker login command: {}'.format(login_cmd))
+        subprocess.check_call(login_cmd.split())
+        push_cmd = 'docker push {}'.format(dest)
+        print('Pushing image: {}'.format(push_cmd))
+        subprocess.check_call(push_cmd.split())

setup.py

@@ -24,9 +24,13 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 
+def read_version():
+    return read('VERSION').strip()
+
+
 setup(
     name='sagemaker_tensorflow_container',
-    version='2.0.0',
+    version=read_version(),
     description='Open source library for creating '
                 'TensorFlow containers to run on Amazon SageMaker.',
 

tox.ini

@@ -50,17 +50,18 @@ require-code = True
 # TEAMCITY_VERSION environment variable exists during build on Teamcity. teamcity-messages uses it in order to enable
 # reporting to TeamCity.
 passenv =
-    TEAMCITY_VERSION
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
     AWS_SESSION_TOKEN
+    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
+    AWS_DEFAULT_REGION
 # {posargs} can be passed in by additional arguments specified when invoking tox.
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
     coverage run --rcfile .coveragerc_{envname} --source sagemaker_tensorflow_container -m py.test {posargs}
     {env:IGNORE_COVERAGE:} coverage report --include *sagemaker_tensorflow_container* --show-missing
-deps = .[test]
-       sagemaker-containers
+deps = sagemaker-containers
+extras = test
 
 [testenv:flake8]
 basepython = python