simplify tfs versioning (#5)

Author: jesterhazy

buildspec.yml

@@ -3,20 +3,44 @@ version: 0.2
 phases:
   pre_build:
     commands:
-      - start-dockerd.sh
+      - start-dockerd
 
       # fix permissions dropped by CodePipeline
       - chmod +x ./scripts/*.sh
       - chmod +x ./container/sagemaker/serve
   build:
     commands:
+      # prepare the release (update versions, changelog etc.)
+      - if-release-build git-release --prepare
+
       - tox -e flake8,pylint,jshint
 
       # build images
-      - ./scripts/build.sh --version 1.11 --arch cpu
-      - ./scripts/build.sh --version 1.11 --arch gpu
-      - ./scripts/build.sh --version 1.12 --arch cpu
-      - ./scripts/build.sh --version 1.12 --arch gpu
+      - ./scripts/build.sh --version 1.11.1 --arch cpu
+      - ./scripts/build.sh --version 1.11.1 --arch gpu
+      - ./scripts/build.sh --version 1.12.0 --arch cpu
+      - ./scripts/build.sh --version 1.12.0 --arch gpu
 
       # run tests
       - tox -e py36
+
+      # push docker images to ECR
+      - if-release-build ./scripts/publish.sh --version 1.11.1 --arch cpu
+      - if-release-build ./scripts/publish.sh --version 1.11.1 --arch gpu
+      - if-release-build ./scripts/publish.sh --version 1.12.0 --arch cpu
+      - if-release-build ./scripts/publish.sh --version 1.12.0 --arch gpu
+
+      # write images to manifest (used by deploy step)
+      - if-release-build create-manifest docker
+            --images
+              sagemaker-tensorflow-serving:1.11.1-cpu
+              sagemaker-tensorflow-serving:1.11.1-gpu
+              sagemaker-tensorflow-serving:1.12.0-cpu
+              sagemaker-tensorflow-serving:1.12.0-gpu
+
+      # publish the release to github
+      - if-release-build git-release --publish
+
+artifacts:
+  files:
+    - release-manifest.json

docker/1.11/Dockerfile.gpu

@@ -1,4 +1,6 @@
-FROM tensorflow/serving:1.11.1 as tfs
+ARG TFS_VERSION
+
+FROM tensorflow/serving:${TFS_VERSION} as tfs
 FROM ubuntu:16.04
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 
@@ -16,5 +18,6 @@ RUN \
 
 COPY ./ /
 
-ENV SAGEMAKER_TFS_VERSION "1.11"
+ARG TFS_SHORT_VERSION
+ENV SAGEMAKER_TFS_VERSION "${TFS_SHORT_VERSION}"
 ENV PATH "$PATH:/sagemaker"

docker/1.12/Dockerfile.cpu

@@ -1,4 +1,6 @@
-FROM tensorflow/serving:1.12.0-gpu as tfs
+ARG TFS_VERSION
+
+FROM tensorflow/serving:${TFS_VERSION}-gpu as tfs
 FROM nvidia/cuda:9.0-base-ubuntu16.04
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 
@@ -55,5 +57,7 @@ RUN \
 
 COPY ./ /
 
-ENV SAGEMAKER_TFS_VERSION "1.12"
+
+ARG TFS_SHORT_VERSION
+ENV SAGEMAKER_TFS_VERSION "${TFS_SHORT_VERSION}"
 ENV PATH "$PATH:/sagemaker"

docker/1.11/Dockerfile.cpu renamed to docker/Dockerfile.cpu

@@ -10,12 +10,14 @@ parse_std_args "$@"
 
 echo "pulling previous image for layer cache... "
 $(aws ecr get-login --no-include-email --registry-id $aws_account) &>/dev/null || echo 'warning: ecr login failed'
-docker pull $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$minor_version-$arch &>/dev/null || echo 'warning: pull failed'
+docker pull $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$full_version-$arch &>/dev/null || echo 'warning: pull failed'
 docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com &>/dev/null
 
 echo "building image... "
 docker build \
-    --cache-from $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$minor_version-$arch \
-    -f docker/$major_version/Dockerfile.$arch \
-    -t sagemaker-tensorflow-serving:$minor_version-$arch \
-    -t sagemaker-tensorflow-serving:$major_version-$arch container
+    --cache-from $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$full_version-$arch \
+    --build-arg TFS_VERSION=$full_version \
+    --build-arg TFS_SHORT_VERSION=$short_version \
+    -f docker/Dockerfile.$arch \
+    -t sagemaker-tensorflow-serving:$full_version-$arch \
+    -t sagemaker-tensorflow-serving:$short_version-$arch container

docker/1.12/Dockerfile.gpu renamed to docker/Dockerfile.gpu

@@ -9,8 +9,8 @@ source scripts/shared.sh
 parse_std_args "$@"
 
 $(aws ecr get-login --no-include-email --registry-id $aws_account)
-docker tag sagemaker-tensorflow-serving:$minor_version-$arch $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$minor_version-$arch
-docker tag sagemaker-tensorflow-serving:$minor_version-$arch $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$major_version-$arch
-docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$minor_version-$arch
-docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$major_version-$arch
+docker tag sagemaker-tensorflow-serving:$full_version-$arch $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$full_version-$arch
+docker tag sagemaker-tensorflow-serving:$full_version-$arch $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$short_version-$arch
+docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$full_version-$arch
+docker push $aws_account.dkr.ecr.$aws_region.amazonaws.com/sagemaker-tensorflow-serving:$short_version-$arch
 docker logout https://$aws_account.dkr.ecr.$aws_region.amazonaws.com

scripts/build.sh

@@ -8,14 +8,6 @@ function error() {
     exit 1
 }
 
-function get_latest_major_version() {
-    ls -1 docker/ | sort -r -n -t.  -k 1,1 -k 2,2 -k 3 | head -1
-}
-
-function get_minor_version() {
-    grep 'tensorflow/serving:' docker/$1/Dockerfile.cpu | sed 's#^.*:\([0-9][0-9\.]*\) .*#\1#'
-}
-
 function get_default_region() {
     if [ -n "${AWS_DEFAULT_REGION:-}" ]; then
         echo "$AWS_DEFAULT_REGION"
@@ -24,14 +16,23 @@ function get_default_region() {
     fi
 }
 
+function get_full_version() {
+    echo $1 | sed 's#^\([0-9][0-9]*\.[0-9][0-9]*\)$#\1.0#'
+}
+
+function get_short_version() {
+    echo $1 | sed 's#\([0-9][0-9]*\.[0-9][0-9]*\)\.[0-9][0-9]*#\1#'
+}
+
 function get_aws_account() {
     aws sts get-caller-identity --query 'Account' --output text
 }
 
 function parse_std_args() {
     # defaults
     arch='cpu'
-    major_version=$(get_latest_major_version)
+    version='1.12.0'
+
     aws_region=$(get_default_region)
     aws_account=$(get_aws_account)
 
@@ -40,7 +41,7 @@ function parse_std_args() {
 
     case $key in
         -v|--version)
-        major_version="$2"
+        version="$2"
         shift
         shift
         ;;
@@ -61,11 +62,12 @@ function parse_std_args() {
     esac
     done
 
-    [[ -z "${major_version// }" ]] && error 'missing version'
+    [[ -z "${version// }" ]] && error 'missing version'
     [[ "$arch" =~ ^(cpu|gpu)$ ]] || error "invalid arch: $arch"
     [[ -z "${aws_region// }" ]] && error 'missing aws region'
 
-    minor_version=$(get_minor_version $major_version)
+    full_version=$(get_full_version $version)
+    short_version=$(get_short_version $version)
 
     true
 }

scripts/publish.sh

@@ -23,4 +23,4 @@ $docker_command run \
     -e "SAGEMAKER_TFS_NGINX_LOGLEVEL=error" \
     -e "SAGEMAKER_BIND_TO_PORT=8080" \
     -e "SAGEMAKER_SAFE_PORT_RANGE=9000-9999" \
-    sagemaker-tensorflow-serving:$minor_version-$arch serve > log.txt 2>&1 &
+    sagemaker-tensorflow-serving:$full_version-$arch serve > log.txt 2>&1 &

scripts/shared.sh

@@ -8,4 +8,4 @@ source scripts/shared.sh
 
 parse_std_args "$@"
 
-docker kill $(docker ps -q --filter ancestor=sagemaker-tensorflow-serving:$minor_version-$arch)
+docker kill $(docker ps -q --filter ancestor=sagemaker-tensorflow-serving:$full_version-$arch)