Maskrcnn new features and support for issues (#1078)

* add support for inference fixes issue 1028

* clean up notebook

* add environment variable for pre-trained model

* add support for inference and writing logs to shared file systems

* fix default model name

* document updates

* create log output directory on EFS file system

* add new notebook for experiment trials

Author: ajayvohra2005

advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized/resources/train.py

@@ -1,5 +1,6 @@
 import json
 import os
+import shutil
 import subprocess
 import sys
 import time
@@ -114,13 +115,29 @@ def build_host_arg(host_list, gpu_per_host):
         arg += f'{host}:{gpu_per_host}'
     return arg
 
-
+def copy_files(src, dest):
+    src_files = os.listdir(src)
+    for file in src_files:
+        path = os.path.join(src, file)
+        if os.path.isfile(path):
+            shutil.copy(path, dest)
+            
 def train():
 
     import pprint
     pprint.pprint(dict(os.environ), width = 1) 
 
     model_dir = os.environ['SM_MODEL_DIR']
+    log_dir = None
+    
+    copy_logs_to_model_dir = False
+    
+    try:
+        log_dir = os.environ['SM_CHANNEL_LOG']
+        copy_logs_to_model_dir = True
+    except KeyError:
+        log_dir = model_dir
+        
     train_data_dir = os.environ['SM_CHANNEL_TRAIN']
 
     print("pre-setup check")
@@ -203,6 +220,20 @@ def train():
     except KeyError:
         images_per_epoch = 120000
 
+    try:
+        backbone_weights = hyperparamters['backbone_weights']
+    except KeyError:
+        backbone_weights = 'ImageNet-R50-AlignPadding.npz'
+    
+    try:
+        resnet_arch = hyperparamters['resnet_arch']
+    except KeyError:
+        resnet_arch = 'resnet50'
+        
+    resnet_num_blocks = '[3, 4, 6, 3]'
+    if resnet_arch == 'resnet101':
+        resnet_num_blocks = '[3, 4, 23, 3]'
+        
     gpus_per_host = int(os.environ['SM_NUM_GPUS'])
     numprocesses = len(all_hosts) * int(gpus_per_host)
 
@@ -226,15 +257,16 @@ def train():
 -x LD_LIBRARY_PATH -x PATH \\
 --output-filename {model_dir}  \\
 /usr/local/bin/python3.6 /mask-rcnn-tensorflow/MaskRCNN/train.py \
---logdir {model_dir} \
+--logdir {log_dir} \
 --fp16 \
 --throughput_log_freq=2000 \
 --images_per_epoch {images_per_epoch} \
 --config \
 MODE_FPN={mode_fpn} \
 MODE_MASK={mode_mask} \
 DATA.BASEDIR={train_data_dir} \
-BACKBONE.WEIGHTS={train_data_dir}/pretrained-models/ImageNet-R50-AlignPadding.npz \
+BACKBONE.RESNET_NUM_BLOCKS='{resnet_num_blocks}' \
+BACKBONE.WEIGHTS={train_data_dir}/pretrained-models/{backbone_weights} \
 BACKBONE.NORM={batch_norm} \
 DATA.TRAIN='["{data_train}"]' \
 DATA.VAL='("{data_val}",)' \
@@ -269,6 +301,9 @@ def train():
         print("train exception occured", file=sys.stderr)
         exitcode = 1
         print(str(e), file=sys.stderr)
+    finally:
+        if copy_logs_to_model_dir:
+            copy_files(log_dir, model_dir)
 
     sys.stdout.flush()
     sys.stderr.flush()

advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/Dockerfile

@@ -0,0 +1,63 @@
+FROM  763104351884.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:1.13-horovod-gpu-py36-cu100-ubuntu16.04-v2.0
+
+ENV HOROVOD_VERSION=0.18.1
+
+RUN pip install --upgrade pip
+
+# Need to reinstall some libraries the DL container provides due to custom Tensorflow binary
+RUN pip uninstall -y tensorflow tensorboard tensorflow-estimator keras h5py horovod numpy
+
+# Download and install custom Tensorflow binary
+RUN wget https://github.com/aws-samples/mask-rcnn-tensorflow/releases/download/v0.0.0/tensorflow-1.13.0-cp36-cp36m-linux_x86_64.whl && \
+    pip install tensorflow-1.13.0-cp36-cp36m-linux_x86_64.whl && \
+    pip install tensorflow-estimator==1.13.0 && \
+    rm tensorflow-1.13.0-cp36-cp36m-linux_x86_64.whl
+
+# Install Horovod, temporarily using CUDA stubs
+RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs && \
+    HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horovod==${HOROVOD_VERSION} && \
+    ldconfig
+
+# Install OpenSSH for MPI to communicate between containers
+RUN apt-get install -y --no-install-recommends openssh-client openssh-server
+RUN mkdir -p /var/run/sshd && \
+  sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+RUN rm -rf /root/.ssh/ && \
+  mkdir -p /root/.ssh/ && \
+  ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \
+  cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
+  printf "Host *\n  StrictHostKeyChecking no\n" >> /root/.ssh/config
+
+RUN pip install awscli
+RUN pip install boto3
+RUN pip install ujson==1.35
+RUN pip install opencv-python==4.1.0.25
+RUN pip install Cython==0.28.4
+RUN pip install pycocotools==2.0.0
+RUN pip install matplotlib==3.0.3
+RUN pip install markdown==3.1
+RUN pip install numpy==1.17.5
+
+RUN git clone https://github.com/aws-samples/mask-rcnn-tensorflow
+
+RUN chmod -R +w /mask-rcnn-tensorflow
+RUN pip install -e /mask-rcnn-tensorflow/
+
+##########################################################################################
+# SageMaker requirements
+##########################################################################################
+## install flask
+RUN pip install flask
+
+### Install nginx notebook
+RUN apt-get -y update && apt-get install -y --no-install-recommends \
+         wget \
+         nginx \
+         ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+ 
+COPY resources/*.* /
+ENV WORKDIR /
+
+ENTRYPOINT ["python", "/serve.py"]

advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/build_tools/build_and_push.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# This script shows how to build the Docker image and push it to ECR to be ready for use
+# by SageMaker.
+
+# The argument to this script is the image name. This will be used as the image on the local
+# machine and combined with the account and region to form the repository name for ECR.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $DIR/set_env.sh
+
+# set region
+region=
+if [ "$#" -eq 1 ]; then
+    region=$1
+else
+    echo "usage: $0 <aws-region>"
+    exit 1
+fi
+  
+
+image=$IMAGE_NAME
+tag=$IMAGE_TAG
+
+# Get the account number associated with the current IAM credentials
+account=$(aws sts get-caller-identity --query Account --output text)
+
+if [ $? -ne 0 ]
+then
+    exit 255
+fi
+
+
+fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:${tag}"
+
+# If the repository doesn't exist in ECR, create it.
+aws ecr describe-repositories --region ${region} --repository-names "${image}" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+    aws ecr create-repository --region ${region} --repository-name "${image}" > /dev/null
+fi
+
+
+# Build the docker image locally with the image name and then push it to ECR
+# with the full name.
+
+# Get the login command from ECR and execute it directly
+$(aws ecr get-login --no-include-email --region us-west-2  --registry-ids 763104351884)
+
+docker build  -t ${image} $DIR/..
+docker tag ${image} ${fullname}
+
+# Get the login command from ECR and execute it directly
+$(aws ecr get-login --region ${region} --no-include-email)
+docker push ${fullname}
+if [ $? -eq 0 ]; then
+	echo "Amazon ECR URI: ${fullname}"
+else
+	echo "Error: Image build and push failed"
+	exit 1
+fi

advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/build_tools/set_env.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+export IMAGE_NAME=mask-rcnn-tensorflow-serving-sagemaker
+export IMAGE_TAG=tf1.13-153442b

advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/ImageNet-R50-AlignPadding.npz

@@ -0,0 +1,39 @@
+worker_processes 1;
+daemon off; # Prevent forking
+
+
+pid /tmp/nginx.pid;
+error_log /var/log/nginx/error.log;
+
+events {
+  # defaults
+}
+
+http {
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+  access_log /var/log/nginx/access.log combined;
+  
+  upstream gunicorn {
+    server unix:/tmp/gunicorn.sock;
+  }
+
+  server {
+    listen 0.0.0.0:8080 deferred;
+    client_max_body_size 5m;
+
+    keepalive_timeout 70;
+    proxy_read_timeout 1200s;
+
+    location ~ ^/(ping|invocations) {
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Host $http_host;
+      proxy_redirect off;
+      proxy_pass http://gunicorn;
+    }
+
+    location / {
+      return 404 "{}";
+    }
+  }
+}