update gluon examples (aws#4)

Author: jesterhazy

im-python-sdk/mxnet_gluon_cifar10/cifar10.ipynb

@@ -16,23 +16,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "import credentials # put your credentials in credentials.py \n",
-    "import logging \n",
+    "import credentials # put your credentials in credentials.py\n",
     "import os\n",
+    "import boto3\n",
     "import im\n",
     "from im.mxnet import MXNet\n",
-    "from im.session import s3_input \n",
     "from mxnet import gluon\n",
     "\n",
-    "os.environ['AWS_DEFAULT_REGION']='us-west-2'\n",
-    "\n",
-    "# Session will use your default credentials\n",
-    "# e.g. from environment variables or your ~/.aws/credentials file.\n",
     "ims = im.Session()\n",
     "\n",
     "# Replace with a role that gives IM access to s3 and cloudwatch\n",
@@ -71,9 +64,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "inputs = ims.upload_data(path='data', key_prefix='data/gluon-cifar10')\n",
@@ -94,9 +85,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "!cat 'cifar10.py'"
@@ -126,8 +115,7 @@
     "          hyperparameters={'batch_size': 128, \n",
     "                           'epochs': 50, \n",
     "                           'learning_rate': 0.1, \n",
-    "                           'momentum': 0.9,\n",
-    "                           '_ps_verbose': 0})"
+    "                           'momentum': 0.9})"
    ]
   },
   {
@@ -160,9 +148,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "predictor = m.deploy(min_instances=1, max_instances=1, instance_type='c4.xlarge')"
@@ -225,14 +211,13 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true,
     "scrolled": true
    },
    "outputs": [],
    "source": [
     "for i, img in enumerate(image_data):\n",
     "    response = predictor.predict(img)\n",
-    "    print('image {}: class: {}'.format(i, int(response)))\n"
+    "    print('image {}: class: {}'.format(i, int(response)))"
    ]
   },
   {

im-python-sdk/mxnet_gluon_mnist/credentials.py

@@ -1,4 +1,5 @@
 import os
 
-os.environ['AWS_ACCESS_KEY_ID'] = 'type your aws access key id here'
-os.environ['AWS_SECRET_ACCESS_KEY'] = 'type your aws secret access key here'
+# uncomment these lines and add your credentials
+# os.environ['AWS_ACCESS_KEY_ID'] = 'type your aws access key id here'
+# os.environ['AWS_SECRET_ACCESS_KEY'] = 'type your aws secret access key here'

im-python-sdk/mxnet_gluon_mnist/mnist.py

@@ -6,6 +6,7 @@
 from mxnet.gluon import nn
 import numpy as np
 import json
+import time
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -48,6 +49,7 @@ def train(channel_input_dirs, hyperparameters, **kwargs):
     for epoch in range(epochs):
         # reset data iterator and metric at begining of epoch.
         metric.reset()
+        btic = time.time()
         for i, (data, label) in enumerate(train_data):
             # Copy data to ctx if necessary
             data = data.as_in_context(ctx)
@@ -65,7 +67,10 @@ def train(channel_input_dirs, hyperparameters, **kwargs):
 
             if i % log_interval == 0 and i > 0:
                 name, acc = metric.get()
-                print('[Epoch %d Batch %d] Training: %s=%f' % (epoch, i, name, acc))
+                print('[Epoch %d Batch %d] Training: %s=%f, %f samples/s' %
+                      (epoch, i, name, acc, batch_size / (time.time() - btic)))
+
+            btic = time.time()
 
         name, acc = metric.get()
         print('[Epoch %d] Training: %s=%f' % (epoch, name, acc))

im-python-sdk/mxnet_gluon_mnist/mnist_with_gluon.ipynb

@@ -18,15 +18,13 @@
    },
    "outputs": [],
    "source": [
-    "import os\n",
-    "\n",
     "import credentials # put your credentials in credentials.py\n",
+    "import os\n",
+    "import boto3\n",
     "import im\n",
     "from im.mxnet import MXNet\n",
-    "from im.session import s3_input \n",
     "from mxnet import gluon\n",
     "\n",
-    "os.environ['AWS_DEFAULT_REGION']='us-west-2'\n",
     "\n",
     "ims = im.Session()\n",
     "\n",
@@ -45,7 +43,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "gluon.data.vision.MNIST('./data/train', train=True)\n",
@@ -64,7 +64,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "inputs = ims.upload_data(path='data', key_prefix='data/mnist')"
@@ -84,7 +86,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "!cat 'mnist.py'"
@@ -96,7 +100,7 @@
    "source": [
     "## Run the training script on IM\n",
     "\n",
-    "The ```MXNet``` class allows us to run our training function on IM infrastructure. We need to configure it with our training script, an IAM role, the number of training instances, and the training instance type. In this case we will run our training job on a single m4.xlarge instance. "
+    "The ```MXNet``` class allows us to run our training function on IM infrastructure. We need to configure it with our training script, an IAM role, the number of training instances, and the training instance type. In this case we will run our training job on a single c4.xlarge instance. "
    ]
   },
   {
@@ -110,9 +114,10 @@
     "m = MXNet(\"mnist.py\", \n",
     "          role=role, \n",
     "          train_instance_count=1, \n",
-    "          train_instance_type=\"m4.xlarge\",\n",
+    "          train_instance_type=\"c4.xlarge\",\n",
+    "          enable_cloudwatch_metrics=True,\n",
     "          hyperparameters={'batch_size': 100, \n",
-    "                         'epochs': 10, \n",
+    "                         'epochs': 20, \n",
     "                         'learning_rate': 0.1, \n",
     "                         'momentum': 0.9, \n",
     "                         'log_interval': 100})"
@@ -129,6 +134,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "collapsed": true,
     "scrolled": true
    },
    "outputs": [],
@@ -148,10 +154,13 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true,
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
-    "predictor = m.deploy(instance_type = 'm4.xlarge', min_instances = 1, max_instances = 1)"
+    "predictor = m.deploy(instance_type = 'c4.xlarge', min_instances = 1, max_instances = 1)"
    ]
   },
   {
@@ -164,7 +173,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from IPython.display import HTML\n",
@@ -182,12 +193,22 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "collapsed": true,
     "scrolled": true
    },
    "outputs": [],
    "source": [
     "response = predictor.predict(data)\n",
-    "print int(response)\n"
+    "print int(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Cleanup\n",
+    "\n",
+    "After you have finished with this example, remember to delete the prediction endpoint to release the instance(s) associated with it."
    ]
   },
   {
@@ -197,14 +218,16 @@
     "collapsed": true
    },
    "outputs": [],
-   "source": []
+   "source": [
+    "m.delete_endpoint()"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python [conda env:mxnet_p27]",
    "language": "python",
-   "name": "python2"
+   "name": "conda-env-mxnet_p27-py"
   },
   "language_info": {
    "codemirror_mode": {