merge from master

Author: ajaykarpur

.github/CODEOWNERS

@@ -7,4 +7,7 @@
 #
 # @See https://help.github.com/articles/about-codeowners/
 
-/sagemaker-experiments/* @aws/sagemakerexperimentsadmin
+/sagemaker-experiments/* @aws/sagemakerexperimentsadmin
+
+# Community contributed
+/contrib/ @aws/sagemaker-notebook-sas

.gitignore

@@ -3,3 +3,5 @@
 **/__pycache__
 **/.aws-sam
 .DS_Store
+
+**/_build

.readthedocs.yml

@@ -0,0 +1,15 @@
+# ReadTheDocs environment customization to allow us to use conda to install
+# libraries which have C dependencies for the doc build. See:
+# https://docs.readthedocs.io/en/latest/config-file/v2.html
+
+version: 2
+
+conda:
+  environment: environment.yml
+
+python:
+  version: 3.6
+
+sphinx:
+  configuration: conf.py
+  fail_on_warning: false

Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

README.md

@@ -77,7 +77,6 @@ The following provide examples demonstrating different capabilities of Amazon Sa
 - [Knapsack Problem](reinforcement_learning/rl_knapsack_coach_custom) demonstrates how to solve the knapsack problem using a custom environment.
 - [Mountain Car](reinforcement_learning/rl_mountain_car_coach_gymEnv) Mountain car is a classic RL problem. This notebook explains how to solve this using the OpenAI Gym environment.
 - [Distributed Neural Network Compression](reinforcement_learning/rl_network_compression_ray_custom) This notebook explains how to compress ResNets using RL, using a custom environment and the RLLib toolkit.
-- [Turtlebot Tracker](reinforcement_learning/rl_objecttracker_robomaker_coach_gazebo) This notebook demonstrates object tracking using AWS Robomaker and RL Coach in the Gazebo environment.
 - [Portfolio Management](reinforcement_learning/rl_portfolio_management_coach_customEnv) This notebook uses a custom Gym environment to manage multiple financial investments.
 - [Autoscaling](reinforcement_learning/rl_predictive_autoscaling_coach_customEnv) demonstrates how to adjust load depending on demand. This uses RL Coach and a custom environment.
 - [Roboschool](reinforcement_learning/rl_roboschool_ray) is an open source physics simulator that is commonly used to train RL policies for robotic systems. This notebook demonstrates training a few agents using it.
@@ -141,7 +140,7 @@ These examples provide you an introduction to how to use Neo to optimizes deep l
 - [Distributed TensorFlow](sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist) Adapts form [tensorflow mnist](sagemaker-python-sdk/tensorflow_distributed_mnist) including Neo API and comparsion between the baseline
 - [Predicting Customer Churn](sagemaker_neo_compilation_jobs/xgboost_customer_churn) Adapts form [xgboost customer churn](introduction_to_applying_machine_learning/xgboost_customer_churn) including Neo API and comparsion between the baseline
 
-### Amazon SageMaker Procesing
+### Amazon SageMaker Processing
 
 These examples show you how to use SageMaker Processing jobs to run data processing workloads.
 

_static/js/analytics.js

@@ -0,0 +1,2 @@
+console.log("Starting analytics...");
+var s_code=s.t();if(s_code)document.write(s_code)

_static/product-icon_Amazon_SageMaker_lockup_centered_squid_ink.png

@@ -43,17 +43,18 @@
    },
    "outputs": [],
    "source": [
-    "# Imports\n",
     "import os\n",
     "import boto3\n",
     "import sagemaker\n",
     "from time import sleep\n",
     "from collections import Counter\n",
     "import numpy as np\n",
     "import pandas as pd\n",
-    "from sagemaker import get_execution_role, local, Model, utils, fw_utils, s3\n",
+    "from sagemaker import get_execution_role, local, Model, utils, s3\n",
     "from sagemaker.estimator import Estimator\n",
-    "from sagemaker.predictor import RealTimePredictor, csv_serializer, StringDeserializer\n",
+    "from sagemaker.predictor import Predictor\n",
+    "from sagemaker.serializers import CSVSerializer\n",
+    "from sagemaker.deserializers import StringDeserializer\n",
     "from sklearn.metrics import accuracy_score, classification_report\n",
     "from IPython.core.display import display, HTML\n",
     "from IPython.core.interactiveshell import InteractiveShell\n",
@@ -74,9 +75,10 @@
     "    \"sts\", region_name=region, endpoint_url=utils.sts_regional_endpoint(region)\n",
     "    )\n",
     "account = client.get_caller_identity()['Account']\n",
-    "ecr_uri_prefix = utils.get_ecr_image_uri_prefix(account, region)\n",
-    "registry_id = fw_utils._registry_id(region, 'mxnet', 'py3', account, '1.6.0')\n",
-    "registry_uri = utils.get_ecr_image_uri_prefix(registry_id, region)"
+    "\n",
+    "registry_uri_training = sagemaker.image_uris.retrieve('mxnet', region, version= '1.6.0', py_version='py3', instance_type='ml.m5.2xlarge', image_scope='training')\n",
+    "registry_uri_inference = sagemaker.image_uris.retrieve('mxnet', region, version= '1.6.0', py_version='py3', instance_type='ml.m5.2xlarge', image_scope='inference')\n",
+    "ecr_uri_prefix = account +'.'+'.'.join(registry_uri_training.split('/')[0].split('.')[1:])"
    ]
   },
   {
@@ -291,7 +293,12 @@
     "hyperparameters = {\n",
     "  'fit_args': fit_args,\n",
     "  'feature_importance': True\n",
-    "}"
+    "}\n",
+    "\n",
+    "tags = [{\n",
+    "    'Key' : 'AlgorithmName',\n",
+    "    'Value' : 'AutoGluon-Tabular'\n",
+    "}]"
    ]
   },
   {
@@ -323,19 +330,38 @@
     "\n",
     "ecr_image = f'{ecr_uri_prefix}/{training_algorithm_name}:latest'\n",
     "\n",
-    "estimator = Estimator(image_name=ecr_image,\n",
+    "estimator = Estimator(image_uri=ecr_image,\n",
     "                      role=role,\n",
-    "                      train_instance_count=1,\n",
-    "                      train_instance_type=instance_type,\n",
+    "                      instance_count=1,\n",
+    "                      instance_type=instance_type,\n",
     "                      hyperparameters=hyperparameters,\n",
-    "                      train_volume_size=100)\n",
+    "                      volume_size=100,\n",
+    "                      tags=tags)\n",
     "\n",
     "# Set inputs. Test data is optional, but requires a label column.\n",
     "inputs = {'training': train_s3_path, 'testing': test_s3_path}\n",
     "\n",
     "estimator.fit(inputs)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Review the performance of the trained model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils.ag_utils import launch_viewer\n",
+    "\n",
+    "launch_viewer(is_debug=False)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -354,10 +380,10 @@
    "outputs": [],
    "source": [
     "# Create predictor object\n",
-    "class AutoGluonTabularPredictor(RealTimePredictor):\n",
+    "class AutoGluonTabularPredictor(Predictor):\n",
     "    def __init__(self, *args, **kwargs):\n",
-    "        super().__init__(*args, content_type='text/csv', \n",
-    "                         serializer=csv_serializer, \n",
+    "        super().__init__(*args, \n",
+    "                         serializer=CSVSerializer(), \n",
     "                         deserializer=StringDeserializer(), **kwargs)"
    ]
   },
@@ -372,10 +398,10 @@
     "ecr_image = f'{ecr_uri_prefix}/{inference_algorithm_name}:latest'\n",
     "\n",
     "if instance_type == 'local':\n",
-    "    model = estimator.create_model(image=ecr_image, role=role)\n",
+    "    model = estimator.create_model(image_uri=ecr_image, role=role)\n",
     "else:\n",
     "    model_uri = os.path.join(estimator.output_path, estimator._current_job_name, \"output\", \"model.tar.gz\")\n",
-    "    model = Model(model_uri, ecr_image, role=role, sagemaker_session=session, predictor_cls=AutoGluonTabularPredictor)"
+    "    model = Model(ecr_image, model_data=model_uri, role=role, sagemaker_session=session, predictor_cls=AutoGluonTabularPredictor)"
    ]
   },
   {

_static/sagemaker_gears.jpg

@@ -1,5 +1,5 @@
 ARG REGISTRY_URI
-FROM ${REGISTRY_URI}/mxnet-inference:1.6.0-cpu-py3
+FROM ${REGISTRY_URI}
 
 RUN pip install autogluon
 RUN pip install PrettyTable

advanced_functionality/autogluon-tabular/AutoGluon_Tabular_SageMaker.ipynb

@@ -1,5 +1,5 @@
 ARG REGISTRY_URI
-FROM ${REGISTRY_URI}/mxnet-training:1.6.0-cpu-py3
+FROM ${REGISTRY_URI}
 
 RUN pip install autogluon
 RUN pip install PrettyTable
@@ -9,6 +9,9 @@ ENV PATH="/opt/ml/code:${PATH}"
 COPY container-training/train.py /opt/ml/code/train.py
 COPY container-training/inference.py /opt/ml/code/inference.py
 
+# Install seaborn for plot
+RUN pip install seaborn
+
 # this environment variable is used by the SageMaker PyTorch container to determine our user code directory.
 ENV SAGEMAKER_SUBMIT_DIRECTORY /opt/ml/code
 

advanced_functionality/autogluon-tabular/container-inference/Dockerfile.inference

@@ -19,6 +19,9 @@
     import autogluon as ag
     from autogluon import TabularPrediction as task
     from autogluon.task.tabular_prediction import TabularDataset
+    from autogluon.utils.tabular.ml.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS
+    
+    print(f'DEBUG AutoGluon version : {ag.__version__}')
 
 
 # ------------------------------------------------------------ #
@@ -49,7 +52,59 @@ def format_for_print(df):
         table.add_row(row[1:])
     return str(table)
 
+def get_roc_auc(y_test_true, y_test_pred, labels, class_labels_internal, model_output_dir):
+    from sklearn.preprocessing import label_binarize
+    from sklearn.metrics import roc_curve, auc
+
+    from itertools import cycle
+        
+    y_test_true_binalized = label_binarize(y_test_true, classes=labels)
+    
+    if len(labels) == 2:
+        # binary classification
+        true_label_index = class_labels_internal.index(1)
+        y_test_pred = y_test_pred[:,true_label_index]
+        y_test_pred = np.reshape(y_test_pred, (-1, 1))
+        labels = labels[true_label_index:true_label_index+1]
+        n_classes = 1
+    else:
+        # multiclass classification
+        n_classes = len(labels)
+    
+    # Compute ROC curve and ROC area for each class
+    fpr = dict()
+    tpr = dict()
+    roc_auc = dict()
+
+    for i in range(n_classes):
+        fpr[i], tpr[i], _ = roc_curve(y_test_true_binalized[:, i], y_test_pred[:, i])
+        roc_auc[i] = auc(fpr[i], tpr[i])
+
+    # Compute micro-average ROC curve and ROC area
+    fpr["micro"], tpr["micro"], _ = roc_curve(y_test_true_binalized.ravel(), y_test_pred.ravel())
+    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
+    
+    sns.set(font_scale=1)
+    plt.figure()
+    lw = 2
+    colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
+
+    for i, color in zip(range(n_classes), colors):
+        plt.plot(fpr[i], tpr[i], color=color,
+                 lw=lw, label=f'ROC curve for {labels[i]} (area = %0.2f)' % roc_auc[i])
+    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
+    plt.xlim([0.0, 1.0])
+    plt.ylim([0.0, 1.05])
+    plt.xlabel('False Positive Rate')
+    plt.ylabel('True Positive Rate')
+    plt.title('Receiver operating characteristic example')
+    plt.legend(loc="lower right")
+    plt.show()
+    plt.savefig(f'{model_output_dir}/roc_auc_curve.png')
+    
 def train(args):
+    model_output_dir = f'{args.output_dir}/data'
+    
     is_distributed = len(args.hosts) > 1
     host_rank = args.hosts.index(args.current_host)
     dist_ip_addrs = args.hosts
@@ -74,7 +129,23 @@ def train(args):
     )
 
     # Results summary
-    predictor.fit_summary(verbosity=1)
+    predictor.fit_summary(verbosity=3)
+    model_summary_fname_src = os.path.join(predictor.output_directory, 'SummaryOfModels.html')
+    model_summary_fname_tgt = os.path.join(model_output_dir, 'SummaryOfModels.html')
+    
+    if os.path.exists(model_summary_fname_src):
+        shutil.copy(model_summary_fname_src, model_summary_fname_tgt)
+    
+    # ensemble visualization
+    G = predictor._trainer.model_graph
+    remove = [node for node,degree in dict(G.degree()).items() if degree < 1]
+    G.remove_nodes_from(remove)
+    A = nx.nx_agraph.to_agraph(G)
+    A.graph_attr.update(rankdir='BT')
+    A.node_attr.update(fontsize=10)
+    for node in A.iternodes():
+        node.attr['shape'] = 'rectagle'
+    A.draw(os.path.join(model_output_dir, 'ensemble-model.png'), format='png', prog='dot')
 
     # Optional test data
     if args.test:
@@ -86,6 +157,7 @@ def train(args):
             print('Running model on test data and getting Leaderboard...')
             leaderboard = predictor.leaderboard(dataset=test_data, silent=True)
             print(format_for_print(leaderboard), end='\n\n')
+            leaderboard.to_csv(f'{model_output_dir}/leaderboard.csv', index=False)
 
             # Feature importance on test data
             # Note: Feature importance must be calculated on held-out (test) data.
@@ -94,7 +166,36 @@ def train(args):
                 print('Feature importance:')
                 # Increase rows to print feature importance                
                 pd.set_option('display.max_rows', 500)
-                print(predictor.feature_importance(test_data))
+                feature_importance = predictor.feature_importance(test_data)
+                feature_importance_df = pd.DataFrame(feature_importance, columns=['Importance score']).rename_axis(index='Feature')
+                print(feature_importance_df)
+                feature_importance_df.to_csv(f'{model_output_dir}/feature_importance.csv', index=True)
+            
+            # Classification report and confusion matrix for classification model
+            if predictor.problem_type in [BINARY, MULTICLASS]:
+                from sklearn.metrics import classification_report, confusion_matrix
+                
+                X_test = test_data.drop(args.fit_args['label'], axis=1)
+                y_test_true = test_data[args.fit_args['label']]
+                y_test_pred = predictor.predict(X_test)
+                y_test_pred_prob = predictor.predict_proba(X_test, as_multiclass=True)
+                
+                report_dict = classification_report(y_test_true, y_test_pred, output_dict=True, labels=predictor.class_labels)
+                report_dict_df = pd.DataFrame(report_dict).T
+                report_dict_df.to_csv(f'{model_output_dir}/classification_report.csv', index=True)
+                
+                cm = confusion_matrix(y_test_true, y_test_pred, labels=predictor.class_labels)
+                cm_df = pd.DataFrame(cm, predictor.class_labels, predictor.class_labels)
+                sns.set(font_scale=1)
+                cmap = 'coolwarm'
+                sns.heatmap(cm_df, annot=True, fmt='d', cmap=cmap)
+                plt.title('Confusion Matrix')
+                plt.ylabel('true label')
+                plt.xlabel('predicted label')
+                plt.show()
+                plt.savefig(f'{model_output_dir}/confusion_matrix.png')
+                
+                get_roc_auc(y_test_true, y_test_pred_prob, predictor.class_labels, predictor.class_labels_internal, model_output_dir)
         else:
             warnings.warn('Skipping eval on test data since label column is not included.')
 
@@ -119,6 +220,7 @@ def parse_args():
     parser.add_argument('--current-host', type=str, default=os.environ['SM_CURRENT_HOST'])
     parser.add_argument('--num-gpus', type=int, default=os.environ['SM_NUM_GPUS'])
     parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR'])
+    parser.add_argument('--output-dir', type=str, default=os.environ['SM_OUTPUT_DIR'])
     parser.add_argument('--train', type=str, default=os.environ['SM_CHANNEL_TRAINING'])
     # Arguments to be passed to task.fit()
     parser.add_argument('--fit_args', type=lambda s: ast.literal_eval(s),
@@ -164,4 +266,5 @@ def parse_args():
     subprocess.call('cp columns.pkl /opt/ml/model/code/'.split())
 
     elapsed_time = round(timer()-start,3)
-    print(f'Elapsed time: {elapsed_time} seconds. Training Completed!')
+    print(f'Elapsed time: {elapsed_time} seconds. Training Completed!')
+