Add Horovod tests (#151)

Author: mvsusp

docker/1.12.0/Dockerfile.cpu

@@ -15,9 +15,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Install Open MPI
 RUN mkdir /tmp/openmpi && \
     cd /tmp/openmpi && \
-    curl -fSsL -O  https://www.open-mpi.org/software/ompi/v3.0/downloads/openmpi-3.0.0.tar.gz && \
-    tar zxf openmpi-3.0.0.tar.gz && \
-    cd openmpi-3.0.0 && \
+    curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.2.tar.gz && \
+    tar zxf openmpi-3.1.2.tar.gz && \
+    cd openmpi-3.1.2 && \
     ./configure --enable-orterun-prefix-by-default && \
     make -j $(nproc) all && \
     make install && \
@@ -76,7 +76,7 @@ COPY $framework_support_installable .
 
 RUN pip install --no-cache-dir -U \
         keras==2.2.4 \
-        sagemaker-containers==2.4.0 \
+        sagemaker-containers==2.4.2 \
         $framework_support_installable \
         "sagemaker-tensorflow>=1.12,<1.13" && \
     # Let's install TensorFlow separately in the end to avoid

docker/1.12.0/Dockerfile.gpu

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
         libnccl-dev=${NCCL_VERSION} \
         libgomp1 \
         wget \
+        openssh-client \
         openssh-server \
         build-essential && \
     # The 'apt-get install' of nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0
@@ -50,9 +51,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
 # Install Open MPI
 RUN mkdir /tmp/openmpi && \
     cd /tmp/openmpi && \
-    wget https://www.open-mpi.org/software/ompi/v3.0/downloads/openmpi-3.0.0.tar.gz && \
-    tar zxf openmpi-3.0.0.tar.gz && \
-    cd openmpi-3.0.0 && \
+    curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.2.tar.gz && \
+    tar zxf openmpi-3.1.2.tar.gz && \
+    cd openmpi-3.1.2 && \
     ./configure --enable-orterun-prefix-by-default && \
     make -j $(nproc) all && \
     make install && \
@@ -111,7 +112,7 @@ COPY $framework_support_installable .
 
 RUN pip install --no-cache-dir -U \
     keras==2.2.4 \
-    sagemaker-containers==2.4.0 \
+    sagemaker-containers==2.4.2 \
     $framework_support_installable \
     "sagemaker-tensorflow>=1.12,<1.13" \
     # Let's install TensorFlow separately in the end to avoid

test/integration/local/test_horovod.py

@@ -22,7 +22,6 @@
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 
 
-@pytest.mark.skip(reason="Horovod feature is not officially launched")
 @pytest.mark.parametrize('instances, processes', [
     [1, 2],
     (2, 1),

test/integration/sagemaker/test_horovod.py

@@ -14,19 +14,19 @@
 
 import os
 
-import pytest
 import sagemaker
 from sagemaker.tensorflow import TensorFlow
 
 RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
 
 
-@pytest.mark.skip(reason="Horovod feature is not officially launched")
 def test_distributed_training_horovod(sagemaker_session,
                                       sagemaker_local_session,
                                       instance_type,
                                       ecr_image,
                                       tmpdir):
+
+    mpi_options = '-verbose -x orte_base_help_aggregate=0'
     estimator = TensorFlow(
         entry_point=os.path.join(RESOURCE_PATH, 'mnist', 'horovod_mnist.py'),
         role='SageMakerRole',
@@ -37,7 +37,7 @@ def test_distributed_training_horovod(sagemaker_session,
         py_version='py3',
         script_mode=True,
         hyperparameters={'sagemaker_mpi_enabled': True,
-                         'sagemaker_mpi_custom_mpi_options': '-verbose',
+                         'sagemaker_mpi_custom_mpi_options': mpi_options,
                          'sagemaker_mpi_num_of_processes_per_host': 1})
 
     estimator.fit()

test/resources/mnist/horovod_mnist.py

@@ -13,6 +13,7 @@
 from __future__ import absolute_import, print_function
 
 import os
+import subprocess
 
 import keras
 from keras.datasets import mnist
@@ -23,6 +24,7 @@
 import tensorflow as tf
 import horovod.keras as hvd
 
+
 # Horovod: initialize Horovod.
 hvd.init()