Skip to content

Add Horovod tests #151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jan 10, 2019
Merged

Add Horovod tests #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3c34058
Update sagemaker containers
mvsusp Jan 9, 2019
350a1b8
Exclude docker interface
mvsusp Jan 10, 2019
9f846aa
Update openmpi version
mvsusp Jan 10, 2019
4298879
Added info for debugging
mvsusp Jan 10, 2019
34d036f
Fix mpi options
mvsusp Jan 10, 2019
dca6427
Testing newer sagemaker-containers version
mvsusp Jan 10, 2019
5c1705d
New test run
mvsusp Jan 10, 2019
80f752c
Updated sagemaker containers version
mvsusp Jan 10, 2019
63ecdf9
Update version
mvsusp Jan 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docker/1.12.0/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
# Install Open MPI
RUN mkdir /tmp/openmpi && \
cd /tmp/openmpi && \
curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.0/downloads/openmpi-3.0.0.tar.gz && \
tar zxf openmpi-3.0.0.tar.gz && \
cd openmpi-3.0.0 && \
curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.2.tar.gz && \
tar zxf openmpi-3.1.2.tar.gz && \
cd openmpi-3.1.2 && \
./configure --enable-orterun-prefix-by-default && \
make -j $(nproc) all && \
make install && \
Expand Down Expand Up @@ -76,7 +76,7 @@ COPY $framework_support_installable .

RUN pip install --no-cache-dir -U \
keras==2.2.4 \
sagemaker-containers==2.4.0 \
sagemaker-containers==2.4.2 \
$framework_support_installable \
"sagemaker-tensorflow>=1.12,<1.13" && \
# Let's install TensorFlow separately in the end to avoid
Expand Down
9 changes: 5 additions & 4 deletions docker/1.12.0/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
libnccl-dev=${NCCL_VERSION} \
libgomp1 \
wget \
openssh-client \
openssh-server \
build-essential && \
# The 'apt-get install' of nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0
Expand All @@ -50,9 +51,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
# Install Open MPI
RUN mkdir /tmp/openmpi && \
cd /tmp/openmpi && \
wget https://www.open-mpi.org/software/ompi/v3.0/downloads/openmpi-3.0.0.tar.gz && \
tar zxf openmpi-3.0.0.tar.gz && \
cd openmpi-3.0.0 && \
curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.2.tar.gz && \
tar zxf openmpi-3.1.2.tar.gz && \
cd openmpi-3.1.2 && \
./configure --enable-orterun-prefix-by-default && \
make -j $(nproc) all && \
make install && \
Expand Down Expand Up @@ -111,7 +112,7 @@ COPY $framework_support_installable .

RUN pip install --no-cache-dir -U \
keras==2.2.4 \
sagemaker-containers==2.4.0 \
sagemaker-containers==2.4.2 \
$framework_support_installable \
"sagemaker-tensorflow>=1.12,<1.13" \
# Let's install TensorFlow separately in the end to avoid
Expand Down
1 change: 0 additions & 1 deletion test/integration/local/test_horovod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')


@pytest.mark.skip(reason="Horovod feature is not officially launched")
@pytest.mark.parametrize('instances, processes', [
[1, 2],
(2, 1),
Expand Down
6 changes: 3 additions & 3 deletions test/integration/sagemaker/test_horovod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,19 @@

import os

import pytest
import sagemaker
from sagemaker.tensorflow import TensorFlow

RESOURCE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')


@pytest.mark.skip(reason="Horovod feature is not officially launched")
def test_distributed_training_horovod(sagemaker_session,
sagemaker_local_session,
instance_type,
ecr_image,
tmpdir):

mpi_options = '-verbose -x orte_base_help_aggregate=0'
estimator = TensorFlow(
entry_point=os.path.join(RESOURCE_PATH, 'mnist', 'horovod_mnist.py'),
role='SageMakerRole',
Expand All @@ -37,7 +37,7 @@ def test_distributed_training_horovod(sagemaker_session,
py_version='py3',
script_mode=True,
hyperparameters={'sagemaker_mpi_enabled': True,
'sagemaker_mpi_custom_mpi_options': '-verbose',
'sagemaker_mpi_custom_mpi_options': mpi_options,
'sagemaker_mpi_num_of_processes_per_host': 1})

estimator.fit()
Expand Down
2 changes: 2 additions & 0 deletions test/resources/mnist/horovod_mnist.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from __future__ import absolute_import, print_function

import os
import subprocess

import keras
from keras.datasets import mnist
Expand All @@ -23,6 +24,7 @@
import tensorflow as tf
import horovod.keras as hvd


# Horovod: initialize Horovod.
hvd.init()

Expand Down