Skip to content

Commit 5517cc2

Browse files
Vikas-kumVikas-kum
authored
Update version and add build script (aws#88)
Fixing bugs in import and Update version and add build script
2 parents efd600e + e234f4b commit 5517cc2

File tree

10 files changed

+183
-39
lines changed

10 files changed

+183
-39
lines changed

‎bin/build_binaries.sh

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
#!/usr/bin/env bash
2-
3-
VERSION = '0.2.1'
1+
export VERSION='0.3'
42

53
for FRAMEWORK in tensorflow mxnet pytorch
64
do
75
CAPITALIZED_FRAMEWORK=`echo "$FRAMEWORK" | tr '[a-z]' '[A-Z]'`
8-
TORNASOLE_WITH_$CAPITALIZED_FRAMEWORK=1 python setup.py bdist_wheel --universal
9-
# aws s3 cp dist/tornasole-$VERSION-py2.py3-none-any.whl s3://tornasole-binaries-use1/tornasole_$FRAMEWORK/py3/
10-
done
6+
export TORNASOLE_WITH_$CAPITALIZED_FRAMEWORK=1
7+
python setup.py bdist_wheel --universal
8+
unset TORNASOLE_WITH_$CAPITALIZED_FRAMEWORK
9+
# aws s3 cp dist/tornasole-$VERSION-py2.py3-none-any.whl s3://tornasole-binaries-use1/tornasole_$FRAMEWORK/py3/
10+
rm -rf dist build *.egg-info
11+
done

‎bin/sagemaker_containers/tf/1.13.1.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
# run from tornasole_core folder
3+
set -ex
4+
5+
export TF_BINARY_LOCATION=https://files.pythonhosted.org/packages/77/63/a9fa76de8dffe7455304c4ed635be4aa9c0bacef6e0633d87d5f54530c5c/tensorflow-1.13.1-cp36-cp36m-manylinux1_x86_64.whl
6+
export TF_BINARY_NAME=tensorflow-1.13.1-cp36-cp36m-manylinux1_x86_64.whl
7+
export TORNASOLE_BINARY_PATH=s3://tornasole-binaries-use1/tornasole_tensorflow/py3/tornasole-0.3-py2.py3-none-any.whl
8+
export TORNASOLE_BINARY_NAME=tornasole-0.3-py2.py3-none-any.whl
9+
export ECR_REPO_NAME=tornasole-preprod-tf-1.13.1-gpu
10+
export ECR_TAG_NAME=latest
11+
12+
pushd .
13+
cd ~
14+
git clone https://github.com/aws/sagemaker-tensorflow-container.git
15+
cd sagemaker-tensorflow-container
16+
git checkout script-mode
17+
python setup.py sdist
18+
popd
19+
cp ~/sagemaker-tensorflow-container/dist/sagemaker_tensorflow_container-*.tar.gz bin/sagemaker_tf_container/
20+
cd bin/sagemaker_tf_container/
21+
aws s3 cp $TORNASOLE_BINARY_PATH .
22+
curl -O $TF_BINARY_LOCATION
23+
$(aws ecr get-login --no-include-email --region us-east-1)
24+
docker build -t $ECR_REPO_NAME --build-arg py_version=3 \
25+
--build-arg framework_installable=$TF_BINARY_NAME \
26+
--build-arg framework_support_installable=sagemaker_tensorflow_container-*.tar.gz \
27+
--build-arg tornasole_framework_installable=$TORNASOLE_BINARY_NAME \
28+
-f Dockerfile.gpu .
29+
docker tag $ECR_REPO_NAME:$ECR_TAG_NAME 072677473360.dkr.ecr.us-east-1.amazonaws.com/$ECR_REPO_NAME:$ECR_TAG_NAME
30+
docker push 072677473360.dkr.ecr.us-east-1.amazonaws.com/$ECR_REPO_NAME:$ECR_TAG_NAME

‎bin/sagemaker_containers/tf/Dockerfile.gpu

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
FROM nvidia/cuda:10.0-base-ubuntu16.04
2+
3+
LABEL maintainer="Amazon AI"
4+
5+
RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
6+
software-properties-common && \
7+
add-apt-repository ppa:deadsnakes/ppa -y && \
8+
rm -rf /var/lib/apt/lists/*
9+
10+
RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
11+
ca-certificates \
12+
cuda-command-line-tools-10-0 \
13+
cuda-cublas-dev-10-0 \
14+
cuda-cudart-dev-10-0 \
15+
cuda-cufft-dev-10-0 \
16+
cuda-curand-dev-10-0 \
17+
cuda-cusolver-dev-10-0 \
18+
cuda-cusparse-dev-10-0 \
19+
curl \
20+
libcudnn7=7.4.1.5-1+cuda10.0 \
21+
# TensorFlow doesn't require libnccl anymore but Open MPI still depends on it
22+
libnccl2 \
23+
libnccl-dev \
24+
libfreetype6-dev \
25+
libhdf5-serial-dev \
26+
libpng12-dev \
27+
libzmq3-dev \
28+
wget \
29+
openssh-client \
30+
openssh-server \
31+
build-essential && \
32+
# The 'apt-get install' of nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0
33+
# adds a new list which contains libnvinfer library, so it needs another
34+
# 'apt-get update' to retrieve that list before it can actually install the
35+
# library.
36+
# We don't install libnvinfer-dev since we don't need to build against TensorRT,
37+
# and libnvinfer4 doesn't contain libnvinfer.a static library.
38+
apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
39+
nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 && \
40+
apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
41+
libnvinfer5=5.0.2-1+cuda10.0 && \
42+
rm /usr/lib/x86_64-linux-gnu/libnvinfer_plugin* && \
43+
rm /usr/lib/x86_64-linux-gnu/libnvcaffe_parser* && \
44+
rm /usr/lib/x86_64-linux-gnu/libnvparsers* && \
45+
rm -rf /var/lib/apt/lists/*
46+
47+
###########################################################################
48+
# Horovod & its dependencies
49+
###########################################################################
50+
51+
# Install Open MPI
52+
RUN mkdir /tmp/openmpi && \
53+
cd /tmp/openmpi && \
54+
curl -fSsL -O https://www.open-mpi.org/software/ompi/v3.1/downloads/openmpi-3.1.2.tar.gz && \
55+
tar zxf openmpi-3.1.2.tar.gz && \
56+
cd openmpi-3.1.2 && \
57+
./configure --enable-orterun-prefix-by-default && \
58+
make -j $(nproc) all && \
59+
make install && \
60+
ldconfig && \
61+
rm -rf /tmp/openmpi
62+
63+
ARG py_version
64+
ARG framework_installable
65+
ARG framework_support_installable
66+
67+
RUN if [ $py_version -eq 3 ]; then PYTHON_VERSION=python3.6; else PYTHON_VERSION=python2.7; fi && \
68+
apt-get update && apt-get install -y --no-install-recommends $PYTHON_VERSION-dev --allow-unauthenticated && \
69+
ln -s -f /usr/bin/$PYTHON_VERSION /usr/bin/python && \
70+
rm -rf /var/lib/apt/lists/*
71+
72+
# Create a wrapper for OpenMPI to allow running as root by default
73+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
74+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
75+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
76+
chmod a+x /usr/local/bin/mpirun
77+
78+
# Configure OpenMPI to run good defaults:
79+
# --bind-to none --map-by slot --mca btl_tcp_if_exclude lo,docker0
80+
RUN echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf && \
81+
echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf
82+
83+
# Set default NCCL parameters
84+
RUN echo NCCL_DEBUG=INFO >> /etc/nccl.conf
85+
86+
ENV LD_LIBRARY_PATH=/usr/local/openmpi/lib:$LD_LIBRARY_PATH
87+
ENV PATH /usr/local/openmpi/bin/:$PATH
88+
ENV PATH=/usr/local/nvidia/bin:$PATH
89+
90+
# SSH login fix. Otherwise user is kicked off after login
91+
RUN mkdir -p /var/run/sshd && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
92+
93+
# Create SSH key.
94+
RUN mkdir -p /root/.ssh/ && \
95+
ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \
96+
cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
97+
printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config
98+
99+
###########################################################################
100+
# Python won’t try to write .pyc or .pyo files on the import of source modules
101+
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8
102+
103+
RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
104+
python get-pip.py --disable-pip-version-check --no-cache-dir "pip==18.1" && \
105+
rm get-pip.py
106+
107+
WORKDIR /
108+
109+
COPY $framework_installable tensorflow-1.13.1-py2.py3-none-any.whl
110+
COPY $framework_support_installable .
111+
112+
RUN pip install --no-cache-dir -U \
113+
keras==2.2.4 \
114+
mpi4py==3.0.1 \
115+
$framework_support_installable \
116+
"sagemaker-tensorflow>=1.13,<1.14" \
117+
# Let's install TensorFlow separately in the end to avoid
118+
# the library version to be overwritten
119+
&& pip install --force-reinstall --no-cache-dir -U tensorflow-1.13.1-py2.py3-none-any.whl \
120+
\
121+
&& rm -f tensorflow-1.13.1-py2.py3-none-any.whl \
122+
&& rm -f $framework_support_installable \
123+
&& pip uninstall -y --no-cache-dir \
124+
markdown \
125+
tensorboard
126+
127+
# Install Horovod, temporarily using CUDA stubs
128+
RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs && \
129+
HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horovod && \
130+
ldconfig
131+
132+
ARG tornasole_framework_installable
133+
COPY $tornasole_framework_installable .
134+
135+
RUN pip install $tornasole_framework_installable
136+
137+
ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main

‎docs/mxnet/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ You can activate this by doing: `source activate mxnet_p36`.
4343
Once your account is whitelisted, you should be able to install the `tornasole` package built for MXNet as follows:
4444

4545
```
46-
aws s3 cp s3://tornasole-binaries-use1/tornasole_mxnet/py3/tornasole-0.2.1-py2.py3-none-any.whl .
47-
pip install tornasole-0.2.1-py2.py3-none-any.whl
46+
aws s3 cp s3://tornasole-binaries-use1/tornasole_mxnet/py3/tornasole-0.3-py2.py3-none-any.whl .
47+
pip install tornasole-0.3-py2.py3-none-any.whl
4848
```
4949

5050
**Please note** : If, while installing tornasole, you get a version conflict issue between botocore and boto3,

‎docs/pytorch/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ You can activate this by doing: `source activate pytorch_p36`.
4343
Once your account is whitelisted, you should be able to install the `tornasole` package built for PyTorch as follows:
4444

4545
```
46-
aws s3 cp s3://tornasole-binaries-use1/tornasole_pytorch/py3/tornasole-0.2.1-py2.py3-none-any.whl .
47-
pip install tornasole-0.2.1-py2.py3-none-any.whl
46+
aws s3 cp s3://tornasole-binaries-use1/tornasole_pytorch/py3/tornasole-0.3-py2.py3-none-any.whl .
47+
pip install tornasole-0.3-py2.py3-none-any.whl
4848
```
4949

5050
**Please note** : If, while installing tornasole, you get a version conflict issue between botocore and boto3,

‎docs/tensorflow/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ You can activate this by doing: `source activate tensorflow_p36`.
4343
Once your account is whitelisted, you should be able to install the `tornasole` package built for TensorFlow as follows:
4444

4545
```
46-
aws s3 cp s3://tornasole-binaries-use1/tornasole_tensorflow/py3/tornasole-0.2.1-py2.py3-none-any.whl .
47-
pip install tornasole-0.2.1-py2.py3-none-any.whl
46+
aws s3 cp s3://tornasole-binaries-use1/tornasole_tensorflow/py3/tornasole-0.3-py2.py3-none-any.whl .
47+
pip install tornasole-0.3-py2.py3-none-any.whl
4848
```
4949

5050
**Please note** : If, while installing tornasole, you get a version conflict issue between botocore and boto3,

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import sys
33
import setuptools
44

5-
CURRENT_VERSION = '0.2.1'
5+
CURRENT_VERSION = '0.3'
66
FRAMEWORKS = ['tensorflow', 'pytorch', 'mxnet']
77

88
def compile_summary_protobuf():

‎tornasole/pytorch/setup.py

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎tornasole/trials/local_trial.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from tornasole.core.tfevent.util import EventFileLocation
44
from tornasole.core.collection_manager import CollectionManager
55
from tornasole.core.reader import FileReader
6-
from tornasole_core.access_layer.utils import has_training_ended
6+
from tornasole.core.access_layer.utils import has_training_ended
77

88
import time
99
import os

‎tornasole/trials/s3_trial.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import os
33

44
from tornasole.core.access_layer.s3handler import ReadObjectRequest, ListRequest, S3Handler
5-
from tornasole_core.access_layer.utils import has_training_ended
5+
from tornasole.core.access_layer.utils import has_training_ended
66
from tornasole.core.tfevent.util import EventFileLocation
77
from tornasole.core.collection_manager import CollectionManager
88
from tornasole.core.tfrecord.tensor_reader import TensorReader

0 commit comments

Comments
 (0)