change: merge dockerfiles

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.1.0 (2019-05-22)
+
+### Bug fixes and other changes
+
 ## v2.0.7 (2019-08-15)
 
 ### Bug fixes and other changes

docker/1.13.1/Dockerfile.cpu

@@ -9,7 +9,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     openssh-server \
     ca-certificates \
     curl \
-    && add-apt-repository ppa:deadsnakes/ppa -y \
+    git \
+    wget \
+    vim \
+    zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Open MPI
@@ -54,39 +57,54 @@ ENV KMP_AFFINITY=granularity=fine,compact,1,0 KMP_BLOCKTIME=1 KMP_SETTINGS=0
 
 WORKDIR /
 
-ARG py_version
-ARG framework_installable
-ARG framework_support_installable=sagemaker_tensorflow_container-2.0.0.tar.gz
+ARG PYTHON=python3
+ARG PYTHON_PIP=python3-pip
+ARG PIP=pip3
+ARG PYTHON_VERSION=3.6.6
 
-RUN if [ $py_version -eq 3 ]; then PYTHON_VERSION=python3.6; else PYTHON_VERSION=python2.7; fi && \
-        apt-get update && apt-get install -y --no-install-recommends $PYTHON_VERSION-dev --allow-unauthenticated  && \
-        ln -s -f /usr/bin/$PYTHON_VERSION /usr/bin/python && \
-        ln -s -f /usr/bin/$PYTHON_VERSION /usr/local/bin/python && \
-    rm -rf /var/lib/apt/lists/*
+RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \
+    tar -xvf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION && \
+    ./configure && make && make install && \
+    apt-get update && apt-get install -y --no-install-recommends libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \
+    make && make install && rm -rf ../Python-$PYTHON_VERSION* && \
+    ln -s /usr/local/bin/pip3 /usr/bin/pip
 
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8
 
-RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
-    python get-pip.py --disable-pip-version-check --no-cache-dir "pip==18.1" && \
-    rm get-pip.py
-
-COPY $framework_installable tensorflow-1.13.1-py2.py3-none-any.whl
+ARG framework_support_installable=sagemaker_tensorflow_container-2.0.0.tar.gz
 COPY $framework_support_installable .
-
-RUN pip install --no-cache-dir -U \
-        keras==2.2.4 \
-        mpi4py==3.0.1 \
-        "sagemaker-tensorflow>=1.13,<1.14" && \
+ARG TF_URL="https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl"
+
+RUN ${PIP} --no-cache-dir install --upgrade pip setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
+
+RUN ${PIP} install --no-cache-dir -U \
+           numpy==1.16.2 \
+           scipy==1.2.1 \
+           scikit-learn==0.20.3 \
+           pandas==0.24.2 \
+           Pillow==5.4.1 \
+           h5py==2.9.0 \
+           keras_applications==1.0.7 \
+           keras_preprocessing==1.0.9 \
+           keras==2.2.4 \
+           requests==2.21.0 \
+           awscli==1.16.130 \
+           mpi4py==3.0.1 \
+           "sagemaker-tensorflow>=1.13,<1.14" && \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
-    pip install --force-reinstall --no-cache-dir -U \
-        tensorflow-1.13.1-py2.py3-none-any.whl \
-        horovod && \
-    pip install --no-cache-dir -U $framework_support_installable && \
-    rm -f tensorflow-1.13.1-py2.py3-none-any.whl && \
-    rm -f $framework_support_installable && \
-    pip uninstall -y --no-cache-dir \
-        markdown \
-        tensorboard
+    ${PIP} install --force-reinstall --no-cache-dir -U \
+           ${TF_URL} \
+           horovod==0.16.4 && \
+    ${PIP} install --no-cache-dir -U $framework_support_installable && \
+           rm -f $framework_support_installable && \
+    ${PIP} uninstall -y --no-cache-dir \
+           markdown \
+           tensorboard
 
 ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main
+
+CMD ["bin/bash"]

docker/1.13.1/Dockerfile.gpu

@@ -2,11 +2,6 @@ FROM nvidia/cuda:10.0-base-ubuntu16.04
 
 LABEL maintainer="Amazon AI"
 
-RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
-        software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y && \
-    rm -rf /var/lib/apt/lists/*
-
 RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
         ca-certificates \
         cuda-command-line-tools-10-0 \
@@ -17,18 +12,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
         cuda-cusolver-dev-10-0 \
         cuda-cusparse-dev-10-0 \
         curl \
-        libcudnn7=7.4.1.5-1+cuda10.0 \
+        libcudnn7=7.5.1.10-1+cuda10.0 \
         # TensorFlow doesn't require libnccl anymore but Open MPI still depends on it
-        libnccl2 \
-        libnccl-dev \
+        libnccl2=2.4.7-1+cuda10.0 \
+        libgomp1 \
+        libnccl-dev=2.4.7-1+cuda10.0 \
         libfreetype6-dev \
         libhdf5-serial-dev \
         libpng12-dev \
         libzmq3-dev \
+        git \
         wget \
+        vim \
+        build-essential \
         openssh-client \
         openssh-server \
-        build-essential && \
+        zlib1g-dev && \
     # The 'apt-get install' of nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0
     # adds a new list which contains libnvinfer library, so it needs another
     # 'apt-get update' to retrieve that list before it can actually install the
@@ -42,7 +41,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
     rm /usr/lib/x86_64-linux-gnu/libnvinfer_plugin* && \
     rm /usr/lib/x86_64-linux-gnu/libnvcaffe_parser* && \
     rm /usr/lib/x86_64-linux-gnu/libnvparsers* && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir -p /var/run/sshd
 
 ###########################################################################
 # Horovod & its dependencies
@@ -60,14 +60,17 @@ RUN mkdir /tmp/openmpi && \
     ldconfig && \
     rm -rf /tmp/openmpi
 
-ARG py_version
-ARG framework_installable
-ARG framework_support_installable=sagemaker_tensorflow_container-2.0.0.tar.gz
+ARG PYTHON=python3
+ARG PYTHON_PIP=python3-pip
+ARG PIP=pip3
+ARG PYTHON_VERSION=3.6.6
 
-RUN if [ $py_version -eq 3 ]; then PYTHON_VERSION=python3.6; else PYTHON_VERSION=python2.7; fi && \
-        apt-get update && apt-get install -y --no-install-recommends $PYTHON_VERSION-dev --allow-unauthenticated  && \
-        ln -s -f /usr/bin/$PYTHON_VERSION /usr/bin/python && \
-    rm -rf /var/lib/apt/lists/*
+RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \
+    tar -xvf Python-$PYTHON_VERSION.tgz && cd Python-$PYTHON_VERSION && \
+    ./configure && make && make install && \
+    apt-get update && apt-get install -y --no-install-recommends libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \
+    make && make install && rm -rf ../Python-$PYTHON_VERSION* && \
+    ln -s /usr/local/bin/pip3 /usr/bin/pip
 
 # Create a wrapper for OpenMPI to allow running as root by default
 RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
@@ -100,33 +103,51 @@ RUN mkdir -p /root/.ssh/ && \
 # Python won’t try to write .pyc or .pyo files on the import of source modules
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8
 
-RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
-    python get-pip.py --disable-pip-version-check --no-cache-dir "pip==18.1" && \
-    rm get-pip.py
-
 WORKDIR /
 
-COPY $framework_installable tensorflow-1.13.1-py2.py3-none-any.whl
+ARG TF_URL="https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl"
+
+RUN ${PIP} --no-cache-dir install --upgrade pip setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
+
+ARG framework_support_installable=sagemaker_tensorflow_container-2.0.0.tar.gz
 COPY $framework_support_installable .
 
-RUN pip install --no-cache-dir -U \
+RUN ${PIP} install --no-cache-dir -U \
+    numpy==1.16.2 \
+    scipy==1.2.1 \
+    scikit-learn==0.20.3 \
+    pandas==0.24.2 \
+    Pillow==5.4.1 \
+    h5py==2.9.0 \
+    keras_applications==1.0.7 \
+    keras_preprocessing==1.0.9 \
+    requests==2.21.0 \
     keras==2.2.4 \
+    awscli==1.16.130 \
     mpi4py==3.0.1 \
-    $framework_support_installable \
     "sagemaker-tensorflow>=1.13,<1.14" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
-    && pip install --force-reinstall --no-cache-dir -U tensorflow-1.13.1-py2.py3-none-any.whl \
-    \
-    && rm -f tensorflow-1.13.1-py2.py3-none-any.whl  \
-    && rm -f $framework_support_installable \
-    && pip uninstall -y --no-cache-dir \
+    && ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
+    && ${PIP} install --no-cache-dir -U $framework_support_installable && \
+           rm -f $framework_support_installable \
+    && ${PIP} uninstall -y --no-cache-dir \
     markdown \
     tensorboard
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs && \
-    HOROVOD_GPU_ALLREDUCE=NCCL  HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horovod && \
+    HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_TENSORFLOW=1 ${PIP} install --no-cache-dir horovod==0.16.4 && \
     ldconfig
 
-ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main
+# Allow OpenSSH to talk to containers without asking for confirmation
+RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
+    echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
+    mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
+
+ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main
+
+CMD ["bin/bash"]

docker/1.14.0/py2/Dockerfile.cpu

@@ -0,0 +1,127 @@
+FROM ubuntu:16.04
+
+LABEL maintainer="Amazon AI"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common \
+    build-essential \
+    openssh-client \
+    openssh-server \
+    ca-certificates \
+    curl \
+    git \
+    wget \
+    vim \
+    gcc-4.9 \
+    g++-4.9 \
+    gcc-4.9-base \
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Open MPI
+RUN mkdir /tmp/openmpi && \
+    cd /tmp/openmpi && \
+    curl -fSsL -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.1.tar.gz && \
+    tar zxf openmpi-4.0.1.tar.gz && \
+    cd openmpi-4.0.1 && \
+    ./configure --enable-orterun-prefix-by-default && \
+    make -j $(nproc) all && \
+    make install && \
+    ldconfig && \
+    rm -rf /tmp/openmpi
+
+# Create a wrapper for OpenMPI to allow running as root by default
+RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
+    echo '#!/bin/bash' > /usr/local/bin/mpirun && \
+    echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
+    chmod a+x /usr/local/bin/mpirun
+
+RUN echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf && \
+    echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf
+
+ENV LD_LIBRARY_PATH=/usr/local/openmpi/lib:$LD_LIBRARY_PATH
+
+ENV PATH /usr/local/openmpi/bin/:$PATH
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+# Create SSH key.
+RUN mkdir -p /root/.ssh/ && \
+    mkdir -p /var/run/sshd && \
+    ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \
+    cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
+    printf "Host *\n  StrictHostKeyChecking no\n" >> /root/.ssh/config
+
+# Set environment variables for MKL
+# For more about MKL with TensorFlow see:
+# https://www.tensorflow.org/performance/performance_guide#tensorflow_with_intel%C2%AE_mkl_dnn
+ENV KMP_AFFINITY=granularity=fine,compact,1,0 KMP_BLOCKTIME=1 KMP_SETTINGS=0
+
+WORKDIR /
+
+ARG PYTHON=python
+ARG PYTHON_PIP=python-pip
+ARG PIP=pip
+
+RUN apt-get update && apt-get install -y \
+    ${PYTHON} \
+    ${PYTHON_PIP}
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+ARG framework_support_installable=sagemaker_tensorflow_container-2.0.0.tar.gz
+ARG sagemaker_tensorflow_extensions=sagemaker_tensorflow-1.14.0.1.0.0-cp27-cp27mu-manylinux1_x86_64.whl
+COPY $framework_support_installable .
+COPY $sagemaker_tensorflow_extensions .
+ARG TF_URL="https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.14/AmazonLinux/cpu/final/tensorflow-1.14.0-cp27-cp27mu-linux_x86_64.whl"
+
+# Pin GCC to 4.9 (priority 200) to compile correctly against TensorFlow, PyTorch, and MXNet with horovod
+# Backup existing GCC installation as priority 100, so that it can be recovered later.
+RUN update-alternatives --install /usr/bin/gcc gcc $(readlink -f $(which gcc)) 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc $(readlink -f $(which gcc)) 100 && \
+    update-alternatives --install /usr/bin/g++ g++ $(readlink -f $(which g++)) 100 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ $(readlink -f $(which g++)) 100
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 200 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-4.9 200 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 200 && \
+    update-alternatives --install /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/g++-4.9 200
+
+RUN ${PIP} --no-cache-dir install --upgrade pip setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
+
+RUN ${PIP} install --no-cache-dir -U \
+            numpy==1.16.4 \
+            scipy==1.2.2 \
+            scikit-learn==0.20.3 \
+            pandas==0.24.2 \
+            Pillow==6.1.0 \
+            h5py==2.9.0 \
+            keras_applications==1.0.8 \
+            keras_preprocessing==1.1.0 \
+            requests==2.22.0 \
+            keras==2.2.4 \
+            awscli==1.16.196 \
+            mpi4py==3.0.2 \
+            $sagemaker_tensorflow_extensions \
+            # Let's install TensorFlow separately in the end to avoid
+            # the library version to be overwritten
+    && ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
+    && ${PIP} install --no-cache-dir -U $framework_support_installable && \
+        rm -f $framework_support_installable \
+    && ${PIP} install --no-cache-dir -U horovod==0.16.4 \
+    && ${PIP} uninstall -y --no-cache-dir \
+        markdown
+
+# Remove GCC pinning
+RUN update-alternatives --remove gcc /usr/bin/gcc-4.9 && \
+    update-alternatives --remove x86_64-linux-gnu-gcc /usr/bin/gcc-4.9 && \
+    update-alternatives --remove g++ /usr/bin/g++-4.9 && \
+    update-alternatives --remove x86_64-linux-gnu-g++ /usr/bin/g++-4.9
+
+
+ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main
+
+CMD ["bin/bash"]