update

Author: clee2000

.ci/docker/Dockerfile

@@ -9,10 +9,6 @@ ENV DEBIAN_FRONTEND noninteractive
 COPY ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
-# Install user
-COPY ./common/install_user.sh install_user.sh
-RUN bash ./install_user.sh && rm install_user.sh
-
 COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
 RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
 
@@ -26,10 +22,4 @@ COPY ./common/install_conda.sh install_conda.sh
 COPY ./common/common_utils.sh common_utils.sh
 RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements.txt
 
-# Install CUDNN
-ENV CUDNN_VERSION=8
-COPY ./common/install_cudnn.sh install_cudnn.sh
-RUN bash ./install_cudnn.sh && rm install_cudnn.sh
-
-USER jenkins
 CMD ["bash"]

.ci/docker/build.sh

@@ -13,7 +13,7 @@ shift
 export CUDA_VERSION="12.1.1"
 export UBUNTU_VERSION="20.04"
 
-export BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
+export BASE_IMAGE="ubuntu:${UBUNTU_VERSION}"
 echo "Building ${IMAGE_NAME} Docker image"
 
 docker build \

.ci/docker/common/common_utils.sh

@@ -1,26 +1,26 @@
 #!/bin/bash
 
 # Work around bug where devtoolset replaces sudo and breaks it.
-as_jenkins() {
+as_ci_user() {
   # NB: unsetting the environment variables works around a conda bug
   # https://github.com/conda/conda/issues/6576
   # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
-  # NB: This must be run from a directory that jenkins has access to,
+  # NB: This must be run from a directory that the user has access to,
   # works around https://github.com/conda/conda-package-handling/pull/34
-  sudo -E -H -u jenkins env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
+  sudo -E -H env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
 }
 
 conda_install() {
   # Ensure that the install command don't upgrade/downgrade Python
   # This should be called as
   #   conda_install pkg1 pkg2 ... [-c channel]
-  as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
+  as_ci_user conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
 }
 
 conda_run() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
+  as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
 }
 
 pip_install() {
-  as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
+  as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
 }

.ci/docker/common/install_conda.sh

@@ -12,14 +12,13 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
 
   mkdir -p /opt/conda
-  chown jenkins:jenkins /opt/conda
 
   source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
 
   pushd /tmp
   wget -q "${BASE_URL}/${CONDA_FILE}"
   # NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
-  as_jenkins bash "${CONDA_FILE}" -b -f -p "/opt/conda"
+  as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
   popd
 
   # NB: Don't do this, rely on the rpath to get it right
@@ -28,46 +27,28 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
   export PATH="/opt/conda/bin:$PATH"
 
-  # Ensure we run conda in a directory that jenkins has write access to
+  # Ensure we run conda in a directory that the user has write access to
   pushd /opt/conda
 
   # Prevent conda from updating to 4.14.0, which causes docker build failures
   # See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
   # Uncomment the below when resolved to track the latest conda update
-  # as_jenkins conda update -y -n base conda
+  # as_ci_user conda update -y -n base conda
 
   # Install correct Python version
-  as_jenkins conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
-
-  # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
-  CONDA_COMMON_DEPS="astunparse pyyaml mkl=2021.4.0 mkl-include=2021.4.0 setuptools"
-
-  # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-  # and libpython-static for torch deploy
-  conda_install llvmdev=8.0.0 "libpython-static=${ANACONDA_PYTHON_VERSION}"
+  as_ci_user conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
 
   # Use conda cmake in some cases. Conda cmake will be newer than our supported
   # min version (3.5 for xenial and 3.10 for bionic), so we only do it in those
   # following builds that we know should use conda. Specifically, Ubuntu bionic
   # and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
   conda_install cmake
 
-  conda_install "magma-cuda$(TMP=${CUDA_VERSION/./};echo ${TMP%.*[0-9]})" -c pytorch
-
-  # Install some other packages, including those needed for Python test reporting
+  # Install pip packages
   pip_install -r /opt/conda/requirements.txt
 
   apt-get update
   apt-get -y install expect-dev
 
-  # HACK HACK HACK
-  # gcc-9 for ubuntu-18.04 from http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu
-  # Pulls llibstdc++6 13.1.0-8ubuntu1~18.04 which is too new for conda
-  # So remove libstdc++6.so.3.29 installed by https://anaconda.org/anaconda/libstdcxx-ng/files?version=11.2.0
-  # Same is true for gcc-12 from Ubuntu-22.04
-  if grep -e [12][82].04.[623] /etc/issue >/dev/null; then
-    rm /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/lib/libstdc++.so.6
-  fi
-
   popd
 fi

.ci/docker/common/install_cudnn.sh

@@ -4,8 +4,12 @@
 sphinx==5.0.0
 sphinx-gallery==0.11.1
 sphinx_design
+nbsphinx
 docutils==0.16
 sphinx-copybutton
+pypandoc==1.12
+pandocfilters
+markdown
 tqdm==4.66.1
 numpy==1.24.4
 matplotlib
@@ -28,7 +32,7 @@ torchx
 torchrl==0.3.0
 tensordict==0.3.0
 ax-platform
-nbformat>=4.2.0
+nbformat>==5.9.2
 datasets
 transformers
 torchmultimodal-nightly # needs to be updated to stable as soon as it's avaialable

.ci/docker/common/install_user.sh

@@ -86,11 +86,10 @@ jobs:
             --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
             --tty \
             --detach \
-            --user jenkins \
             --shm-size=2gb \
             --name="${container_name}" \
-            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
-            -w /var/lib/jenkins/workspace \
+            -v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
+            -w /var/lib/workspace \
             "${DOCKER_IMAGE}"
           )
 
@@ -157,10 +156,9 @@ jobs:
             --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
             --tty \
             --detach \
-            --user jenkins \
             --name="${container_name}" \
-            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
-            -w /var/lib/jenkins/workspace \
+            -v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
+            -w /var/lib/workspace \
             "${DOCKER_IMAGE}"
           )