Merge pull request #1040 from ROCmSoftwarePlatform/IFU-master-2022-07-11

IFU master 2022 07 11

Author: jithunnair-amd

.bazelrc

@@ -31,6 +31,10 @@ build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
 # rules_cuda configuration
 build:cpu-only --@rules_cuda//cuda:enable_cuda=False
 
+# Definition of --config=shell
+# interactive shell immediately before execution
+build:shell --run_under="//tools/bazel_tools:shellwrap"
+
 # Disable all warnings for external repositories. We don't care about
 # their warnings.
 build --per_file_copt=^external/@-w

.buckconfig.oss

@@ -3,16 +3,23 @@
 
 [buildfile]
   name = BUCK.oss
+  includes = //tools/build_defs/select.bzl
 
 [repositories]
   bazel_skylib = third_party/bazel-skylib/
+  ovr_config = .
 
 [download]
   in_build = true
 
 [cxx]
   cxxflags = -std=c++17
   should_remap_host_platform = true
+  cpp = /usr/bin/clang
+  cc = /usr/bin/clang
+  cxx = /usr/bin/clang++
+  cxxpp = /usr/bin/clang++
+  ld = /usr/bin/clang++
 
 [project]
   default_flavors_mode=all

.circleci/cimodel/data/pytorch_build_data.py

@@ -75,6 +75,7 @@ def child_constructor(self):
             "vulkan": VulkanConfigNode,
             "parallel_tbb": ParallelTBBConfigNode,
             "crossref": CrossRefConfigNode,
+            "dynamo": DynamoConfigNode,
             "parallel_native": ParallelNativeConfigNode,
             "onnx": ONNXConfigNode,
             "libtorch": LibTorchConfigNode,
@@ -179,6 +180,14 @@ def child_constructor(self):
         return ImportantConfigNode
 
 
+class DynamoConfigNode(TreeConfigNode):
+    def init2(self, node_name):
+        self.props["is_dynamo"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
 class ParallelNativeConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELNATIVE=" + str(label)

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -240,6 +240,7 @@ def instantiate_configs(only_slow_gradcheck):
         is_xla = fc.find_prop("is_xla") or False
         is_asan = fc.find_prop("is_asan") or False
         is_crossref = fc.find_prop("is_crossref") or False
+        is_dynamo = fc.find_prop("is_dynamo") or False
         is_onnx = fc.find_prop("is_onnx") or False
         is_pure_torch = fc.find_prop("is_pure_torch") or False
         is_vulkan = fc.find_prop("is_vulkan") or False
@@ -286,6 +287,9 @@ def instantiate_configs(only_slow_gradcheck):
         if is_crossref:
             parms_list_ignored_for_docker_image.append("crossref")
 
+        if is_dynamo:
+            parms_list_ignored_for_docker_image.append("dynamo")
+
         if is_onnx:
             parms_list.append("onnx")
             python_version = fc.find_prop("pyver")

.circleci/docker/build.sh

@@ -54,6 +54,8 @@ elif [[ "$image" == *-bionic* ]]; then
   UBUNTU_VERSION=18.04
 elif [[ "$image" == *-focal* ]]; then
   UBUNTU_VERSION=20.04
+elif [[ "$image" == *-jammy* ]]; then
+  UBUNTU_VERSION=22.04
 elif [[ "$image" == *ubuntu* ]]; then
   extract_version_from_image_name ubuntu UBUNTU_VERSION
 elif [[ "$image" == *centos* ]]; then
@@ -70,7 +72,8 @@ else
 fi
 
 DOCKERFILE="${OS}/Dockerfile"
-if [[ "$image" == *cuda* ]]; then
+# When using ubuntu - 22.04, start from Ubuntu docker image, instead of nvidia/cuda docker image.
+if [[ "$image" == *cuda* && "$UBUNTU_VERSION" != "22.04" ]]; then
   DOCKERFILE="${OS}-cuda/Dockerfile"
 elif [[ "$image" == *rocm* ]]; then
   DOCKERFILE="${OS}-rocm/Dockerfile"
@@ -136,7 +139,7 @@ case "$image" in
     KATEX=yes
     ;;
   pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
-    CUDA_VERSION=11.6.0
+    CUDA_VERSION=11.6.2
     CUDNN_VERSION=8
     ANACONDA_PYTHON_VERSION=3.7
     GCC_VERSION=7
@@ -173,6 +176,13 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
+  pytorch-linux-focal-py3-clang10-onnx)
+    ANACONDA_PYTHON_VERSION=3.7
+    CLANG_VERSION=10
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
   pytorch-linux-xenial-py3-clang5-android-ndk-r19c)
     ANACONDA_PYTHON_VERSION=3.7
     CLANG_VERSION=5.0
@@ -224,15 +234,15 @@ case "$image" in
     DB=yes
     VISION=yes
     ;;
-  pytorch-linux-bionic-rocm5.0-py3.7)
+  pytorch-linux-focal-rocm5.0-py3.7)
     ANACONDA_PYTHON_VERSION=3.7
     GCC_VERSION=9
     PROTOBUF=yes
     DB=yes
     VISION=yes
     ROCM_VERSION=5.0
     ;;
-  pytorch-linux-bionic-rocm5.1-py3.7)
+  pytorch-linux-focal-rocm5.1-py3.7)
     ANACONDA_PYTHON_VERSION=3.7
     GCC_VERSION=9
     PROTOBUF=yes
@@ -249,6 +259,15 @@ case "$image" in
     VISION=yes
     KATEX=yes
     ;;
+  pytorch-linux-jammy-cuda11.6-cudnn8-py3.8-clang12)
+    ANACONDA_PYTHON_VERSION=3.8
+    CUDA_VERSION=11.6
+    CUDNN_VERSION=8
+    CLANG_VERSION=12
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    ;;
   *)
     # Catch-all for builds that are not hardcoded.
     PROTOBUF=yes

.circleci/docker/build_docker.sh

@@ -18,6 +18,7 @@ tag="${DOCKER_TAG}"
 
 registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
 image="${registry}/pytorch/${IMAGE_NAME}"
+ghcr_image="ghcr.io/pytorch/ci-image"
 
 login() {
   aws ecr get-authorization-token --region us-east-1 --output text --query 'authorizationData[].authorizationToken' |
@@ -48,7 +49,19 @@ fi
 
 # Only push if `DOCKER_SKIP_PUSH` = false
 if [ "${DOCKER_SKIP_PUSH:-true}" = "false" ]; then
-  docker push "${image}:${tag}"
+  # Only push if docker image doesn't exist already.
+  # ECR image tags are immutable so this will avoid pushing if only just testing if the docker jobs work
+  # NOTE: The only workflow that should push these images should be the docker-builds.yml workflow
+  if ! docker manifest inspect "${image}:${tag}" >/dev/null 2>/dev/null; then
+    docker push "${image}:${tag}"
+  fi
+
+  if [ "${PUSH_GHCR_IMAGE:-}" = "true" ]; then
+    # Push docker image to the ghcr.io
+    echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin
+    docker tag "${image}:${tag}" "${ghcr_image}:${IMAGE_NAME}-${tag}"
+    docker push "${ghcr_image}:${IMAGE_NAME}-${tag}"
+  fi
 fi
 
 if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then

.circleci/docker/common/install_base.sh

@@ -15,11 +15,22 @@ install_ubuntu() {
   elif [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
     cmake3="cmake=3.16*"
     maybe_libiomp_dev=""
+  elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
+    cmake3="cmake=3.22*"
+    maybe_libiomp_dev=""
   else
     cmake3="cmake=3.5*"
     maybe_libiomp_dev="libiomp-dev"
   fi
 
+  if [[ "$CLANG_VERSION" == 12 ]]; then
+    maybe_libomp_dev="libomp-12-dev"
+  elif [[ "$CLANG_VERSION" == 10 ]]; then
+    maybe_libomp_dev="libomp-10-dev"
+  else
+    maybe_libomp_dev=""
+  fi
+
   # TODO: Remove this once nvidia package repos are back online
   # Comment out nvidia repositories to prevent them from getting apt-get updated, see https://github.com/pytorch/pytorch/issues/74968
   # shellcheck disable=SC2046
@@ -51,15 +62,31 @@ install_ubuntu() {
     libjpeg-dev \
     libasound2-dev \
     libsndfile-dev \
+    ${maybe_libomp_dev} \
     software-properties-common \
     wget \
     sudo \
-    vim
+    vim \
+    jq
 
   # Should resolve issues related to various apt package repository cert issues
   # see: https://github.com/pytorch/pytorch/issues/65931
   apt-get install -y libgnutls30
 
+  # cuda-toolkit does not work with gcc-11.2.0 which is default in Ubunutu 22.04
+  # see: https://github.com/NVlabs/instant-ngp/issues/119
+  if [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
+    apt-get install -y g++-10
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
+    update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 30
+
+    # https://www.spinics.net/lists/libreoffice/msg07549.html
+    sudo rm -rf /usr/lib/gcc/x86_64-linux-gnu/11
+    wget https://github.com/gcc-mirror/gcc/commit/2b2d97fc545635a0f6aa9c9ee3b017394bc494bf.patch -O noexecpt.patch
+    sudo patch  /usr/include/c++/10/bits/range_access.h noexecpt.patch
+  fi
+
   # Cleanup package manager
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.circleci/docker/common/install_cache.sh

@@ -5,7 +5,9 @@ set -ex
 install_ubuntu() {
   echo "Preparing to build sccache from source"
   apt-get update
-  apt-get install -y cargo pkg-config libssl-dev
+  # libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
+  # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
+  apt-get install -y cargo
   echo "Checking out sccache repo"
   git clone https://github.com/pytorch/sccache
   cd sccache

.circleci/docker/common/install_conda.sh

@@ -73,19 +73,18 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   }
 
   # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
-  # DO NOT install cmake here as it would install a version newer than 3.10, but
-  # we want to pin to version 3.10.
+  # DO NOT install cmake here as it would install a version newer than 3.13, but
+  # we want to pin to version 3.13.
+  CONDA_COMMON_DEPS="astunparse pyyaml mkl=2022.0.1 mkl-include=2022.0.1 setuptools cffi future six"
   if [ "$ANACONDA_PYTHON_VERSION" = "3.9" ]; then
     # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy=1.19.2 astunparse pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
+    conda_install numpy=1.19.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
   elif [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
     # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
-    conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
-  elif [ "$ANACONDA_PYTHON_VERSION" = "3.7" ]; then
-    # DO NOT install dataclasses if installing python-3.7, since its part of python-3.7 core packages
-    conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six typing_extensions
+    conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
   else
-    conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six dataclasses typing_extensions
+    # Install `typing_extensions` for 3.7
+    conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} typing_extensions
   fi
 
   # Magma package names are concatenation of CUDA major and minor ignoring revision

.circleci/docker/common/install_openssl.sh

@@ -10,5 +10,7 @@ cd "${OPENSSL}"
 ./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
 # NOTE: openssl install errors out when built with the -j option
 make -j6; make install_sw
+# Link the ssl libraries to the /usr/lib folder.
+sudo ln -s /opt/openssl/lib/lib* /usr/lib
 cd ..
 rm -rf "${OPENSSL}"

.circleci/docker/ubuntu-cuda/Dockerfile

@@ -65,6 +65,7 @@ ENV INSTALLED_VISION ${VISION}
 ADD ./common/install_openssl.sh install_openssl.sh
 ENV OPENSSL_ROOT_DIR /opt/openssl
 RUN bash ./install_openssl.sh
+ENV OPENSSL_DIR /opt/openssl
 
 # (optional) Install non-default CMake version
 ARG CMAKE_VERSION

.circleci/docker/ubuntu/Dockerfile

@@ -6,14 +6,15 @@ ARG UBUNTU_VERSION
 
 ENV DEBIAN_FRONTEND noninteractive
 
+ARG CLANG_VERSION
+
 # Install common dependencies (so that this step can be cached separately)
 ARG EC2
 ADD ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
 # Install clang
 ARG LLVMDEV
-ARG CLANG_VERSION
 ADD ./common/install_clang.sh install_clang.sh
 RUN bash ./install_clang.sh && rm install_clang.sh
 
@@ -50,6 +51,13 @@ RUN bash ./install_gcc.sh && rm install_gcc.sh
 ADD ./common/install_lcov.sh install_lcov.sh
 RUN  bash ./install_lcov.sh && rm install_lcov.sh
 
+# Install cuda and cudnn
+ARG CUDA_VERSION
+RUN wget -q https://raw.githubusercontent.com/pytorch/builder/main/common/install_cuda.sh -O install_cuda.sh
+RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
+ENV DESIRED_CUDA ${CUDA_VERSION}
+ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
+
 # (optional) Install protobuf for ONNX
 ARG PROTOBUF
 ADD ./common/install_protobuf.sh install_protobuf.sh
@@ -111,6 +119,8 @@ RUN rm install_ninja.sh
 ADD ./common/install_openssl.sh install_openssl.sh
 RUN bash ./install_openssl.sh
 ENV OPENSSL_ROOT_DIR /opt/openssl
+ENV OPENSSL_DIR /opt/openssl
+RUN rm install_openssl.sh
 
 # Install ccache/sccache (do this last, so we get priority in PATH)
 ADD ./common/install_cache.sh install_cache.sh
@@ -122,12 +132,22 @@ ADD ./common/install_jni.sh install_jni.sh
 ADD ./java/jni.h jni.h
 RUN bash ./install_jni.sh && rm install_jni.sh
 
+# Install Open MPI for CUDA
+ADD ./common/install_openmpi.sh install_openmpi.sh
+RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
+RUN rm install_openmpi.sh
+
 # Include BUILD_ENVIRONMENT environment variable in image
 ARG BUILD_ENVIRONMENT
 ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 
 # Install LLVM dev version (Defined in the pytorch/builder github repository)
 COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
 
+# AWS specific CUDA build guidance
+ENV TORCH_CUDA_ARCH_LIST Maxwell
+ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
+ENV CUDA_PATH /usr/local/cuda
+
 USER jenkins
 CMD ["bash"]

.circleci/scripts/binary_linux_test.sh

@@ -53,9 +53,7 @@ if [[ "\$python_nodot" = *39*  ]]; then
   NUMPY_PIN=">=1.20"
 fi
 
-if [[ "$DESIRED_CUDA" == "cu116" ]]; then
-  EXTRA_CONDA_FLAGS="-c=conda-forge"
-fi
+
 
 # Move debug wheels out of the the package dir so they don't get installed
 mkdir -p /tmp/debug_final_pkgs
@@ -88,13 +86,14 @@ if [[ "$PACKAGE_TYPE" == conda ]]; then
     if [[ "$DESIRED_CUDA" == 'cpu' ]]; then
       retry conda install -c pytorch -y cpuonly
     else
-      # DESIRED_CUDA is in format cu90 or cu102
-      if [[ "${#DESIRED_CUDA}" == 4 ]]; then
-        cu_ver="${DESIRED_CUDA:2:1}.${DESIRED_CUDA:3}"
-      else
-        cu_ver="${DESIRED_CUDA:2:2}.${DESIRED_CUDA:4}"
+
+      cu_ver="${DESIRED_CUDA:2:2}.${DESIRED_CUDA:4}"
+      CUDA_PACKAGE="cudatoolkit"
+      if [[ "$DESIRED_CUDA" == "cu116" ]]; then
+        CUDA_PACKAGE="cuda"
       fi
-      retry conda install \${EXTRA_CONDA_FLAGS} -yq -c nvidia -c pytorch "cudatoolkit=\${cu_ver}"
+
+      retry conda install \${EXTRA_CONDA_FLAGS} -yq -c nvidia -c pytorch "\${CUDA_PACKAGE}=\${cu_ver}"
     fi
     conda install \${EXTRA_CONDA_FLAGS} -y "\$pkg" --offline
   )

.circleci/scripts/binary_windows_build.sh

@@ -6,7 +6,7 @@ mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR"
 
 export CUDA_VERSION="${DESIRED_CUDA/cu/}"
 export USE_SCCACHE=1
-export SCCACHE_BUCKET=ossci-compiler-cache-windows
+export SCCACHE_BUCKET=ossci-compiler-cache
 export SCCACHE_IGNORE_SERVER_IO_ERROR=1
 export VC_YEAR=2019