Merge pull request #993 from ROCmSoftwarePlatform/IFU-master-2022-04-11

IFU-master-2022-04-11

Author: pruthvistony

.circleci/cimodel/data/dimensions.py

@@ -4,6 +4,7 @@
     "102",
     "113",
     "115",
+    "116",
 ]
 
 ROCM_VERSIONS = [

.circleci/docker/build.sh

@@ -145,6 +145,17 @@ case "$image" in
     VISION=yes
     KATEX=yes
     ;;
+  pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
+    CUDA_VERSION=11.6.0
+    CUDNN_VERSION=8
+    ANACONDA_PYTHON_VERSION=3.7
+    CMAKE_VERSION=3.10.3
+    GCC_VERSION=7
+    PROTOBUF=yes
+    DB=yes
+    VISION=yes
+    KATEX=yes
+    ;;
   pytorch-linux-xenial-py3-clang5-asan)
     ANACONDA_PYTHON_VERSION=3.7
     CLANG_VERSION=5.0

.circleci/docker/common/install_user.sh

@@ -3,8 +3,9 @@
 set -ex
 
 # Mirror jenkins user in container
-echo "jenkins:x:1014:1014::/var/lib/jenkins:" >> /etc/passwd
-echo "jenkins:x:1014:" >> /etc/group
+# jenkins user as ec2-user should have the same user-id
+echo "jenkins:x:1000:1000::/var/lib/jenkins:" >> /etc/passwd
+echo "jenkins:x:1000:" >> /etc/group
 
 # Create $HOME
 mkdir -p /var/lib/jenkins

.circleci/scripts/binary_linux_test.sh

@@ -53,7 +53,7 @@ if [[ "\$python_nodot" = *39*  ]]; then
   NUMPY_PIN=">=1.20"
 fi
 
-if [[ "$DESIRED_CUDA" == "cu112" || "$DESIRED_CUDA" == "cu115" ]]; then
+if [[ "$DESIRED_CUDA" == "cu115" || "$DESIRED_CUDA" == "cu116" ]]; then
   EXTRA_CONDA_FLAGS="-c=conda-forge"
 fi
 

.circleci/scripts/cpp_doc_push_script.sh

@@ -65,8 +65,7 @@ cp torch/_utils_internal.py tools/shared
 
 # Generate PyTorch files
 time python tools/setup_helpers/generate_code.py \
-  --native-functions-path aten/src/ATen/native/native_functions.yaml \
-  --nn-path aten/src/
+  --native-functions-path aten/src/ATen/native/native_functions.yaml
 
 # Build the docs
 pushd docs/cpp

.circleci/scripts/setup_ci_environment.sh

@@ -32,7 +32,7 @@ if ! command -v aws >/dev/null; then
 fi
 
 if [ -n "${USE_CUDA_DOCKER_RUNTIME:-}" ]; then
-  DRIVER_FN="NVIDIA-Linux-x86_64-495.44.run"
+  DRIVER_FN="NVIDIA-Linux-x86_64-510.60.02.run"
   wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
   sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
   nvidia-smi

.circleci/scripts/windows_cuda_install.sh

@@ -22,6 +22,10 @@ case ${CUDA_VERSION} in
         cuda_installer_name="cuda_11.5.0_496.13_win10"
         cuda_install_packages="thrust_11.5 nvcc_11.5 cuobjdump_11.5 nvprune_11.5 nvprof_11.5 cupti_11.5 cublas_11.5 cublas_dev_11.5 cudart_11.5 cufft_11.5 cufft_dev_11.5 curand_11.5 curand_dev_11.5 cusolver_11.5 cusolver_dev_11.5 cusparse_11.5 cusparse_dev_11.5 npp_11.5 npp_dev_11.5 nvrtc_11.5 nvrtc_dev_11.5 nvml_dev_11.5"
         ;;
+    11.6)
+        cuda_installer_name="cuda_11.6.0_511.23_windows"
+        cuda_install_packages="thrust_11.6 nvcc_11.6 cuobjdump_11.6 nvprune_11.6 nvprof_11.6 cupti_11.6 cublas_11.6 cublas_dev_11.6 cudart_11.6 cufft_11.6 cufft_dev_11.6 curand_11.6 curand_dev_11.6 cusolver_11.6 cusolver_dev_11.6 cusparse_11.6 cusparse_dev_11.6 npp_11.6 npp_dev_11.6 nvrtc_11.6 nvrtc_dev_11.6 nvml_dev_11.6"
+        ;;
     *)
         echo "CUDA_VERSION $CUDA_VERSION is not supported yet"
         exit 1

.circleci/scripts/windows_cudnn_install.sh

@@ -22,6 +22,10 @@ case ${CUDA_VERSION} in
         # Since cudnn 8.3  the filename have changed
         cudnn_file_name="cudnn-windows-x86_64-8.3.2.44_cuda${CUDA_VERSION}-archive"
         ;;
+    11.6)
+        # Use cudnn8.3 with hard-coded cuda11.5 version
+        cudnn_file_name="cudnn-windows-x86_64-8.3.2.44_cuda11.5-archive"
+        ;;
     *)
         echo "CUDA_VERSION: ${CUDA_VERSION} not supported yet"
         exit 1

.github/actions/download-build-artifacts/action.yml

@@ -15,7 +15,7 @@ runs:
   steps:
     - name: Download PyTorch Build Artifacts from S3
       if: ${{ !inputs.use-gha }}
-      uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
+      uses: seemethere/download-artifact-s3@v3
       with:
         name: ${{ inputs.name }}
 

.github/actions/get-workflow-job-id/action.yml

@@ -0,0 +1,31 @@
+name: Get workflow job id
+
+description: Get the ID of the workflow job that is currently running.
+
+inputs:
+  github-token:
+    description: GITHUB_TOKEN
+    required: true
+
+outputs:
+  job-id:
+    description: The retrieved workflow job id
+    value: ${{ steps.get-job-id.outputs.job-id }}
+
+runs:
+  using: composite
+  steps:
+    - uses: nick-fields/retry@71062288b76e2b6214ebde0e673ce0de1755740a
+      id: get-job-id
+      env:
+        GITHUB_TOKEN: ${{ inputs.github-token }}
+      with:
+        shell: bash
+        timeout_minutes: 10
+        max_attempts: 5
+        retry_wait_seconds: 30
+        command: |
+          set -x
+          python3 -m pip install requests==2.26.0
+          GHA_WORKFLOW_JOB_ID=$(python3 .github/scripts/get_workflow_job_id.py "${GITHUB_RUN_ID}" "${RUNNER_NAME}")
+          echo "::set-output name=job-id::${GHA_WORKFLOW_JOB_ID}"

.github/actions/setup-win/action.yml

@@ -31,6 +31,12 @@ runs:
       run: |
         Set-ItemProperty -Path "HKLM:\\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1
 
+    # Since it's just a defensive command, the workflow should continue even the command fails
+    - name: Disables Windows Defender scheduled and real-time scanning for files in pytorch directory.
+      shell: powershell
+      run: |
+        Add-MpPreference -ExclusionPath $(Get-Location).tostring() -ErrorAction Ignore
+
     - name: Install Visual Studio 2019 toolchain
       shell: powershell
       env:

.github/actions/upload-test-artifacts/action.yml

@@ -7,15 +7,17 @@ inputs:
     description: If set to any value, upload GHA. Otherwise upload to S3.
     required: false
   file-suffix:
-    description: Suffix to add to the filename of the artifacts
+    description: |
+      Suffix to add to the filename of the artifacts. This should include the
+      workflow job id, see [Job id in artifacts].
     required: true
 
 runs:
   using: composite
   steps:
     # Mac/Linux zip
     - name: Zip JSONs for upload
-      if: runner.os != 'Windows'
+      if: runner.os != 'Windows' && !inputs.use-gha
       shell: bash
       env:
         FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -25,7 +27,7 @@ runs:
         zip -r "test-jsons-${FILE_SUFFIX}.zip" test -i '*.json'
 
     - name: Zip test reports for upload
-      if: runner.os != 'Windows'
+      if: runner.os != 'Windows' && !inputs.use-gha
       shell: bash
       env:
         FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -36,7 +38,7 @@ runs:
 
     # Windows zip
     - name: Zip JSONs for upload
-      if: runner.os == 'Windows'
+      if: runner.os == 'Windows' && !inputs.use-gha
       shell: powershell
       env:
         FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -45,7 +47,7 @@ runs:
         7z a "test-jsons-$Env:FILE_SUFFIX.zip" -ir'!test\*.json'
 
     - name: Zip test reports for upload
-      if: runner.os == 'Windows'
+      if: runner.os == 'Windows' && !inputs.use-gha
       shell: powershell
       env:
         FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -55,15 +57,15 @@ runs:
 
     # S3 upload
     - name: Store Test Downloaded JSONs on S3
-      uses: seemethere/upload-artifact-s3@v3
+      uses: seemethere/upload-artifact-s3@v4
       if: ${{ !inputs.use-gha }}
       with:
         retention-days: 14
         if-no-files-found: warn
         path: test-jsons-*.zip
 
     - name: Store Test Reports on S3
-      uses: seemethere/upload-artifact-s3@v3
+      uses: seemethere/upload-artifact-s3@v4
       if: ${{ !inputs.use-gha }}
       with:
         retention-days: 14
@@ -75,15 +77,18 @@ runs:
       uses: actions/upload-artifact@v2
       if: inputs.use-gha
       with:
+        # Add the run attempt, see [Artifact run attempt]
+        name: test-jsons-runattempt${{ github.run_attempt }}-${{ inputs.file-suffix }}.zip
         retention-days: 14
         if-no-files-found: warn
-        path: test-jsons-*.zip
+        path: test/**/*.json
 
     - name: Store Test Reports on Github
       uses: actions/upload-artifact@v2
       if: inputs.use-gha
       with:
-        name: test-reports
+        # Add the run attempt, see [Artifact run attempt]
+        name: test-reports-runattempt${{ github.run_attempt }}-${{ inputs.file-suffix }}.zip
         retention-days: 14
         if-no-files-found: error
-        path: test-reports-*.zip
+        path: test/**/*.xml

.github/generated-ciflow-ruleset.json

@@ -44,80 +44,7 @@
    {
       "name": "superuser",
       "patterns": ["*"],
-      "approved_by": [
-        "aaronenyeshi",
-        "alanwaketan",
-        "albanD",
-        "angelayi",
-        "anijain2305",
-        "anjali411",
-        "ansley",
-        "awgu",
-        "bertmaher",
-        "bdhirsh",
-        "cbalioglu",
-        "Chillee",
-        "cpuhrsch",
-        "dagitses",
-        "davidberard98",
-        "desertfire",
-        "eellison",
-        "ejguan",
-        "ezyang",
-        "fduwjj",
-        "Gamrix",
-        "gchanan",
-        "george-qi",
-        "gmagogsfm",
-        "hlu1",
-        "huiguoo",
-        "H-Huang",
-        "jaceyca",
-        "jamesr66a",
-        "jansel",
-        "jbschlosser",
-        "jiayisuse",
-        "Krovatkin",
-        "kumpera",
-        "kwen2501",
-        "malfet",
-        "mikaylagawarecki",
-        "mikeiovine",
-        "mingzhe09088",
-        "mrshenli",
-        "mruberry",
-        "navahgar",
-        "ngimel",
-        "NicolasHug",
-        "nikithamalgifb",
-        "NivekT",
-        "osalpekar",
-        "PaliC",
-        "pbelevich",
-        "pritamdamania87",
-        "qihqi",
-        "robieta",
-        "rohan-varma",
-        "saketh-are",
-        "samdow",
-        "seemethere",
-        "SherlockNoMad",
-        "shunting314",
-        "soulitzer",
-        "suo",
-        "tugsbayasgalan",
-        "VitalyFedyunin",
-        "vmoens",
-        "wanchaol",
-        "wconstab",
-        "wenleix",
-        "yaroslavvb",
-        "zdevito",
-        "zhaojuanmao",
-        "zhxchen17",
-        "ZolotukhinM",
-        "zou3519"
-      ],
+      "approved_by": ["pytorch/metamates"],
       "mandatory_checks_name": ["Facebook CLA Check", "Lint"]
    }
 ]

.github/merge_rules.json

@@ -30,7 +30,7 @@ runner_types:
   linux.2xlarge:
     instance_type: c5.2xlarge
     os: linux
-    max_available: 500
+    max_available: 750
     disk_size: 150
     is_ephemeral: false
   linux.4xlarge: # for binary-builds