[CI] Add LLVM test suite jobs to reusable workflow (#5097)

alexbatashev · web-flow · commit a9ef82d82aa5 · 2021-12-19T17:48:30.000+03:00
Overview of changes:
1. Refactor cleanup into a separate action to make sure that's the last thing we do
2. Introduce new llvm_test_suite action, that downloads and executes tests from intel/llvm-test-suite
3. Add 4 new jobs: OCL gen9, L0 gen9, OCL x64, HIP AMD GPU
4. Make cuda and hip args optional
5. Upload LIT as artifact to make it usable from LLVM Test suite

General idea: one build job compiles SYCL toolchain with all available backends enabled. Test jobs wait for build to complete and download pre-built artifact to run tests in parallel on various hardware. Amount of testing can be configured from outside by specifying `lts_config` parameter with one or more of the following values, separated with semicolon: `ocl_x64`, `ocl_gen9`, `l0_gen9`, `hip_amdgpu`. When tests are done, results from all the jobs are uploaded to lit_results artifact, which will contain all the logs in JSON format. Also, GitHub will automatically send notification to PR owner if one of the jobs has failed.
diff --git a/.github/workflows/sycl_linux_build_and_test.yml b/.github/workflows/sycl_linux_build_and_test.yml
@@ -40,10 +40,30 @@ on:
       build_configure_extra_args:
         type: string
         required: false
-        default: ""
+        default: "--hip --hip-amd-arch=gfx906 --cuda"
       build_artifact_suffix:
         type: string
         required: true
+      intel_drivers_image:
+        type: string
+        required: false
+        default: "ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest"
+      lts_config:
+        type: string
+        required: false
+        default: ""
+      gen9_runs_on:
+        type: string
+        required: false
+        default: "gen9"
+      amdgpu_image:
+        type: string
+        required: false
+        default: "ghcr.io/intel/llvm/ubuntu2004_build:latest"
+      amdgpu_runs_on:
+        type: string
+        required: false
+        default: "amdgpu"
 
 jobs:
   configure:
@@ -60,15 +80,20 @@ jobs:
             INPUTS="{
               \"cc\":\"gcc\",
               \"cxx\":\"g++\",
-              \"build_runs_on\":\"sycl-precommit-linux\",
+              \"build_runs_on\":\"build\",
               \"build_image\":\"ghcr.io/intel/llvm/ubuntu2004_build:latest\",
               \"build_github_cache\":\"false\",
               \"build_cache_root\":\"/__w/\",
               \"build_cache_suffix\":\"default\",
               \"build_cache_size\":\"2G\",
-              \"build_configure_extra_args\":\"\",
+              \"build_configure_extra_args\":\"--hip --hip-amd-arch=gfx906 --cuda\",
               \"build_artifact_suffix\":\"default\",
-              \"build_upload_artifact\":\"false\"
+              \"build_upload_artifact\":\"false\",
+              \"intel_drivers_image\":\"ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest\",
+              \"amdgpu_image\":\"ghcr.io/intel/llvm/ubuntu2004_build:latest\",
+              \"lts_config\":\"ocl_x64;hip_amdgpu\",
+              \"gen9_runs_on\":\"gen9\",
+              \"amdgpu_runs_on\":\"amdgpu\"
             }"
           fi
           INPUTS="${INPUTS//'%'/'%25'}"
@@ -77,7 +102,7 @@ jobs:
           echo "::set-output name=params::$INPUTS"
           echo "$INPUTS"
   build:
-    name: Build SYCL toolchain
+    name: Build + LIT
     needs: configure
     runs-on: ${{ fromJSON(needs.configure.outputs.params).build_runs_on }}
     container:
@@ -87,6 +112,8 @@ jobs:
     - uses: actions/checkout@v2
       with:
         path: src
+    - name: Register cleanup after job is finished
+      uses: ./src/devops/actions/cleanup
     - name: Setup Cache
       uses: actions/cache@v2
       if: ${{ steps.parameters.build_github_cache }}
@@ -111,10 +138,11 @@ jobs:
         cd $GITHUB_WORKSPACE/build
         python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \
           -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \
-          --ci-defaults $ARGS --cuda --hip --hip-amd-arch="gfx906" \
+          --ci-defaults $ARGS \
           --cmake-opt="-DLLVM_CCACHE_BUILD=ON" \
           --cmake-opt="-DLLVM_CCACHE_DIR=$CACHE_ROOT/build_cache_$CACHE_SUFFIX" \
           --cmake-opt="-DLLVM_CCACHE_MAXSIZE=$CACHE_SIZE" \
+          --cmake-opt="-DLLVM_INSTALL_UTILS=ON" \
           --cmake-opt="-DSYCL_PI_TESTS=OFF"
     - name: Compile
       run: cmake --build $GITHUB_WORKSPACE/build
@@ -157,15 +185,128 @@ jobs:
         cmake --build $GITHUB_WORKSPACE/build --target utils/llvm-lit/install
         cmake --build $GITHUB_WORKSPACE/build --target install-clang-format
         cmake --build $GITHUB_WORKSPACE/build --target install-clang-tidy
+        cmake --build $GITHUB_WORKSPACE/build --target install-llvm-size
+        # TODO this should be resolved in CMakeLists.txt
+        cmake --build $GITHUB_WORKSPACE/build --target install-lld || echo "skipped"
 
-    - name: Pack
+    - name: Pack toolchain
       run: tar -cJf llvm_sycl.tar.xz -C $GITHUB_WORKSPACE/build/install .
-    - name: Upload artifacts
+    - name: Pack LIT
+      run: tar -cJf lit.tar.xz -C $GITHUB_WORKSPACE/src/llvm/utils/lit .
+    - name: Upload toolchain
       uses: actions/upload-artifact@v1
       with:
         name: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
         path: llvm_sycl.tar.xz
-    - name: Cleanup
-      if: always()
-      run: rm -rf $GITHUB_WORKSPACE/*
+    - name: Upload LIT
+      uses: actions/upload-artifact@v1
+      with:
+        name: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        path: lit.tar.xz
 
+  llvm_test_suite_l0_gen9:
+    name: L0 GEN9 Test Suite
+    needs: [build, configure]
+    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'l0_gen9') }}
+    runs-on: ${{ fromJSON(needs.configure.outputs.params).gen9_runs_on }}
+    container:
+      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
+      options: -u 1001 --device=/dev/dri
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        path: llvm
+    - name: Register cleanup after job is finished
+      uses: ./llvm/devops/actions/cleanup
+    - uses: ./llvm/devops/actions/llvm_test_suite
+      name: Run LLVM Test Suite
+      with:
+        # TODO allow custom test references
+        test_ref: 'intel'
+        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        sycl_archive: llvm_sycl.tar.xz
+        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        lit_archive: lit.tar.xz
+        check_sycl_all: 'level_zero:gpu,host'
+        results_name_suffix: l0_gpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        cmake_args: '-DGPU_AOT_TARGET_OPTS="\\\"-device gen9\\\""'
+
+  llvm_test_suite_ocl_gen9:
+    name: OCL GEN9 Test Suite
+    needs: [build, configure]
+    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'ocl_gen9') }}
+    runs-on: ${{ fromJSON(needs.configure.outputs.params).gen9_runs_on }}
+    container:
+      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
+      options: -u 1001 --device=/dev/dri
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        path: llvm
+    - name: Register cleanup after job is finished
+      uses: ./llvm/devops/actions/cleanup
+    - uses: ./llvm/devops/actions/llvm_test_suite
+      name: Run LLVM Test Suite
+      with:
+        # TODO allow custom test references
+        test_ref: 'intel'
+        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        sycl_archive: llvm_sycl.tar.xz
+        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        lit_archive: lit.tar.xz
+        check_sycl_all: 'opencl:gpu,host'
+        results_name_suffix: ocl_gpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        cmake_args: '-DGPU_AOT_TARGET_OPTS="\\\"-device gen9\\\""'
+
+  llvm_test_suite_ocl_x64:
+    name: OCL x64 Test Suite
+    needs: [build, configure]
+    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'ocl_x64') }}
+    runs-on: ${{ fromJSON(needs.configure.outputs.params).gen9_runs_on }}
+    container:
+      image: ${{ fromJSON(needs.configure.outputs.params).intel_drivers_image }}
+      options: -u 1001
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        path: llvm
+    - name: Register cleanup after job is finished
+      uses: ./llvm/devops/actions/cleanup
+    - uses: ./llvm/devops/actions/llvm_test_suite
+      name: Run LLVM Test Suite
+      with:
+        # TODO allow custom test references
+        test_ref: 'intel'
+        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        sycl_archive: llvm_sycl.tar.xz
+        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        lit_archive: lit.tar.xz
+        check_sycl_all: 'opencl:cpu,host'
+        results_name_suffix: ocl_x64_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+
+  llvm_test_suite_hip_amdgpu:
+    name: HIP AMD GPU Test Suite
+    needs: [build, configure]
+    if: ${{ contains(fromJSON(needs.configure.outputs.params).lts_config, 'hip_amdgpu') }}
+    runs-on: ${{ fromJSON(needs.configure.outputs.params).amdgpu_runs_on }}
+    container:
+      image: ${{ fromJSON(needs.configure.outputs.params).amdgpu_image }}
+      options: --device=/dev/dri --device=/dev/kfd
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        path: llvm
+    - name: Register cleanup after job is finished
+      uses: ./llvm/devops/actions/cleanup
+    - uses: ./llvm/devops/actions/llvm_test_suite
+      name: Run LLVM Test Suite
+      with:
+        # TODO allow custom test references
+        test_ref: 'intel'
+        sycl_artifact: sycl_linux_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        sycl_archive: llvm_sycl.tar.xz
+        lit_artifact: sycl_lit_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        lit_archive: lit.tar.xz
+        check_sycl_all: 'hip:gpu,host'
+        results_name_suffix: hip_amdgpu_${{ fromJSON(needs.configure.outputs.params).build_artifact_suffix }}
+        cmake_args: '-DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031"'
diff --git a/devops/actions/cleanup/action.yml b/devops/actions/cleanup/action.yml
@@ -0,0 +1,8 @@
+name: 'Cleanup'
+description: 'Cleanup work directory upon job finish'
+
+runs:
+  using: 'node12'
+  main: 'dummy.js'
+  post: 'cleanup.js'
+
diff --git a/devops/actions/cleanup/cleanup.js b/devops/actions/cleanup/cleanup.js
@@ -0,0 +1,3 @@
+const path = process.env.GITHUB_WORKSPACE + '/*';
+console.log('Cleaning ' + path)
+require('child_process').execSync('rm -rf ' + path);
diff --git a/devops/actions/cleanup/dummy.js b/devops/actions/cleanup/dummy.js
@@ -0,0 +1 @@
+console.log('setup cleanup');
diff --git a/devops/actions/llvm_test_suite/action.yml b/devops/actions/llvm_test_suite/action.yml
@@ -0,0 +1,105 @@
+name: 'Run llvm-test-suite testing'
+description: 'Run E2E SYCL tests from llvm-test-suite'
+inputs:
+  test_ref:
+    description: 'commit-ish identifier for test repo'
+    required: true
+    default: 'intel'
+  sycl_artifact:
+    description: 'Name of the artifact, that contains compiler toolchain'
+    required: true
+  sycl_archive:
+    description: 'Name of SYCL toolchain archive file'
+    required: false
+    default: 'llvm_sycl.tar.xz'
+  lit_artifact:
+    description: 'Name of the artifact, that contains LIT tool'
+    required: true
+  lit_archive:
+    description: 'Name of LIT archive file'
+    required: false
+    default: 'lit.tar.xz'
+  results_name_suffix:
+    description: 'Name suffix of the results artifact'
+    required: true
+  check_sycl_all:
+    description: 'List of SYCL backends with set of target devices per each to be tested iteratively'
+    required: true
+  cmake_args:
+    description: 'Extra arguments to cmake command'
+    required: false
+
+post-if: false
+runs:
+  using: "composite"
+  steps:
+  - name: Checkout LLVM Test Suite
+    uses: actions/checkout@v2
+    with:
+      path: llvm_test_suite
+      repository: 'intel/llvm-test-suite'
+      ref: ${{ inputs.test_ref }}
+  - name: Download compiler toolchain
+    uses: actions/download-artifact@v2
+    with:
+      name: ${{ inputs.sycl_artifact }}
+  - name: Extract SYCL toolchain
+    shell: bash
+    run: |
+      mkdir toolchain
+      tar -xf ${{ inputs.sycl_archive }} -C toolchain
+      rm -f ${{ inputs.sycl_archive }}
+  - name: Download LIT
+    uses: actions/download-artifact@v2
+    with:
+      name: ${{ inputs.lit_artifact }}
+  - name: Extract LIT
+    shell: bash
+    run: |
+      mkdir lit
+      ls -la
+      tar -xf ${{ inputs.lit_archive }} -C lit
+      rm -f ${{ inputs.lit_archive }}
+  - name: Configure
+    shell: bash
+    run: |
+      echo "::group::CMake configuration"
+      mkdir build
+      export PATH=$PWD/toolchain/bin/:$PATH
+      cmake -GNinja -B./build -S./llvm_test_suite -DTEST_SUITE_SUBDIRS=SYCL -DCHECK_SYCL_ALL="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DTEST_SUITE_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      echo "::endgroup::"
+  - name: Run testing
+    shell: bash
+    run: |
+      export LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH
+      export PATH=$PWD/toolchain/bin/:$PATH
+      # TODO make this part of container build
+      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/hip/lib/:/opt/rocm/lib
+      export LIT_OPTS="-v --no-progress-bar --show-unsupported --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}.json"
+      if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
+        source /runtimes/oneapi-tbb/env/vars.sh;
+      fi
+      echo "::group::sycl-ls --verbose"
+      sycl-ls --verbose
+      echo "::endgroup::"
+      echo "::group::SYCL_PI_TRACE=-1 sycl-ls"
+      echo $LD_LIBRARY_PATH
+      ldd $PWD/toolchain/lib/libpi_hip.so
+      SYCL_PI_TRACE=-1 sycl-ls
+      echo "::endgroup::"
+      cd build
+      ninja check-sycl-all
+  - name: Upload test results
+    uses: actions/upload-artifact@v1
+    if: always()
+    with:
+      name: lit_results
+      path: build/results_${{ inputs.results_name_suffix }}.json
+  - name: Cleanup
+    shell: bash
+    if: always()
+    run: |
+      rm -rf toolchain
+      rm -rf lit
+      rm -rf build
+      rm -rf llvm_test_suite

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+const path = process.env.GITHUB_WORKSPACE + '/*';`
	`2`	`+console.log('Cleaning ' + path)`
	`3`	`+require('child_process').execSync('rm -rf ' + path);`