[CI] Enable SYCL End-to-End tests on Windows for post-commit (#8656)

uditagarwal97 · Pavel V Chupin · bader · web-flow · commit 19d023b1601f · 2023-04-06T16:34:29.000+01:00
This PR augments Pavel's work (#7477) on running llvm_test_suite in Windows CI for post-commit testing. Keeping in mind that the intel/llvm_test_suite repo is getting merged into intel/llvm, this PR tests in-tree SYCL E2E tests. --------- Co-authored-by: Pavel V Chupin <pavel.v.chupin@intel.com> Co-authored-by: Alexey Bader <alexey.bader@intel.com>
diff --git a/.github/workflows/sycl_gen_test_matrix.yml b/.github/workflows/sycl_gen_test_matrix.yml
@@ -34,21 +34,25 @@ on:
         required: false
         default: ${{ github.sha }}
     outputs:
-      lts_matrix:
-        description: "Generated Matrix"
-        value: ${{ jobs.test_matrix.outputs.lts_matrix }}
+      lts_lx_matrix:
+        description: "Linux LTS"
+        value: ${{ jobs.test_matrix.outputs.lts_lx_matrix }}
+      lts_wn_matrix:
+        description: "Windows LTS"
+        value: ${{ jobs.test_matrix.outputs.lts_wn_matrix }}
       cts_matrix:
-        description: "Generated SYCL CTS Matrix"
+        description: "SYCL CTS"
         value: ${{ jobs.test_matrix.outputs.cts_matrix }}
       lts_aws_matrix:
-        description: "Generated Matrix AWS subset"
+        description: "Linux AWS"
         value: ${{ jobs.test_matrix.outputs.lts_aws_matrix }}
 jobs:
   test_matrix:
     name: Generate Test Matrix
     runs-on: ubuntu-20.04
     outputs:
-      lts_matrix: ${{ steps.work.outputs.lts_matrix }}
+      lts_lx_matrix: ${{ steps.work.outputs.lts_lx_matrix }}
+      lts_wn_matrix: ${{ steps.work.outputs.lts_wn_matrix }}
       cts_matrix: ${{ steps.work.outputs.cts_matrix }}
       lts_aws_matrix: ${{ steps.work.outputs.lts_aws_matrix }}
     steps:
diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml
@@ -22,7 +22,7 @@ jobs:
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_gen_test_matrix.yml
     with:
-      lts_config: "l0_gen9"
+      lts_config: "l0_gen9;win_l0_gen12"
       cts_config: "cuda"
   linux_default:
     name: Linux Default
@@ -33,7 +33,7 @@ jobs:
     with:
       build_cache_root: "/__w/llvm"
       build_artifact_suffix: "post_commit"
-      lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }}
+      lts_matrix: ${{ needs.test_matrix.outputs.lts_lx_matrix }}
       cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }}
       lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }}
       build_configure_extra_args: --hip --cuda --enable-esimd-emulator --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON"
@@ -63,8 +63,11 @@ jobs:
 
   windows_default:
     name: Windows
+    needs: test_matrix
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_windows_build_and_test.yml
+    with:
+      lts_matrix: ${{ needs.test_matrix.outputs.lts_wn_matrix }}
 
   macos_default:
     name: macOS
diff --git a/.github/workflows/sycl_precommit.yml b/.github/workflows/sycl_precommit.yml
@@ -64,5 +64,5 @@ jobs:
       build_cache_size: "8G"
       build_artifact_suffix: "default"
       build_cache_suffix: "default"
-      lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }}
+      lts_matrix: ${{ needs.test_matrix.outputs.lts_lx_matrix }}
       lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }}
diff --git a/.github/workflows/sycl_windows_build_and_test.yml b/.github/workflows/sycl_windows_build_and_test.yml
@@ -10,11 +10,22 @@ on:
       build_ref:
         type: string
         required: false
+      lts_matrix:
+        type: string
+        required: false
+        default: "[]"
+      lts_cmake_extra_args:
+        type: string
+        required: false
+        default: ""
 
 jobs:
   build:
     name: Build + LIT
     runs-on: [Windows, build]
+    strategy:
+      matrix:
+        include: ${{ fromJSON(inputs.lts_matrix) }}
     # TODO use cached checkout
     steps:
     - uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
@@ -40,7 +51,7 @@ jobs:
         mkdir build
         mkdir install
         IF NOT EXIST D:\github\_work\cache MKDIR D:\github\_work\cache
-        IF NOT EXIST D:\github\_work\cache\sycl_${{inputs.build_cache_suffix}} MKDIR D:\github\_work\cache\${{inputs.build_cache_suffix}}
+        IF NOT EXIST D:\github\_work\cache\${{inputs.build_cache_suffix}} MKDIR D:\github\_work\cache\${{inputs.build_cache_suffix}}
         python.exe src/buildbot/configure.py -o build ^
           --ci-defaults ^
           --cmake-opt="-DCMAKE_C_COMPILER=cl" ^
@@ -50,9 +61,9 @@ jobs:
           --cmake-opt="-DCMAKE_C_COMPILER_LAUNCHER=sccache"
     - name: Build
       id: build
+      shell: bash
       run: |
         cmake --build build --target sycl-toolchain
-        sccache --show-stats
     - name: check-llvm
       shell: bash
       if: ${{ always() && !cancelled() && steps.build.outcome == 'success' }}
@@ -80,9 +91,62 @@ jobs:
         cmake --build build --target check-xptifw
     - name: Install
       shell: bash
-      run: cmake --build build --target deploy-sycl-toolchain
+      run: |
+        # TODO replace utility installation with a single CMake target
+        cmake --build build --target utils/count/install
+        cmake --build build --target utils/not/install
+        cmake --build build --target utils/lit/install
+        cmake --build build --target utils/llvm-lit/install
+        cmake --build build --target install-llvm-size
+        cmake --build build --target install-llvm-cov
+        cmake --build build --target install-llvm-profdata
+        # TODO: Figure out why installing FileCheck does not install
+        # the FileCheck binary in the install/bin directory.
+        cmake --build build --target utils/FileCheck/install
+        cmake --build build --target install-compiler-rt
+        cmake --build build --target deploy-sycl-toolchain
+        # Copy FileCheck to install/bin for use by E2E tests.
+        cp build/bin/FileCheck.exe install/bin/.
+    - name: Pack toolchain
+      shell: bash
+      run: |
+        tar -czf llvm_sycl.tar.gz -C install .
     - name: Upload toolchain
       uses: actions/upload-artifact@v3
       with:
         name: sycl_windows_default
-        path: install/**/*
+        path: llvm_sycl.tar.gz
+
+  e2e-tests:
+    needs: build
+    if: ${{ inputs.lts_matrix != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(inputs.lts_matrix) }}
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    env: ${{ matrix.env }}
+    steps:
+    - uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d
+      with:
+        arch: amd64
+    - name: Set env
+      run: |
+        git config --system core.longpaths true
+        git config --global core.autocrlf false
+        echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+    # TODO: use cached_checkout
+    - uses: actions/checkout@v3
+      with:
+        persist-credentials: false
+        ref: ${{ inputs.build_ref || github.sha }}
+        path: llvm
+    - name: Register cleanup after job is finished
+      uses: ./llvm/devops/actions/cleanup
+    - uses: ./llvm/devops/actions/e2e-tests-win
+      name: Run SYCL End-to-End tests
+      with:
+        sycl_artifact: sycl_windows_default
+        targets: ${{ matrix.targets }}
+        cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
diff --git a/devops/actions/e2e-tests-win/action.yml b/devops/actions/e2e-tests-win/action.yml
@@ -0,0 +1,45 @@
+name: 'Run SYCL End-to-End testing'
+description: 'Run SYCL End-to-End testing'
+inputs:
+  sycl_artifact:
+    description: 'Name of the artifact, that contains compiler toolchain'
+    required: true
+  targets:
+    description: 'List of SYCL backends with set of target devices per each to be tested iteratively'
+    required: true
+  cmake_args:
+    description: 'Extra arguments to cmake command'
+    required: false
+
+post-if: false
+runs:
+  using: "composite"
+  steps:
+  - name: Download compiler toolchain
+    uses: actions/download-artifact@v3
+    with:
+      name: ${{ inputs.sycl_artifact }}
+  - name: Extract SYCL toolchain
+    shell: bash
+    run: |
+      mkdir install
+      tar -xf llvm_sycl.tar.gz -C install
+      rm llvm_sycl.tar.gz
+  - name: Configure
+    shell: cmd
+    run: |
+      mkdir build-e2e
+      set PATH=%GITHUB_WORKSPACE%\install\bin;%PATH%
+      cmake -GNinja -B build-e2e -S.\llvm\sycl\test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.targets }}" -DCMAKE_CXX_COMPILER="clang++" -DLLVM_LIT="..\llvm\llvm\utils\lit\lit.py" ${{ inputs.cmake_args }}
+  - name: Run testing
+    shell: bash
+    run: |
+      # Run E2E tests.
+      export LIT_OPTS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests"
+      cmake --build build-e2e --target check-sycl-e2e
+  - name: Cleanup
+    shell: cmd
+    if: always()
+    run: |
+      rmdir /q /s install
+      rmdir /q /s build-e2e
diff --git a/devops/scripts/generate_test_matrix.js b/devops/scripts/generate_test_matrix.js
@@ -2,7 +2,7 @@ module.exports = ({core, process}) => {
   const fs = require('fs');
   fs.readFile('./test_configs.json', 'utf8', (err, data) => {
     if (err) {
-      console.log(`Error reading file from disk: ${err}`);
+      console.error(`Error reading file from disk: ${err}`);
     } else {
       const driverNew =
           JSON.parse(fs.readFileSync('./dependencies.json', 'utf8'));
@@ -28,9 +28,12 @@ module.exports = ({core, process}) => {
 
       const ltsConfigs = inputs.lts_config.split(';');
 
-      const enabledLTSConfigs = [];
+      const enabledLTSLxConfigs = [];
+      const enabledLTSWnConfigs = [];
       const enabledLTSAWSConfigs = [];
 
+      // Process LTS (LLVM Test Suite)
+
       testConfigs.lts.forEach(v => {
         if (ltsConfigs.includes(v.config)) {
           if (needsDrivers) {
@@ -47,32 +50,47 @@ module.exports = ({core, process}) => {
           } else {
             v["env"] = {};
           }
-          enabledLTSConfigs.push(v);
+          if (v["runs-on"].includes("Windows"))
+            enabledLTSWnConfigs.push(v);
+          else if (v["runs-on"].includes("Linux"))
+            enabledLTSLxConfigs.push(v);
+          else
+            console.error("runs-on OS is not recognized");
           if (v["aws-type"]) enabledLTSAWSConfigs.push(v);
         }
       });
 
-      let ltsString = JSON.stringify(enabledLTSConfigs);
+      let ltsLxString = JSON.stringify(enabledLTSLxConfigs);
+      let ltsWnString = JSON.stringify(enabledLTSWnConfigs);
       let ltsAWSString = JSON.stringify(enabledLTSAWSConfigs);
-      console.log(ltsString);
+      console.log("Linux LTS config:")
+      console.log(ltsLxString);
+      console.log("Windows LTS config:")
+      console.log(ltsWnString);
+      console.log("Linux AWS LTS config:")
       console.log(ltsAWSString)
 
+      // drivers update is supported on Linux only
       for (let [key, value] of Object.entries(inputs)) {
-        ltsString = ltsString.replaceAll("${{ inputs." + key + " }}", value);
+        ltsLxString =
+            ltsLxString.replaceAll("${{ inputs." + key + " }}", value);
         ltsAWSString = ltsAWSString.replaceAll("${{ inputs." + key + " }}", value);
       }
       if (needsDrivers) {
-        ltsString = ltsString.replaceAll(
+        ltsLxString = ltsLxString.replaceAll(
             "ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest",
             "ghcr.io/intel/llvm/ubuntu2004_base:latest");
         ltsAWSString = ltsAWSString.replaceAll(
             "ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest",
             "ghcr.io/intel/llvm/ubuntu2004_base:latest");
       }
 
-      core.setOutput('lts_matrix', ltsString);
+      core.setOutput('lts_lx_matrix', ltsLxString);
+      core.setOutput('lts_wn_matrix', ltsWnString);
       core.setOutput('lts_aws_matrix', ltsAWSString);
 
+      // Process CTS (Conformance Test Suite)
+
       const ctsConfigs = inputs.cts_config.split(';');
 
       const enabledCTSConfigs = [];
@@ -98,6 +116,7 @@ module.exports = ({core, process}) => {
       });
 
       let ctsString = JSON.stringify(enabledCTSConfigs);
+      console.log("CTS config:")
       console.log(ctsString);
 
       for (let [key, value] of Object.entries(inputs)) {
diff --git a/devops/test_configs.json b/devops/test_configs.json
@@ -12,6 +12,16 @@
       "targets": "level_zero:gpu",
       "cmake_args": ""
     },
+    {
+      "config": "win_l0_gen12",
+      "name": "L0 GEN12 LLVM Test Suite",
+      "runs-on": [
+        "Windows",
+        "gen12"
+      ],
+      "targets": "level_zero:gpu",
+      "cmake_args": ""
+    },
     {
       "config": "ocl_gen9",
       "name": "OCL GEN9 LLVM Test Suite",
diff --git a/sycl/test-e2e/Basic/build_log.cpp b/sycl/test-e2e/Basic/build_log.cpp
@@ -1,5 +1,5 @@
 // REQUIRES: opencl || level_zero, gpu
-// UNSUPPORTED: gpu-intel-dg1
+// UNSUPPORTED: gpu-intel-dg1 || windows
 //
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_gen -Xsycl-target-backend=spir64_gen "-device dg1" %s -o %t.out
 // RUN: %GPU_RUN_PLACEHOLDER env SYCL_RT_WARNING_LEVEL=2 %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
diff --git a/sycl/test-e2e/ESIMD/dword_atomic_cmpxchg.cpp b/sycl/test-e2e/ESIMD/dword_atomic_cmpxchg.cpp
@@ -10,6 +10,9 @@
 // REQUIRES: gpu
 // UNSUPPORTED: gpu-intel-gen9 && windows
 // UNSUPPORTED: cuda || hip
+// TODO: Re-enable this test on Windows after the following issue gets fixed:
+// https://github.com/intel/llvm/issues/8934
+// UNSUPPORTED: windows
 // TODO: esimd_emulator fails due to random timeouts (_XFAIL_: esimd_emulator)
 // UNSUPPORTED: esimd_emulator
 // RUN: %clangxx -fsycl %s -o %t.out
diff --git a/sycl/test-e2e/ESIMD/dword_atomic_cmpxchg_scalar_off.cpp b/sycl/test-e2e/ESIMD/dword_atomic_cmpxchg_scalar_off.cpp
@@ -10,6 +10,9 @@
 // REQUIRES: gpu
 // UNSUPPORTED: gpu-intel-gen9 && windows
 // UNSUPPORTED: cuda || hip
+// TODO: Re-enable this test on Windows after the following issue gets fixed:
+// https://github.com/intel/llvm/issues/8934
+// UNSUPPORTED: windows
 // TODO: esimd_emulator fails due to random timeouts (_XFAIL_: esimd_emulator)
 // UNSUPPORTED: esimd_emulator
 // RUN: %clangxx -fsycl %s -o %t.out
diff --git a/sycl/test-e2e/ESIMD/dword_atomic_smoke.cpp b/sycl/test-e2e/ESIMD/dword_atomic_smoke.cpp
@@ -10,6 +10,9 @@
 // REQUIRES: gpu
 // UNSUPPORTED: gpu-intel-gen9 && windows
 // UNSUPPORTED: cuda || hip
+// TODO: Re-enable this test on Windows after the following issue gets fixed:
+// https://github.com/intel/llvm/issues/8934
+// UNSUPPORTED: windows
 // RUN: %clangxx -fsycl %s -o %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 
diff --git a/sycl/test-e2e/ESIMD/dword_atomic_smoke_scalar_off.cpp b/sycl/test-e2e/ESIMD/dword_atomic_smoke_scalar_off.cpp
@@ -10,6 +10,9 @@
 // REQUIRES: gpu
 // UNSUPPORTED: gpu-intel-gen9 && windows
 // UNSUPPORTED: cuda || hip
+// TODO: Re-enable this test on Windows after the following issue gets fixed:
+// https://github.com/intel/llvm/issues/8934
+// UNSUPPORTED: windows
 // TODO: esimd_emulator fails due to random timeouts (_XFAIL_: esimd_emulator)
 // UNSUPPORTED: esimd_emulator
 // RUN: %clangxx -fsycl %s -o %t.out
diff --git a/sycl/test-e2e/Regression/reduction_resource_leak_usm.cpp b/sycl/test-e2e/Regression/reduction_resource_leak_usm.cpp
@@ -1,6 +1,6 @@
 // REQUIRES: level_zero, level_zero_dev_kit
 // TODO: ZE_DEBUG=4 produces no output on Windows. Enable when fixed.
-// XFAIL: windows
+// UNSUPPORTED: windows
 //
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %level_zero_options %s -o %t.out
 // RUN: env ONEAPI_DEVICE_SELECTOR="level_zero:*" ZE_DEBUG=4 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
