[CI][SYCL][E2E] Split gen12 pre-commit testing into build and run stages (#16321)

This patch enables E2E build and run splitting on pre-commit CI.
Currently one of the main bottlenecks in pre-commit CI is the Linux
Gen12 testing which takes between 30 ~ 40 minutes to complete. By
instead building these tests on our fast build systems, and then
transferring the binaries to the Gen12 machine we can cut down on this
time by roughly half.

Sadly, the Windows Gen12 testing will still be the main bottleneck so
total time for pre-commit won't improve with this PR, but we will at
least get faster feedback for gen12 on Linux.

Author: ayylol

.github/workflows/sycl-linux-precommit.yml

@@ -72,6 +72,53 @@ jobs:
           else
             echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
           fi
+
+  build_e2e_tests:
+    needs: [build]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Build e2e tests
+      runner: '["Linux", "build"]'
+      image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+      image_options: -u 1001
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      e2e_testing_mode: 'build-only'
+  run_prebuilt_e2e_tests:
+    needs: [build, build_e2e_tests]
+    if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Intel
+            runner: '["Linux", "gen12"]'
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
+            reset_intel_gpu: true
+            extra_lit_opts: --param gpu-intel-gen12=True
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: ${{ matrix.name }}
+      runner: ${{ matrix.runner }}
+      image: ${{ matrix.image }}
+      image_options: ${{ matrix.image_options }}
+      target_devices: ${{ matrix.target_devices }}
+      extra_lit_opts: --param fallback-to-build-if-requires-build-and-run=True ${{ matrix.extra_lit_opts }}
+      reset_intel_gpu: ${{ matrix.reset_intel_gpu }}
+      ref: ${{ github.sha }}
+      merge_ref: ''
+      sycl_toolchain_artifact: sycl_linux_default
+      sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
+      sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}
+      e2e_binaries_artifact: sycl_e2e_bin_default
+      e2e_testing_mode: 'run-only'
+
   test:
     needs: [build, detect_changes, determine_arc_tests]
     if: ${{ always() && !cancelled() && needs.build.outputs.build_conclusion == 'success' }}
@@ -90,13 +137,6 @@ jobs:
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
             target_devices: ext_oneapi_hip:gpu
             reset_intel_gpu: false
-          - name: Intel
-            runner: '["Linux", "gen12"]'
-            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
-            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
-            reset_intel_gpu: true
-            extra_lit_opts: --param gpu-intel-gen12=True
           - name: E2E tests on Intel Arc A-Series Graphics
             runner: '["Linux", "arc"]'
             image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest

.github/workflows/sycl-linux-run-tests.yml

@@ -19,7 +19,7 @@ on:
 
       target_devices:
         type: string
-        required: True
+        required: False
       extra_cmake_args:
         type: string
         required: False
@@ -59,6 +59,30 @@ on:
         default: ''
         required: False
 
+      e2e_binaries_artifact:
+        description: |
+          By setting this the E2E binaries folder will not be created, rather it
+          will be downloaded and extracted from the specified artifact. When
+          running tests in `run-only` mode this must be provided.
+        type: string
+        default: ''
+        required: False
+      e2e_testing_mode:
+        description: |
+          Testing mode to run E2E tests in, can be either `full`, `build-only`
+          or `run-only`. In `build-only` mode an artifact of the E2E binaries
+          will be uploaded.
+        type: string
+        default: 'full'
+      artifact_suffix:
+        description: 'Suffix for E2E binaries artifact that is output when in `build-only`.'
+        type: string
+        default: 'default'
+      retention-days:
+        description: 'E2E binaries artifact retention period.'
+        type: string
+        default: 1
+
       reset_intel_gpu:
         type: string
         required: False
@@ -87,6 +111,7 @@ on:
           - '["amdgpu"]'
           - '["Linux", "arc"]'
           - '["cts-cpu"]'
+          - '["Linux", "build"]'
       image:
         description: |
           Use option ending with ":build" for AMDGPU, ":latest" for the rest.
@@ -142,6 +167,11 @@ on:
         options:
           - false
           - true
+      e2e_testing_mode:
+        type: choice
+        options:
+          - "full"
+          - "build-only"
 
 permissions:
   contents: read
@@ -270,8 +300,19 @@ jobs:
             cat /usr/local/lib/igc/IGCTAG.txt
           fi
 
+    - name: Download E2E Binaries
+      if: inputs.e2e_binaries_artifact != ''
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.e2e_binaries_artifact }}
+    - name: Extract E2E Binaries
+      if: inputs.e2e_binaries_artifact != ''
+      run: |
+        mkdir build-e2e
+        tar -I 'zstd' -xf e2e_binaries.tar.zst -C build-e2e
+
     - name: Deduce E2E CMake options
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       id: cmake_opts
       shell: bash
       env:
@@ -281,14 +322,14 @@ jobs:
           echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT
         fi
     - name: Configure E2E tests
-      if: inputs.tests_selector == 'e2e'
+      if: inputs.tests_selector == 'e2e' && inputs.e2e_binaries_artifact == ''
       run: |
-        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
+        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
     - name: SYCL End-to-end tests
       shell: bash {0}
       if: inputs.tests_selector == 'e2e'
       env:
-        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
+        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests --param test-mode=${{ inputs.e2e_testing_mode }} --param sycl_devices=${{ inputs.target_devices }} ${{ inputs.extra_lit_opts }}
       run: |
         ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
         exit_code=$?
@@ -375,3 +416,13 @@ jobs:
         grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY
 
         exit $ret
+    - name: Pack E2E binaries
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
+      run: tar -I 'zstd -9' -cf e2e_binaries.tar.zst -C ./build-e2e .
+    - name: Upload E2E binaries
+      if: ${{ always() && !cancelled() && inputs.e2e_testing_mode == 'build-only'}}
+      uses: actions/upload-artifact@v4
+      with:
+        name: sycl_e2e_bin_${{ inputs.artifact_suffix }}
+        path: e2e_binaries.tar.zst
+        retention-days: ${{ inputs.retention-days }}

sycl/test-e2e/AOT/cpu.cpp

@@ -7,6 +7,7 @@
 //===---------------------------------------------------------------------===//
 
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %S/Inputs/aot.cpp -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/AOT/double.cpp

@@ -2,6 +2,7 @@
 // using fp64 can be compiled AOT.
 
 // REQUIRES: ocloc, opencl-aot, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_tgllp -o %t.tgllp.out %s
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s
 // RUN: %if cpu %{ %{run} %t.x86.out %}

sycl/test-e2e/AOT/half.cpp

@@ -2,6 +2,7 @@
 // using fp16 can be compiled AOT.
 
 // REQUIRES: ocloc, opencl-aot, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_tgllp -o %t.tgllp.out %s
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s
 // RUN: %if cpu %{ %{run} %t.x86.out %}

sycl/test-e2e/Adapters/sycl-ls-gpu-default-any.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: gpu
+// REQUIRES: build-and-run-mode
 
 // TODO: Remove unsetting SYCL_DEVICE_FILTER when feature is dropped
 // RUN: env --unset=SYCL_DEVICE_FILTER  --unset=ONEAPI_DEVICE_SELECTOR sycl-ls --verbose >%t.default.out

sycl/test-e2e/Adapters/sycl-ls-gpu-default-level-zero.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: gpu, level-zero
+// REQUIRES: build-and-run-mode
 
 // TODO: Remove unsetting SYCL_DEVICE_FILTER when feature is dropped
 // RUN: env --unset=SYCL_DEVICE_FILTER --unset=ONEAPI_DEVICE_SELECTOR sycl-ls --verbose >%t.default.out

sycl/test-e2e/Adapters/sycl-ls-gpu-level-zero.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: gpu, level_zero
+// REQUIRES: build-and-run-mode
 
 // RUN: sycl-ls --verbose >%t.default.out
 // RUN: FileCheck %s --check-prefixes=CHECK-GPU-BUILTIN,CHECK-GPU-CUSTOM --input-file %t.default.out

sycl/test-e2e/Adapters/sycl-ls-gpu-opencl.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: gpu, opencl
+// REQUIRES: build-and-run-mode
 
 // RUN: env ONEAPI_DEVICE_SELECTOR="opencl:*" sycl-ls --verbose >%t.opencl.out
 // RUN: FileCheck %s --check-prefixes=CHECK-GPU-BUILTIN,CHECK-GPU-CUSTOM --input-file %t.opencl.out

sycl/test-e2e/Adapters/sycl-ls-uuid-subdevs.cpp

@@ -1,4 +1,5 @@
 /* Test to check that sycl-ls is outputting UUID and number of sub and sub-sub
+// REQUIRES: build-and-run-mode
  * devices. */
 // REQUIRES:  gpu, level_zero
 

sycl/test-e2e/AddressSanitizer/nullpointer/private_nullptr.cpp

@@ -7,7 +7,7 @@
 // RUN: %{run} not %t3.out 2>&1 | FileCheck %s
 
 // FIXME: There's an issue in gfx driver, so this test pending here.
-// XFAIL: *
+// XFAIL: run-mode
 
 #include <sycl/detail/core.hpp>
 #include <sycl/ext/oneapi/experimental/address_cast.hpp>

sycl/test-e2e/BFloat16/bfloat16_example_aot.cpp

@@ -4,6 +4,7 @@
 ///
 
 // REQUIRES: opencl-aot, ocloc, gpu-intel-gen12, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64 %s -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/BFloat16/bfloat16_example_aot_cpu.cpp

@@ -4,6 +4,7 @@
 ///
 
 // REQUIRES: opencl-aot, ocloc, gpu-intel-gen12, any-device-is-cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64,spir64_gen -Xsycl-target-backend=spir64_gen "-device dg1" %s -o %t.out
 // RUN: %if cpu %{ %{run} %t.out %}

sycl/test-e2e/Compression/compression_aot.cpp

@@ -1,5 +1,6 @@
 // End-to-End test for testing device image compression in AOT.
 // REQUIRES: zstd, opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %O0 --offload-compress --offload-compression-level=3 %S/Inputs/single_kernel.cpp -o %t_compress.out
 // RUN: %{run} %t_compress.out

sycl/test-e2e/Compression/compression_separate_compile.cpp

@@ -2,6 +2,7 @@
 // seperatly compile and link device images.
 
 // REQUIRES: zstd, opencl-aot, cpu, linux
+// REQUIRES: build-and-run-mode
 
 //////////////////////  Compile device images
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fsycl-host-compiler=clang++ -fsycl-host-compiler-options='-std=c++17 -Wno-attributes -Wno-deprecated-declarations -fPIC -DENABLE_KERNEL1' -DENABLE_KERNEL1 -c %s -o %t_kernel1_aot.o

sycl/test-e2e/DeviceArchitecture/device_architecture_on_device_aot.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 %s -o %t.out
 // RUN: %{run} %t.out
 

sycl/test-e2e/DeviceCodeSplit/aot-cpu.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-device-code-split=per_source -fsycl-targets=spir64_x86_64 -I %S/Inputs -o %t.out %S/split-per-source-main.cpp %S/Inputs/split-per-source-second-file.cpp \
 // RUN: -fsycl-dead-args-optimization

sycl/test-e2e/DeviceLib/assert-aot.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu, linux
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -DSYCL_FALLBACK_ASSERT=1 -fsycl -fsycl-targets=spir64_x86_64 %S/assert.cpp -o %t.aot.out
 // RUN: env EXPECTED_SIGNAL=SIGABRT SHOULD_CRASH=1 %{run} %t.aot.out 2>&1 | FileCheck %S/assert.cpp --check-prefixes=CHECK-MESSAGE

sycl/test-e2e/DeviceLib/cmath-aot.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 // UNSUPPORTED: windows
 
 // DEFINE: %{mathflags} = %if cl_options %{/clang:-fno-fast-math%} %else %{-fno-fast-math%}

sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_arg_dim.cpp

@@ -14,7 +14,7 @@
 
 // Waiting for the commit in IGC to be pulled into the driver to resolve the
 // test.
-// XFAIL: !igc-dev || gpu-intel-dg2
+// XFAIL: (!igc-dev || gpu-intel-dg2) && run-mode
 // XFAIL-TRACKER: GSD-10510
 
 #include "common.hpp"

sycl/test-e2e/Matrix/joint_matrix_bf16_fill_k_cache_runtime_dim.cpp

@@ -14,7 +14,7 @@
 
 // Waiting for the commit in IGC to be pulled into the driver to resolve the
 // test.
-// XFAIL: !igc-dev || gpu-intel-dg2
+// XFAIL: (!igc-dev || gpu-intel-dg2) && run-mode
 // XFAIL-TRACKER: GSD-10510
 
 #include "common.hpp"

sycl/test-e2e/NewOffloadDriver/aot-cpu.cpp

@@ -1,4 +1,5 @@
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // Test with `--offload-new-driver`
 // RUN: %clangxx -fsycl -fsycl-device-code-split=per_source -fsycl-targets=spir64_x86_64 -I %S/Inputs -o %t.out %S/split-per-source-main.cpp %S/Inputs/split-per-source-second-file.cpp \

sycl/test-e2e/NewOffloadDriver/cpu.cpp

@@ -7,6 +7,7 @@
 //===---------------------------------------------------------------------===//
 
 // REQUIRES: opencl-aot, cpu
+// REQUIRES: build-and-run-mode
 
 // Test with  `--offload-new-driver`
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64 --offload-new-driver %S/Inputs/aot.cpp -o %t.out

sycl/test-e2e/NonUniformGroups/ballot_group.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 

sycl/test-e2e/NonUniformGroups/ballot_group_algorithms.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_ballot_group

sycl/test-e2e/NonUniformGroups/fixed_size_group.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 // REQUIRES: sg-32

sycl/test-e2e/NonUniformGroups/fixed_size_group_algorithms.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fsycl-device-code-split=per_kernel -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_fixed_size_group

sycl/test-e2e/NonUniformGroups/opportunistic_group.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: hip
 

sycl/test-e2e/NonUniformGroups/opportunistic_group_algorithms.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_opportunistic_group

sycl/test-e2e/NonUniformGroups/tangle_group.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fno-sycl-early-optimizations -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // UNSUPPORTED: cuda || hip
 

sycl/test-e2e/NonUniformGroups/tangle_group_algorithms.cpp

@@ -4,6 +4,7 @@
 // RUN: %if any-device-is-cpu && opencl-aot %{ %clangxx -fsycl -fsycl-targets=spir64_x86_64 -fno-sycl-early-optimizations -o %t.x86.out %s %}
 // RUN: %if cpu %{ %{run} %t.x86.out %}
 //
+// REQUIRES: build-and-run-mode
 // REQUIRES: cpu || gpu
 // REQUIRES: sg-32
 // REQUIRES: aspect-ext_oneapi_tangle_group