[CI] Run E2E tests on PVC on postcommit (#15308)

This PR enables running E2E tests on PVC in the post-commit.

Currently, the latest PVC driver hangs when a large number of jobs are
submitted to it in parallel. Due to this, E2E tests might timeout
spuriously on PVC. The PVC bug is being tracked internally via
GSD-10241. To prevent spurious test timeouts, we restrict the number of
parallel E2E test runs on PVC and to prevent disruption in CI, we are
currently enabling PVC in post-commit only. When the bug gets fixed, we
can move PVC testing to pre-commit.

Author: uditagarwal97

.github/workflows/sycl-linux-run-tests.yml

@@ -110,6 +110,7 @@ on:
           - '["Linux", "gen12"]'
           - '["amdgpu"]'
           - '["Linux", "arc"]'
+          - '["Linux", "pvc"]'
           - '["cts-cpu"]'
           - '["Linux", "build"]'
       image:

.github/workflows/sycl-post-commit.yml

@@ -60,6 +60,21 @@ jobs:
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd
             target_devices: ext_oneapi_hip:gpu
             reset_intel_gpu: false
+          - name: E2E tests on Intel Ponte Vecchio GPU
+            runner: '["Linux", "pvc"]'
+            env: '{"LIT_FILTER_OUT":"ESIMD/unified_memory_api/"}'
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+            target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+            extra_lit_opts: -j 50
+          - name: E2E tests with dev igc on Intel Ponte Vecchio GPU
+            runner: '["Linux", "pvc"]'
+            env: '{"LIT_FILTER_OUT":"ESIMD/unified_memory_api/"}'
+            image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:devigc
+            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+            target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
+            use_igc_dev: true
+            extra_lit_opts: -j 50
           # Performance tests below. Specifics:
           #  - only run performance tests (use LIT_FILTER env)
           #  - ask llvm-lit to show all the output, even for PASS (-a)

sycl/test-e2e/AddressSanitizer/out-of-bounds/DeviceGlobal/device_global.cpp

@@ -6,6 +6,10 @@
 // RUN: %{build} %device_asan_flags -O2 -g -o %t3.out
 // RUN: %{run} not %t3.out 2>&1 | FileCheck %s
 
+// Flakily timesout on PVC
+// UNSUPPORTED: arch-intel_gpu_pvc
+// UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/16401
+
 #include <sycl/detail/core.hpp>
 
 #include <sycl/ext/oneapi/device_global/device_global.hpp>

sycl/test-e2e/Basic/accessor/accessor.cpp

@@ -1,6 +1,10 @@
 // RUN: %{build} -DSYCL2020_DISABLE_DEPRECATION_WARNINGS -o %t.out
 // RUN: %{run} %t.out
 
+// Test flakily fails on PVC.
+// UNSUPPORTED: arch-intel_gpu_pvc
+// UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/16401
+
 //==----------------accessor.cpp - SYCL accessor basic test ----------------==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

sycl/test-e2e/DeviceArchitecture/device_architecture_comparison_on_device_aot.cpp

@@ -1,5 +1,8 @@
 // REQUIRES: arch-intel_gpu_pvc, ocloc
 
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_pvc %s -o %t.out
 // RUN: %{run} %t.out
 

sycl/test-e2e/DeviceCodeSplit/grf.cpp

@@ -14,6 +14,8 @@
 //   compiler option
 
 // REQUIRES: arch-intel_gpu_pvc
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
 
 // RUN: %{build} -Wno-error=deprecated-declarations -o %t1.out
 // RUN: env SYCL_UR_TRACE=2 %{run} %t1.out 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NO-VAR

sycl/test-e2e/KernelAndProgram/persistent-cache-multi-device.cpp

@@ -5,6 +5,9 @@
 // RUN: env NEOReadDebugKeys=1 CreateMultipleRootDevices=3 SYCL_CACHE_PERSISTENT=1 SYCL_CACHE_TRACE=1 SYCL_CACHE_DIR=%t/cache_dir env -u XDG_CACHE_HOME env -u HOME %{run} %t.out 2>&1 | FileCheck %s --check-prefixes=CHECK-BUILD
 // RUN: env NEOReadDebugKeys=1 CreateMultipleRootDevices=3 SYCL_CACHE_PERSISTENT=1 SYCL_CACHE_TRACE=1 SYCL_CACHE_DIR=%t/cache_dir env -u XDG_CACHE_HOME env -u HOME %{run} %t.out 2>&1 | FileCheck %s --check-prefixes=CHECK-CACHE
 
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 // Test checks that persistent cache works correctly with multiple devices.
 
 #include <sycl/detail/core.hpp>

sycl/test-e2e/MemorySanitizer/lit.local.cfg

@@ -1,7 +1,8 @@
 # TRACKER: https://github.com/intel/llvm/issues/16184
-has_arch_gpu_intel_pvc = any('arch-intel_gpu_pvc' in T for T in config.sycl_dev_features.values())
-if not has_arch_gpu_intel_pvc:
-    config.unsupported_features += ['gpu']
+# TRACKER for PVC: https://github.com/intel/llvm/issues/16401
+#has_arch_gpu_intel_pvc = any('arch-intel_gpu_pvc' in T for T in config.sycl_dev_features.values())
+#if not has_arch_gpu_intel_pvc:
+config.unsupported_features += ['gpu']
 
 config.substitutions.append(
     ("%device_msan_flags", "-Xarch_device -fsanitize=memory")

sycl/test-e2e/OnlineCompiler/online_compiler_L0.cpp

@@ -1,5 +1,5 @@
 // REQUIRES: level_zero, level_zero_dev_kit, cm-compiler
-// XFAIL: gpu
+// XFAIL: gpu && !(arch-intel_gpu_pvc && igc-dev)
 // XFAIL-TRACKER: https://github.com/intel/llvm/issues/16406
 // RUN: %{build} -Wno-error=deprecated-declarations -DRUN_KERNELS %level_zero_options -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/OnlineCompiler/online_compiler_OpenCL.cpp

@@ -1,5 +1,5 @@
 // REQUIRES: opencl, opencl_icd, cm-compiler
-// XFAIL: gpu || cpu || accelerator
+// XFAIL: (gpu && !(arch-intel_gpu_pvc && igc-dev)) || cpu || accelerator
 // XFAIL-TRACKER: https://github.com/intel/llvm/issues/16406
 // RUN: %{build} -Wno-error=deprecated-declarations -DRUN_KERNELS %opencl_lib -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/ProgramManager/multi_device_bundle/build_twice.cpp

@@ -6,6 +6,9 @@
 // RUN: %{build} -o %t.out
 // RUN: env NEOReadDebugKeys=1 CreateMultipleRootDevices=3 SYCL_UR_TRACE=2 %{run} %t.out | FileCheck %s
 
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 #include <sycl/detail/core.hpp>
 #include <sycl/kernel_bundle.hpp>
 

sycl/test-e2e/ProgramManager/multi_device_bundle/device_libs_and_caching.cpp

@@ -29,6 +29,9 @@
 // Check the case when in-memory caching of the programs is disabled.
 // RUN: env SYCL_CACHE_IN_MEM=0 NEOReadDebugKeys=1 CreateMultipleRootDevices=4 %{run} %t.out
 
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 #include <cmath>
 #include <complex>
 #include <sycl/detail/core.hpp>

sycl/test-e2e/Regression/multithread_write_accessor.cpp

@@ -1,5 +1,9 @@
 // RUN: %{build} -o %t.out %threads_lib
 // RUN: %{run} %t.out
+
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 #include <sycl/detail/core.hpp>
 
 #include <cassert>

sycl/test-e2e/syclcompat/launch/launch_policy_lmem.cpp

@@ -26,6 +26,9 @@
 // UNSUPPORTED: linux && opencl && (gpu-intel-gen12 || gpu-intel-dg2)
 // UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/15275
 
+// XFAIL: arch-intel_gpu_pvc
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16401
+
 #include <sycl/detail/core.hpp>
 #include <sycl/ext/oneapi/properties/properties.hpp>
 #include <sycl/group_barrier.hpp>