[SYCL] Add support for the in-tree end-to-end tests (#8081)

We plan to move the intel/llvm-test-suite back in-tree. In order to do
that, we need to update the infrastructure first, both in this
repository and internally. Start by adding a single trivial test and
enabling cmake/Linux CI support.

We plan to run the tests in a "standalone" mode, similarly how current
intel/llvm-test-suite is used. As such, CMake target is decoupled from
general check-* processing and is not include into check-all
automatically.

CI support is extended via an additional parameter in test matrix
generator, `in_tree_e2e` boolean flag.

Author: aelovikov-intel

.github/workflows/sycl_linux_build_and_test.yml

@@ -160,6 +160,8 @@ jobs:
       run: tar -cJf llvm_sycl.tar.xz -C $GITHUB_WORKSPACE/build/install .
     - name: Pack LIT
       run: tar -cJf lit.tar.xz -C $GITHUB_WORKSPACE/src/llvm/utils/lit .
+    - name: Pack End-to-End LIT tests
+      run: tar -cJf e2e_tests.tar.xz -C $GITHUB_WORKSPACE/src/sycl/test-e2e .
     - name: Upload toolchain
       uses: actions/upload-artifact@v3
       with:
@@ -170,6 +172,11 @@ jobs:
       with:
         name: sycl_lit_${{ inputs.build_artifact_suffix }}
         path: lit.tar.xz
+    - name: Upload End-to-End LIT tests
+      uses: actions/upload-artifact@v3
+      with:
+        name: sycl_e2e_tests_${{ inputs.build_artifact_suffix }}
+        path: e2e_tests.tar.xz
 
   aws-start:
     name: Start AWS
@@ -241,9 +248,12 @@ jobs:
         sycl_archive: llvm_sycl.tar.xz
         lit_artifact: sycl_lit_${{ inputs.build_artifact_suffix }}
         lit_archive: lit.tar.xz
+        e2e_tests_artifact: sycl_e2e_tests_${{ inputs.build_artifact_suffix }}
+        e2e_tests_archive: e2e_tests.tar.xz
         check_sycl_all: ${{ matrix.check_sycl_all }}
         results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
         cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
+        in_tree_e2e: ${{ matrix.in_tree_e2e }}
 
   khronos_sycl_cts:
     needs: build

devops/actions/llvm_test_suite/action.yml

@@ -19,6 +19,13 @@ inputs:
     description: 'Name of LIT archive file'
     required: false
     default: 'lit.tar.xz'
+  e2e_tests_artifact:
+    description: 'Name of the artifact, that contains End-to-End LIT Tests'
+    required: true
+  e2e_tests_archive:
+    description: 'Name of End-to-End LIT Tests archive file'
+    required: false
+    default: 'e2e_tests.tar.xz'
   results_name_suffix:
     description: 'Name suffix of the results artifact'
     required: true
@@ -28,6 +35,10 @@ inputs:
   cmake_args:
     description: 'Extra arguments to cmake command'
     required: false
+  in_tree_e2e:
+    description: "Flag directing to use in-tree End-to-End LIT tests"
+    required: false
+    default: false
 
 post-if: false
 runs:
@@ -66,13 +77,33 @@ runs:
       ls -la
       tar -xf ${{ inputs.lit_archive }} -C lit
       rm -f ${{ inputs.lit_archive }}
+  - name: Download End-to-End LIT Tests
+    if: inputs.in_tree_e2e
+    uses: actions/download-artifact@v3
+    with:
+      name: ${{ inputs.e2e_tests_artifact }}
+  - name: Extract End-to-End LIT Tests
+    if: inputs.in_tree_e2e
+    shell: bash
+    run: |
+      mkdir test_e2e
+      ls -la
+      tar -xf ${{ inputs.e2e_tests_archive }} -C test_e2e
+      ls -R test_e2e
+      rm -f ${{ inputs.e2e_tests_archive }}
   - name: Configure
     shell: bash
     run: |
       echo "::group::CMake configuration"
       mkdir build
       export PATH=$PWD/toolchain/bin/:$PATH
-      cmake -GNinja -B./build -S./llvm_test_suite -DTEST_SUITE_SUBDIRS=SYCL -DCHECK_SYCL_ALL="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DTEST_SUITE_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      if ${{ inputs.in_tree_e2e }}
+      then
+        # TODO: Rename check_sycl_all input
+        cmake  -GNinja -B./build -S./test_e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DLLVM_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      else
+        cmake -GNinja -B./build -S./llvm_test_suite -DTEST_SUITE_SUBDIRS=SYCL -DCHECK_SYCL_ALL="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DTEST_SUITE_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      fi
       echo "::endgroup::"
   - name: Run testing
     shell: bash
@@ -81,7 +112,7 @@ runs:
       export PATH=$PWD/toolchain/bin/:$PATH
       # TODO make this part of container build
       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/hip/lib/:/opt/rocm/lib
-      export LIT_OPTS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}.json"
+      export LIT_OPTS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}_${{ inputs.in_tree_e2e }}.json"
       if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
         source /runtimes/oneapi-tbb/env/vars.sh;
       elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
@@ -103,13 +134,18 @@ runs:
       SYCL_PI_TRACE=-1 sycl-ls
       echo "::endgroup::"
       cd build
-      ninja check-sycl-all
+      if ${{ inputs.in_tree_e2e }}
+      then
+        ninja check-sycl-e2e
+      else
+        ninja check-sycl-all
+      fi
   - name: Upload test results
     uses: actions/upload-artifact@v1
     if: always()
     with:
       name: lit_results
-      path: build/results_${{ inputs.results_name_suffix }}.json
+      path: build/results_${{ inputs.results_name_suffix }}_${{ inputs.in_tree_e2e}}.json
   - name: Cleanup
     shell: bash
     if: always()
@@ -118,3 +154,4 @@ runs:
       rm -rf lit
       rm -rf build
       rm -rf llvm_test_suite
+      rm -rf test-e2e

devops/scripts/generate_test_matrix.js

@@ -47,8 +47,16 @@ module.exports = ({core, process}) => {
           } else {
             v["env"] = {};
           }
+          // Temporary support for both in-tree/out-of-tree (llvm-test-suite)
+          // end-to-end tests.
+          v["in_tree_e2e"] = false;
           enabledLTSConfigs.push(v);
           if (v["aws-type"]) enabledLTSAWSConfigs.push(v);
+
+          var inTreeV = Object.assign({}, v);
+          inTreeV["in_tree_e2e"] = true;
+          inTreeV["name"] = inTreeV["name"] + " In-Tree"
+          enabledLTSConfigs.push(inTreeV);
         }
       });
 

sycl/CMakeLists.txt

@@ -486,3 +486,6 @@ add_custom_target(deploy-sycl-toolchain
 
 # SYCL Runtime documentation
 add_subdirectory(doc)
+
+# SYCL End-to-End tests (former intel/llvm-test-suite).
+add_subdirectory(test-e2e)

sycl/test-e2e/CMakeLists.txt

@@ -0,0 +1,72 @@
+cmake_minimum_required(VERSION 3.15)
+
+message("Configuring SYCL End-to-End Tests")
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  project(sycl-e2e-test-suite C CXX)
+  set(SYCL_TEST_E2E_STANDALONE TRUE)
+endif()
+
+if(SYCL_TEST_E2E_STANDALONE)
+  if( NOT OpenCL_LIBRARY )
+    find_package(OpenCL)
+  endif()
+endif() # Standalone.
+
+if(SYCL_TEST_E2E_STANDALONE)
+  set(SYCL_CXX_COMPILER ${CMAKE_CXX_COMPILER})
+else()
+  set(SYCL_CXX_COMPILER "${LLVM_BINARY_DIR}/bin/clang++")
+endif() # Standalone.
+
+find_package(Threads REQUIRED)
+set(SYCL_THREADS_LIB ${CMAKE_THREAD_LIBS_INIT})
+
+if(NOT LLVM_LIT)
+  find_program(LLVM_LIT
+               NAMES llvm-lit lit.py lit
+               PATHS "${LLVM_MAIN_SRC_DIR}/utils/lit"
+               DOC "Path to lit.py")
+endif()
+
+set(SYCL_E2E_TESTS_LIT_FLAGS "-sv" CACHE STRING "Flags used when running lit")
+
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in"
+               "${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg")
+
+if(SYCL_TEST_E2E_TARGETS)
+  message("Configure iterative execution on multiple backends")
+  add_custom_target(check-sycl-e2e)
+  foreach(TARGET_STR ${SYCL_TEST_E2E_TARGETS})
+    string(REPLACE ":" ";" TARGET_LIST ${TARGET_STR})
+    list (GET TARGET_LIST 0 TARGET_BE)
+    list (GET TARGET_LIST 1 TARGET_DEVICES)
+
+    if ("${TARGET_BE}" STREQUAL "")
+      message(FATAL_ERROR
+        "invalid empty target backend specification in SYCL_TEST_E2E_TARGETS")
+    elseif("${TARGET_DEVICES}" STREQUAL "")
+      message(FATAL_ERROR
+        "invalid empty target device specification in SYCL_TEST_E2E_TARGETS")
+    endif()
+    message("Run on ${TARGET_DEVICES} for ${TARGET_BE}")
+
+    string(REPLACE "," "_" TARGET check-sycl-e2e-${TARGET_BE}-${TARGET_DEVICES})
+
+    add_custom_target(${TARGET}
+      COMMAND ${Python3_EXECUTABLE} ${LLVM_LIT} ${SYCL_E2E_TESTS_LIT_FLAGS} --param sycl_be=${TARGET_BE} --param target_devices=${TARGET_DEVICES} .
+      COMMENT "Running the SYCL tests for ${TARGET} backend"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      USES_TERMINAL
+    )
+    set_target_properties(${TARGET} PROPERTIES FOLDER "SYCL Level Zero tests")
+    add_dependencies(check-sycl-e2e ${TARGET})
+
+  endforeach()
+endif(SYCL_TEST_E2E_TARGETS)
+
+# TODO: Re-enable when moving the entire intel/llvm-test-suite back in-tree.
+# add_subdirectory(External)
+# add_subdirectory(ExtraTests)

sycl/test-e2e/TestSuiteMove/hello.cpp

@@ -0,0 +1,14 @@
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// RUN: %GPU_RUN_PLACEHOLDER %t.out
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
+
+// XFAIL: cpu
+
+#include <sycl/sycl.hpp>
+
+int main() {
+  sycl::queue q;
+  return q.get_device().get_info<sycl::info::device::device_type>() ==
+         sycl::info::device_type::cpu;
+}