Revert "[UR] Move conformance tests over to lit" (#18382)

Reverts #17998

Author: kbenzie

.github/workflows/ur-build-hw.yml

@@ -13,6 +13,11 @@ on:
       runner_name:
         required: true
         type: string
+      platform:
+        description: "Platform string, `UR_CTS_ADAPTER_PLATFORM` will be set to this."
+        required: false
+        type: string
+        default: ""
       static_loader:
         required: false
         type: string
@@ -41,6 +46,11 @@ on:
       runner_name:
         required: true
         type: string
+      platform:
+        description: "Platform string, `UR_CTS_ADAPTER_PLATFORM` will be set to this."
+        required: false
+        type: string
+        default: ""
       static_loader:
         required: false
         type: string
@@ -79,6 +89,7 @@ jobs:
           {
             name: "${{inputs.adapter_name}}",
             other_name: "${{inputs.other_adapter_name}}",
+            platform: "${{inputs.platform}}",
             static_Loader: "${{inputs.static_loader}}",
             static_adapter: "${{inputs.static_loader}}"
           }
@@ -94,6 +105,7 @@ jobs:
     steps:
     # TODO:
     # - investigate if DUR_CONFORMANCE_AMD_ARCH could be removed
+    # - find better way to handle platform param (e.g. "Intel(R) OpenCL" -> "opencl")
     # - switch to Ninja generator in CMake
     # - downloading DPC++ should be integrated somehow; most likely use nightly release.
     #
@@ -144,6 +156,7 @@ jobs:
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
+        -DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }}
         ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }}
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
@@ -171,8 +184,7 @@ jobs:
     - name: Test adapters
       env:
         ZE_ENABLE_LOADER_DEBUG_TRACE: 1
-        LIT_OPTS: " --timeout 120"
-      run: cmake --build build -j $(nproc) -- check-unified-runtime-conformance
+      run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir build --output-on-failure -L "conformance" --timeout 600 -VV
 
     - name: Get information about platform
       if: ${{ always() }}

.github/workflows/ur-precommit.yml

@@ -77,10 +77,12 @@ jobs:
             image_options: -u 1001 --privileged --cap-add SYS_ADMIN --gpus all
           - name: OPENCL
             runner: UR_OPENCL
+            platform: "Intel(R) OpenCL"
             docker_image: "ghcr.io/intel/llvm/ubuntu2204_build:latest"
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN
           - name: OPENCL
             runner: UR_OPENCL
+            platform: "OPENCL:Intel(R) OpenCL"
             other_adapter: NATIVE_CPU
             docker_image: "ghcr.io/intel/llvm/ubuntu2204_build:latest"
             image_options: -u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN

unified-runtime/CMakeLists.txt

@@ -65,7 +65,8 @@ set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
 set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
-set(UR_CONFORMANCE_SELECTOR "" CACHE STRING "If nonempty, the device selector for conformance tests")
+option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
+option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" OFF)
 option(UR_USE_DEBUG_POSTFIX "Enable debug postfix 'd' for libraries" OFF)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")

unified-runtime/README.md

@@ -128,7 +128,8 @@ List of options provided by CMake:
 | UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON |
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |
 | UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` |
-| UR_CONFORMANCE_SELECTOR | `ONEAPI_DEVICE_SELECTOR` for conformance testing | string | All enabled adapters |
+| UR_CONFORMANCE_ENABLE_MATCH_FILES | Enable CTS match files | ON/OFF | ON |
+| UR_CONFORMANCE_TEST_LOADER | Additionally build and run "loader" tests for the CTS | ON/OFF | OFF |
 | UR_BUILD_ADAPTER_L0     | Build the Level-Zero adapter            | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter                | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_CUDA   | Build the CUDA adapter                  | ON/OFF     | OFF     |

unified-runtime/scripts/core/CONTRIB.rst

@@ -272,21 +272,14 @@ Conformance tests *must* not make assumptions about the adapter under test.
 Tests fixtures or cases *must* query for support of optional features and skip
 testing if unsupported by the adapter.
 
-Conformance tests are ran as part of the main ``test`` target, but can also be
-executed using the `check-unified-runtime-conformance` target as follows:
+All tests in the Unified Runtime project are configured to use CTest to run. 
+All conformance tests have the ``conformance`` label attached to them which 
+allows them to be run independently. To run all the conformance tests, execute 
+the following command from the build directory.
 
 .. code-block:: console
      
-    cmake --build build --target check-unified-runtime-conformance
-
-A specific device/adapter can be specified by using ``ONEAPI_DEVICE_SELECTOR``
-in the same way as in the `DPC++ compiler
-<https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md#oneapi_device_selector>`_.
-
-A number of tests are skipped on certain adapters due to being known failures.
-To force all tests to run, including known failures,
-``UR_CTS_ALSO_RUN_KNOWN_FAILURES`` may be set to ``1`` as an environment
-variable.
+    ctest -L "conformance"
 
 Experimental Features
 =====================

unified-runtime/scripts/core/INTRO.rst

@@ -417,12 +417,26 @@ CTS Environment Variables
 
 The following environment variables are used by the CTS runner and can be used to specify the platform that the test
 framework should run on. This can be used during development and testing to run CTS tests in case multiple platforms
-are available. Conformance tests will be skipped if there are no valid platforms.
+are available. If both filters are specified, then they both must match a platform for it to be selected. If there are
+no valid platforms, then the tests will fail. Command line arguments take priority over these variables.
 
-.. envvar:: ONEAPI_DEVICE_SELECTOR
+.. envvar:: UR_CTS_ADAPTER_PLATFORM
 
-   If ``ONEAPI_DEVICE_SELECTOR`` is set in the environment, only devices
-   matched by the selector will be tested.
+    A specifier list in the form of `[(backend):](platform name)[;[(backend)]:(platform name)]...`. If a backend
+    specific specifier is present in the list and the test is running for that backend, the device with the given name
+    is chosen. Otherwise, it must match the name of the specifier from the list with no backend. Backend names are case-
+    insensitive, however platform names must match exactly.
+
+    For example, if the test device has multiple platforms and you want to run tests on the "ABC Corp" backend when
+    testing OpenCL and "XYZ Org" when testing level zero, you'd use `OPENCL:ABC Corp;LEVEL_ZERO:XYZ Org`. This form is
+    useful when running the `build` target with a build with multiple backends.
+
+    For testing only one platform, the backend can be omitted. For example, just `ABC Corp` is sufficient if the tests
+    are only going to be testing OpenCL.
+
+.. envvar:: UR_CTS_BACKEND
+
+    A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
 
 .. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
 

unified-runtime/scripts/ctest_parser.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2022 Intel Corporation
+#
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from subprocess import Popen, DEVNULL, PIPE
+import argparse
+import os
+import json
+
+TMP_RESULTS_FILE = "tmp-results-file.json"
+
+
+def get_cts_test_suite_names(working_directory):
+    process = Popen(
+        ["ctest", "--show-only=json-v1"],
+        cwd=working_directory,
+        stdout=PIPE,
+        env=os.environ.copy(),
+    )
+    out, _ = process.communicate()
+    testsuites = json.loads(out)
+    return [test["name"][: test["name"].rfind("-")] for test in testsuites["tests"]]
+
+
+def percent(amount, total):
+    return round((amount / (total or 1)) * 100, 2)
+
+
+def summarize_results(results):
+    total = results["Total"]
+    total_passed = len(results["Passed"])
+    total_skipped = len(results["Skipped"])
+    total_failed = len(results["Failed"])
+    total_crashed = total - (total_passed + total_skipped + total_failed)
+
+    pass_rate_incl_skipped = percent(total_passed + total_skipped, total)
+    pass_rate_excl_skipped = percent(total_passed, total)
+
+    skipped_rate = percent(total_skipped, total)
+    failed_rate = percent(total_failed, total)
+    crash_rate = percent(total_crashed, total)
+
+    ljust_param = len(str(total))
+
+    print(
+        f"""[CTest Parser] Results:
+    Total    [{str(total).ljust(ljust_param)}]
+    Passed   [{str(total_passed).ljust(ljust_param)}]    ({pass_rate_incl_skipped}%) - ({pass_rate_excl_skipped}% with skipped tests excluded)
+    Skipped  [{str(total_skipped).ljust(ljust_param)}]    ({skipped_rate}%)
+    Failed   [{str(total_failed).ljust(ljust_param)}]    ({failed_rate}%)
+    Crashed  [{str(total_crashed).ljust(ljust_param)}]    ({crash_rate}%)
+"""
+    )
+
+
+def parse_results(results):
+    parsed_results = {
+        "Passed": {},
+        "Skipped": {},
+        "Failed": {},
+        "Crashed": {},
+        "Total": 0,
+        "Success": True,
+    }
+    for _, result in results.items():
+        if result["actual"] is None:
+            parsed_results["Success"] = False
+            parsed_results["Total"] += result["expected"]["tests"]
+            continue
+
+        parsed_results["Total"] += result["actual"]["tests"]
+        for testsuite in result["actual"].get("testsuites"):
+            for test in testsuite.get("testsuite"):
+                test_name = f"{testsuite['name']}.{test['name']}"
+                test_time = test["time"]
+                if "failures" in test:
+                    parsed_results["Failed"][test_name] = {"time": test_time}
+                elif test["result"] == "SKIPPED":
+                    parsed_results["Skipped"][test_name] = {"time": test_time}
+                else:
+                    parsed_results["Passed"][test_name] = {"time": test_time}
+    return parsed_results
+
+
+def run(args):
+    results = {}
+
+    tmp_results_file = f"{args.ctest_path}/{TMP_RESULTS_FILE}"
+    env = os.environ.copy()
+    env["GTEST_OUTPUT"] = f"json:{tmp_results_file}"
+
+    test_suite_names = get_cts_test_suite_names(f"{args.ctest_path}/test/conformance/")
+
+    ## try and list all the available tests
+    for suite in test_suite_names:
+        results[suite] = {}
+        test_executable = f"{args.ctest_path}/bin/test-{suite}"
+        process = Popen(
+            [test_executable, "--gtest_list_tests"],
+            env=env,
+            stdout=DEVNULL if args.quiet else None,
+            stderr=DEVNULL if args.quiet else None,
+        )
+        process.wait()
+        try:
+            with open(tmp_results_file, "r") as test_list:
+                all_tests = json.load(test_list)
+                results[suite]["expected"] = all_tests
+            os.remove(tmp_results_file)
+        except FileNotFoundError:
+            print(f"Could not discover tests for {suite}")
+
+    for suite in test_suite_names:
+        ctest_path = f"{args.ctest_path}/test/conformance/{suite}"
+        process = Popen(
+            ["ctest", ctest_path],
+            env=env,
+            cwd=ctest_path,
+            stdout=DEVNULL if args.quiet else None,
+            stderr=DEVNULL if args.quiet else None,
+        )
+        process.wait()
+
+        try:
+            with open(tmp_results_file, "r") as results_file:
+                json_data = json.load(results_file)
+                results[suite]["actual"] = json_data
+            os.remove(tmp_results_file)
+        except FileNotFoundError:
+            results[suite]["actual"] = None
+            print(
+                "\033[91m"
+                + f"Conformance test suite '{suite}' : likely crashed!"
+                + "\033[0m"
+            )
+
+    return results
+
+
+def dir_path(string):
+    if os.path.isdir(string):
+        return os.path.abspath(string)
+    else:
+        raise NotADirectoryError(string)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "ctest_path",
+        type=dir_path,
+        nargs="?",
+        default=".",
+        help="Optional path to test directory containing "
+        "CTestTestfile. Defaults to current directory.",
+    )
+    parser.add_argument(
+        "-q", "--quiet", action="store_true", help="Output only failed tests."
+    )
+    args = parser.parse_args()
+
+    raw_results = run(args)
+    parsed_results = parse_results(raw_results)
+    summarize_results(parsed_results)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        exit(130)

unified-runtime/source/adapters/level_zero/image.cpp

@@ -20,9 +20,7 @@
 
 #include "loader/ze_loader.h"
 
-namespace {
-
-} // namespace
+namespace {} // namespace
 
 namespace ur::level_zero {
 

unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp

@@ -31,9 +31,8 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py)
 add_custom_target(deps_check-unified-runtime)
 if(UR_STANDALONE_BUILD)
   add_custom_target(check-unified-runtime
-    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
+    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
     DEPENDS deps_check-unified-runtime
-    USES_TERMINAL
   )
 else()
   # Use the LLVM method to add the test suite - this also registers it under `check-all`
@@ -61,8 +60,7 @@ function(add_ur_lit_testsuite suite)
 
   if(UR_STANDALONE_BUILD)
     add_custom_target(${TARGET}
-      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
-      USES_TERMINAL
+      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
     )
   else()
     add_lit_testsuite(${TARGET} "Running Unified Runtime ${suite} tests"