Merge pull request #2623 from aarongreig/aaron/documentFixtures

Document CTS fixtures and macros.

Author: aarongreig

test/adapters/cuda/context_tests.cpp

@@ -11,7 +11,7 @@
 #include <thread>
 
 using cudaUrContextCreateTest = uur::urDeviceTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(cudaUrContextCreateTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(cudaUrContextCreateTest);
 
 constexpr unsigned int known_cuda_api_version = 3020;
 

test/adapters/cuda/event_tests.cpp

@@ -10,7 +10,7 @@
 #include "raii.h"
 
 using cudaEventTest = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(cudaEventTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(cudaEventTest);
 
 // Testing the urEventGetInfo behaviour for natively constructed (Cuda) events.
 // Backend interop APIs can lead to creating event objects that are not fully

test/adapters/cuda/kernel_tests.cpp

@@ -9,7 +9,7 @@
 #include "uur/raii.h"
 
 using cudaKernelTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(cudaKernelTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(cudaKernelTest);
 
 // The first argument stores the implicit global offset
 inline constexpr size_t NumberOfImplicitArgsCUDA = 1;

test/adapters/cuda/memory_tests.cpp

@@ -8,7 +8,7 @@
 #include "uur/raii.h"
 
 using cudaMemoryTest = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(cudaMemoryTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(cudaMemoryTest);
 
 TEST_P(cudaMemoryTest, urMemBufferNoActiveContext) {
   constexpr size_t memSize = 1024u;

test/adapters/cuda/urContextGetNativeHandle.cpp

@@ -7,7 +7,7 @@
 #include "fixtures.h"
 
 using urCudaContextGetNativeHandle = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urCudaContextGetNativeHandle);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCudaContextGetNativeHandle);
 
 TEST_P(urCudaContextGetNativeHandle, Success) {
   ur_native_handle_t native_context = 0;

test/adapters/cuda/urDeviceCreateWithNativeHandle.cpp

@@ -7,7 +7,7 @@
 #include "fixtures.h"
 
 using urCudaDeviceCreateWithNativeHandle = uur::urPlatformTest;
-UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urCudaDeviceCreateWithNativeHandle);
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urCudaDeviceCreateWithNativeHandle);
 
 TEST_P(urCudaDeviceCreateWithNativeHandle, Success) {
   // get a device from cuda

test/adapters/cuda/urDeviceGetNativeHandle.cpp

@@ -7,7 +7,7 @@
 #include "fixtures.h"
 
 using urCudaGetDeviceNativeHandle = uur::urDeviceTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urCudaGetDeviceNativeHandle);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCudaGetDeviceNativeHandle);
 
 TEST_P(urCudaGetDeviceNativeHandle, Success) {
   ur_native_handle_t native_handle;

test/adapters/cuda/urEventCreateWithNativeHandle.cpp

@@ -8,7 +8,7 @@
 #include "raii.h"
 
 using urCudaEventCreateWithNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urCudaEventCreateWithNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCudaEventCreateWithNativeHandleTest);
 
 TEST_P(urCudaEventCreateWithNativeHandleTest, Success) {
   RAIICUevent cuda_event;

test/adapters/cuda/urEventGetNativeHandle.cpp

@@ -8,7 +8,7 @@
 #include "uur/raii.h"
 
 using urCudaEventGetNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urCudaEventGetNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCudaEventGetNativeHandleTest);
 
 TEST_P(urCudaEventGetNativeHandleTest, Success) {
   constexpr size_t buffer_size = 1024;

test/adapters/cuda/urQueueGetNativeHandle.cpp

@@ -8,7 +8,7 @@
 #include "queue.hpp"
 
 using urCudaQueueGetNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urCudaQueueGetNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urCudaQueueGetNativeHandleTest);
 
 TEST_P(urCudaQueueGetNativeHandleTest, Success) {
   CUstream Stream;

test/adapters/hip/test_context.cpp

@@ -12,7 +12,7 @@
 #include "uur/raii.h"
 
 using urHipContextTest = uur::urDeviceTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urHipContextTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urHipContextTest);
 
 TEST_P(urHipContextTest, ActiveContexts) {
   uur::raii::Context context = nullptr;

test/adapters/hip/test_event.cpp

@@ -25,7 +25,7 @@ struct RAIIHipEvent {
 };
 
 using urHipEventTest = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urHipEventTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urHipEventTest);
 
 // Testing the urEventGetInfo behaviour for natively constructed (HIP) events.
 // Backend interop APIs can lead to creating event objects that are not fully

test/adapters/hip/urContextGetNativeHandle.cpp

@@ -7,7 +7,7 @@
 #include "fixtures.h"
 
 using urHipContextGetNativeHandleTest = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urHipContextGetNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urHipContextGetNativeHandleTest);
 
 TEST_P(urHipContextGetNativeHandleTest, Success) {
   ur_native_handle_t native_context = 0;

test/adapters/hip/urDeviceGetNativeHandle.cpp

@@ -7,7 +7,7 @@
 #include "fixtures.h"
 
 using urHipGetDeviceNativeHandle = uur::urDeviceTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urHipGetDeviceNativeHandle);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urHipGetDeviceNativeHandle);
 
 TEST_P(urHipGetDeviceNativeHandle, Success) {
   ur_native_handle_t native_handle;

test/adapters/hip/urEventGetNativeHandle.cpp

@@ -8,7 +8,7 @@
 #include "uur/raii.h"
 
 using urHipEventGetNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urHipEventGetNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urHipEventGetNativeHandleTest);
 
 TEST_P(urHipEventGetNativeHandleTest, Success) {
   constexpr size_t buffer_size = 1024;

test/adapters/level_zero/event_cache_tests.cpp

@@ -183,18 +183,18 @@ printFlags(const testing::TestParamInfo<typename T::ParamType> &info) {
   return platform_device_name + "__" + str;
 }
 
-UUR_DEVICE_TEST_SUITE_P(urEventCacheTest,
-                        ::testing::
-                            Combine(
-                                testing::Values(0,
-                                                UR_QUEUE_FLAG_DISCARD_EVENTS),
-                                testing::Values(
-                                    0,
-                                    UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE),
-                                // TODO: why the test fails with
-                                // UR_QUEUE_FLAG_SUBMISSION_BATCHED?
-                                testing::
-                                    Values(UR_QUEUE_FLAG_SUBMISSION_IMMEDIATE /*, UR_QUEUE_FLAG_SUBMISSION_BATCHED */),
-                                testing::Values(
-                                    0, UR_QUEUE_FLAG_PROFILING_ENABLE)),
-                        printFlags<urEventCacheTest>);
+UUR_DEVICE_TEST_SUITE_WITH_PARAM(urEventCacheTest,
+                                 ::testing::Combine(
+                                     testing::Values(
+                                         0, UR_QUEUE_FLAG_DISCARD_EVENTS),
+                                     testing::Values(
+                                         0,
+                                         UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE),
+                                     // TODO: why the test fails with
+                                     // UR_QUEUE_FLAG_SUBMISSION_BATCHED?
+                                     testing::
+                                         Values(
+                                             UR_QUEUE_FLAG_SUBMISSION_IMMEDIATE /*, UR_QUEUE_FLAG_SUBMISSION_BATCHED */),
+                                     testing::Values(
+                                         0, UR_QUEUE_FLAG_PROFILING_ENABLE)),
+                                 printFlags<urEventCacheTest>);

test/adapters/level_zero/ipc.cpp

@@ -13,7 +13,7 @@
 #endif
 
 using urL0IpcTest = uur::urContextTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urL0IpcTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urL0IpcTest);
 
 TEST_P(urL0IpcTest, SuccessHostL0Ipc) {
   ur_device_usm_access_capability_flags_t hostUSMSupport = 0;

test/adapters/level_zero/multi_device_event_cache_tests.cpp

@@ -24,7 +24,7 @@ static std::shared_ptr<_zel_tracer_handle_t> tracer = [] {
 }();
 
 using urMultiQueueMultiDeviceEventCacheTest = uur::urAllDevicesTest;
-UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMultiQueueMultiDeviceEventCacheTest);
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urMultiQueueMultiDeviceEventCacheTest);
 
 TEST_P(urMultiQueueMultiDeviceEventCacheTest,
        GivenMultiSubDeviceWithQueuePerSubDeviceThenEventIsSharedBetweenQueues) {

test/adapters/level_zero/urEnqueueMemBufferMapHostPtr.cpp

@@ -11,7 +11,7 @@
 using urEnqueueMemBufferMapTestWithParamL0 =
     uur::urMemBufferQueueTestWithParam<uur::mem_buffer_test_parameters_t>;
 
-UUR_DEVICE_TEST_SUITE_P(
+UUR_DEVICE_TEST_SUITE_WITH_PARAM(
     urEnqueueMemBufferMapTestWithParamL0,
     ::testing::ValuesIn(uur::mem_buffer_test_parameters),
     uur::printMemBufferTestString<urEnqueueMemBufferMapTestWithParamL0>);

test/adapters/level_zero/urEventCreateWithNativeHandle.cpp

@@ -15,7 +15,7 @@
 
 using namespace std::chrono_literals;
 using urLevelZeroEventNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urLevelZeroEventNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urLevelZeroEventNativeHandleTest);
 
 #define TEST_MEMCPY_SIZE 4096
 

test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp

@@ -10,7 +10,7 @@
 #include <uur/fixtures.h>
 
 using urLevelZeroKernelNativeHandleTest = uur::urQueueTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urLevelZeroKernelNativeHandleTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urLevelZeroKernelNativeHandleTest);
 
 TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) {
   ze_context_handle_t native_context;

test/adapters/level_zero/urProgramLink.cpp

@@ -8,7 +8,7 @@
 #include <uur/fixtures.h>
 
 using urLevelZeroProgramLinkTest = uur::urProgramTest;
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urLevelZeroProgramLinkTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urLevelZeroProgramLinkTest);
 
 TEST_P(urLevelZeroProgramLinkTest, InvalidLinkOptionsPrintedInLog) {
   ur_program_handle_t linked_program = nullptr;

test/adapters/level_zero/v2/command_list_cache_test.cpp

@@ -21,7 +21,7 @@
 
 struct CommandListCacheTest : public uur::urContextTest {};
 
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(CommandListCacheTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(CommandListCacheTest);
 
 TEST_P(CommandListCacheTest, CanStoreAndRetriveImmediateAndRegularCmdLists) {
   v2::command_list_cache_t cache(context->getZeHandle());

test/adapters/level_zero/v2/deferred_kernel.cpp

@@ -22,7 +22,7 @@ struct urEnqueueKernelLaunchTest : uur::urKernelExecutionTest {
   size_t global_offset = 0;
   size_t n_dimensions = 1;
 };
-UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urEnqueueKernelLaunchTest);
+UUR_INSTANTIATE_DEVICE_TEST_SUITE(urEnqueueKernelLaunchTest);
 
 TEST_P(urEnqueueKernelLaunchTest, DeferredKernelRelease) {
   ur_mem_handle_t buffer = nullptr;
@@ -111,7 +111,7 @@ struct urMultiQueueLaunchKernelDeferFreeTest
   }
 };
 
-UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMultiQueueLaunchKernelDeferFreeTest);
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urMultiQueueLaunchKernelDeferFreeTest);
 
 TEST_P(urMultiQueueLaunchKernelDeferFreeTest, Success) {
   auto zeEvent1 = createZeEvent(context, devices[0]);

test/adapters/level_zero/v2/event_pool_test.cpp

@@ -147,8 +147,8 @@ static ProviderParams test_cases[] = {
     //{TEST_PROVIDER_COUNTER, EVENT_COUNTER, QUEUE_IMMEDIATE},
 };
 
-UUR_DEVICE_TEST_SUITE_P(EventPoolTest, testing::ValuesIn(test_cases),
-                        printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_WITH_PARAM(EventPoolTest, testing::ValuesIn(test_cases),
+                                 printParams<EventPoolTest>);
 
 TEST_P(EventPoolTest, InvalidDevice) {
   auto pool = cache->borrow(MAX_DEVICES, getParam().flags);
@@ -240,8 +240,9 @@ TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) {
 
 using EventPoolTestWithQueue = uur::urQueueTestWithParam<ProviderParams>;
 
-UUR_DEVICE_TEST_SUITE_P(EventPoolTestWithQueue, testing::ValuesIn(test_cases),
-                        printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_WITH_PARAM(EventPoolTestWithQueue,
+                                 testing::ValuesIn(test_cases),
+                                 printParams<EventPoolTest>);
 
 // TODO: actual min version is unknown, retest after drivers on CI are
 // updated.

test/adapters/level_zero/v2/memory_residency.cpp

@@ -8,7 +8,7 @@
 #include "uur/utils.h"
 
 using urMemoryResidencyTest = uur::urMultiDeviceContextTestTemplate<1>;
-UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMemoryResidencyTest);
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urMemoryResidencyTest);
 
 TEST_P(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) {
   static constexpr size_t allocSize = 1024 * 1024;

test/conformance/README.md

@@ -70,3 +70,82 @@ The following adapter matcher objects are available:
 * `uur::CUDA`
 * `uur::HIP`
 * `uur::NativeCPU`
+
+## Writing a new CTS test
+
+If you're writing a new CTS test you'll need to make use of the existing test
+fixtures and instantiation macros to access the adapters available for testing.
+The definitions for these can all be found in
+[fixtures.h](https://github.com/oneapi-src/unified-runtime/blob/main/test/conformance/testing/include/uur/fixtures.h).
+
+There are five "base" fixtures in that header that each correspond to an
+instantiation macro - specific macros are needed due to how gtest handles
+parameterization and printing. All of these make use of gtest's [value
+parameterization](http://google.github.io/googletest/advanced.html#how-to-write-value-parameterized-tests)
+to instantiate the tests for all the available backends. Two of the five base
+fixtures have a wrapper to allow for tests defining their own parameterization.
+
+The base fixtures and their macros are detailed below. Tests inheriting from
+fixtures other than the base ones (`urContextTest`, etc.) must be instantiated
+with the macro that corresponds to whatever base class is ultimately being
+inherited from. In the case of `urContextTest` we can see that it inherits
+directly from `urDeviceTest`, so we should use the `urDeviceTest` macro. For
+other fixtures you'll need to follow the inheritance back a few steps to figure
+it out.
+
+### `urAdapterTest`
+
+This fixture is intended for tests that will be run once for each adapter
+available. The only data member it provides is a `ur_adapter_handle_t`.
+
+Instantiated with the `UUR_INSTANTIATE_ADAPTER_TEST_SUITE` macro.
+
+### `urPlatformTest`
+
+This fixture is intended for tests that will be run once for each platform
+available (note the potentially one-to-many relationship between adapters and
+platforms). The only data member it provides is a `ur_platform_handle_t`.
+
+Instantiated with the `UUR_INSTANTIATE_PLATFORM_TEST_SUITE` macro.
+
+### `urDeviceTest`
+
+This fixture is intended for tests that will be run once for each device
+available. It provides a `ur_adapter_handle_t`, `ur_platform_handle_t` and a
+`ur_device_handle_t` (the former two corresponding to the latter).
+
+Instantiated with the `UUR_INSTANTIATE_DEVICE_TEST_SUITE` macro.
+
+### `urPlatformTestWithParam`
+
+This fixture functions the same as `urPlatformTest` except it allows value
+parameterization via a template parameter. Note the parameter you specify is
+accessed with `getParam()` rather than gtest's `GetParam()`. A platform handle
+is provided the same way it is in the non-parameterized variant.
+
+Instantiated with the `UUR_PLATFORM_TEST_SUITE_WITH_PARAM` macro, which, in
+addition to the fixture, takes a `::testing::Values` list of parameter values
+and a printer (more detail about that below).
+
+### `urDeviceTestWithParam`
+
+As with the parameterized platform fixture this functions identically to
+`urDeviceTest` with the addition of the template parameter for
+parameterization.
+
+Instantiated with the `UUR_DEVICE_TEST_SUITE_WITH_PARAM` macro.
+
+### Parameter printers
+
+When instantiating tests based on the parameterized fixtures you need to
+provide a printer function along with the value list. This determines how the
+parameter values are incorporated into the test name. If your parameter type
+has a `<<` operator defined for it, you can simply use the
+`platformTestWithParamPrinter`/`deviceTestWithParamPrinter` helper functions for
+this.
+
+If you find yourself needing to write a custom printer function, bear in mind
+that due to the parameterization wrapper it'll need to deal with a tuple
+containing the platform/device information and your parameter. You should
+reference the implementations of `platformTestWithParamPrinter` and
+`deviceTestWithParamPrinter` to see how this is handled.

test/conformance/adapter/urAdapterGetInfo.cpp

@@ -10,7 +10,7 @@
 
 using urAdapterGetInfoTest = uur::urAdapterTest;
 
-UUR_INSTANTIATE_ADAPTER_TEST_SUITE_P(urAdapterGetInfoTest);
+UUR_INSTANTIATE_ADAPTER_TEST_SUITE(urAdapterGetInfoTest);
 
 TEST_P(urAdapterGetInfoTest, SuccessBackend) {
   ur_adapter_info_t property_name = UR_ADAPTER_INFO_BACKEND;

test/conformance/adapter/urAdapterGetLastError.cpp

@@ -11,7 +11,7 @@ struct urAdapterGetLastErrorTest : uur::urAdapterTest {
   const char *message = nullptr;
 };
 
-UUR_INSTANTIATE_ADAPTER_TEST_SUITE_P(urAdapterGetLastErrorTest);
+UUR_INSTANTIATE_ADAPTER_TEST_SUITE(urAdapterGetLastErrorTest);
 
 TEST_P(urAdapterGetLastErrorTest, Success) {
   // We can't reliably generate a UR_RESULT_ERROR_ADAPTER_SPECIFIC error to

test/conformance/adapter/urAdapterRelease.cpp

@@ -7,7 +7,7 @@
 #include <uur/fixtures.h>
 
 using urAdapterReleaseTest = uur::urAdapterTest;
-UUR_INSTANTIATE_ADAPTER_TEST_SUITE_P(urAdapterReleaseTest);
+UUR_INSTANTIATE_ADAPTER_TEST_SUITE(urAdapterReleaseTest);
 
 TEST_P(urAdapterReleaseTest, Success) {
   uint32_t referenceCountBefore = 0;