[SYCL][OpenCL] Update E2E Graph tests to run using OpenCL, and new helper function to return early if Graphs are not supported by the device. Added OpenCL section to CommandGraph docs.

martygrant · reble · martygrant · commit 71895868b34f · 2023-11-28T10:41:53.000Z
Co-authored-by: Pablo Reble &lt;pablo@reble.org&gt;
diff --git a/sycl/doc/design/CommandGraph.md b/sycl/doc/design/CommandGraph.md
@@ -149,7 +149,7 @@ yet been implemented.
 Implementation of UR command-buffers
 for each of the supported SYCL 2020 backends.
 
-Currently Level Zero and CUDA backends are implemented.
+Backends which are implemented currently are: Level Zero, CUDA and OpenCL.
 More sub-sections will be added here as other backends are supported.
 
 ### Level Zero
@@ -246,3 +246,62 @@ the executable CUDA Graph that represent this series of operations.
 An executable CUDA Graph, which contains all commands and synchronization
 information, is saved in the UR command-buffer to allow for efficient
 graph resubmission.
+
+### OpenCL
+
+Command Buffers are defined in the OpenCL spec in the [cl_khr_command_buffer](https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Ext.html#cl_khr_command_buffer) extension.
+
+There are some gaps in both the OpenCL and UR specifications for Command 
+Buffers shown in the list below. There are implementations in the UR OpenCL
+adapter where there is matching support for each function in the list.
+
+| UR | OpenCL | Supported |
+| --- | --- | --- |
+| urCommandBufferCreateExp | clCreateCommandBufferKHR | Yes |
+| urCommandBufferRetainExp | clRetainCommandBufferKHR | Yes |
+| urCommandBufferReleaseExp | clReleaseCommandBufferKHR | Yes |
+| urCommandBufferFinalizeExp | clFinalizeCommandBufferKHR | Yes |
+| urCommandBufferAppendKernelLaunchExp | clCommandNDRangeKernelKHR | Yes |
+| urCommandBufferAppendUSMMemcpyExp |  | No |
+| urCommandBufferAppendUSMFillExp |  | No |
+| urCommandBufferAppendMembufferCopyExp | clCommandCopyBufferKHR | Yes |
+| urCommandBufferAppendMemBufferWriteExp |  | No |
+| urCommandBufferAppendMemBufferReadExp |  | No |
+| urCommandBufferAppendMembufferCopyRectExp | clCommandCopyBufferRectKHR | Yes |
+| urCommandBufferAppendMemBufferWriteRectExp |  | No |
+| urCommandBufferAppendMemBufferReadRectExp |  | No |
+| urCommandBufferAppendMemBufferFillExp | clCommandFillBufferKHR | Yes |
+| urCommandBufferEnqueueExp | clEnqueueCommandBufferKHR | Yes |
+|  | clCommandBarrierWithWaitListKHR | No |
+|  | clCommandCopyImageKHR | No |
+|  | clCommandCopyImageToBufferKHR | No |
+|  | clCommandFillImageKHR | No |
+|  | clGetCommandBufferInfoKHR | No |
+|  | clCommandSVMMemcpyKHR | No |
+|  | clCommandSVMMemFillKHR | No |
+
+Many of the OpenCL functions take a `cl_command_queue` parameter which is not
+present in most of the UR functions. Instead, when a new command buffer is
+created in `urCommandBufferCreateExp` we also create and maintain a new 
+internal `ur_queue_handle_t` with a reference stored inside of the 
+`ur_exp_command_buffer_handle_t_` struct. This internal queue is then used with
+the various append functions. The internal queue is retained and released 
+whenever the owning command buffer is retained or released.
+
+With command buffers being an OpenCL extension, each function is accessed by
+loading a function pointer to its implementation. These are defined in a common
+header file in the UR OpenCL adapter. The symbols for the functions are however
+defined in [OpenCL-Headers](https://github.com/KhronosGroup/OpenCL-Headers/blob/main/CL/cl_ext.h) but it is not known at this time what version of the headers will be used in the UR GitHub CI configuration, so loading the function
+pointers will be used until this can be verified. A future piece of work would 
+be replacing the custom defined symbols with the ones from OpenCL-Headers.
+
+The `UR_DEVICE_INFO_EXTENSIONS` enum can be used with `urDeviceGetInfo` to
+query if a specified device supports OpenCL command buffers. This will append 
+`ur_exp_command_buffer` to a string pointer passed to the function if the 
+extension is supported.
+
+Known implementations of cl_khr_command_buffer:
+- [OneAPI-Construction-Kit](https://github.com/codeplaysoftware/oneapi-construction-kit) (must enable `OCL_EXTENSION_cl_khr_command_buffer` when building)
+- [PoCL](http://portablecl.org/)
+- [Command-Buffer Emulation Layer](https://github.com/bashbaug/SimpleOpenCLSamples/tree/main/layers/10_cmdbufemu)
+
diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt
@@ -56,14 +56,14 @@ endif()
 if(SYCL_PI_UR_USE_FETCH_CONTENT)
   include(FetchContent)
 
-  set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git")
-  # commit 31b654f981f6098936e7f04c65803395a2ea343a
-  # Merge: 71957e84 3da21336
+  set(UNIFIED_RUNTIME_REPO "https://github.com/martygrant/unified-runtime.git")
+  # commit ec7982bac6cb3a6b9ed610cd6b7cb41fcbc780dc
+  # Merge: 62e6d2f9 5fb82924
   # Author: Kenneth Benzie (Benie) <k.benzie@codeplay.com>
-  # Date:   Wed Nov 22 11:27:33 2023 +0000
-  #     Merge pull request #1053 from jandres742/url0leakkey
-  #     [UR][L0] Add UR_L0_LEAKS_DEBUG key
-  set(UNIFIED_RUNTIME_TAG 31b654f981f6098936e7f04c65803395a2ea343a)
+  # Date:   Wed Nov 8 13:32:46 2023 +0000
+  #     Merge pull request #1022 from 0x12CC/l0_usm_error_checking_2
+  #     [UR][L0] Propagate OOM errors from `USMAllocationMakeResident`
+  set(UNIFIED_RUNTIME_TAG martin/openclCommandBuffers)
 
   if(SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO)
     set(UNIFIED_RUNTIME_REPO "${SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO}")
diff --git a/sycl/plugins/unified_runtime/pi2ur.hpp b/sycl/plugins/unified_runtime/pi2ur.hpp
@@ -127,7 +127,6 @@ static pi_result ur2piResult(ur_result_t urResult) {
     return PI_ERROR_INVALID_WORK_DIMENSION;
   case UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION:
     return PI_ERROR_INVALID_VALUE;
-
   case UR_RESULT_ERROR_PROGRAM_UNLINKED:
     return PI_ERROR_INVALID_PROGRAM_EXECUTABLE;
   case UR_RESULT_ERROR_OVERLAPPING_REGIONS:
@@ -140,6 +139,10 @@ static pi_result ur2piResult(ur_result_t urResult) {
     return PI_ERROR_OUT_OF_RESOURCES;
   case UR_RESULT_ERROR_ADAPTER_SPECIFIC:
     return PI_ERROR_PLUGIN_SPECIFIC_ERROR;
+  case UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP:
+    return PI_ERROR_INVALID_COMMAND_BUFFER_KHR;
+  case UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP:
+    return PI_ERROR_INVALID_SYNC_POINT_WAIT_LIST_KHR;
   case UR_RESULT_ERROR_UNKNOWN:
   default:
     return PI_ERROR_UNKNOWN;
diff --git a/sycl/test-e2e/Graph/Explicit/cycle_error.cpp b/sycl/test-e2e/Graph/Explicit/cycle_error.cpp
@@ -80,6 +80,14 @@ void CreateGraphWithCyclesTest(bool DisableCycleChecks) {
 }
 
 int main() {
+  {
+    queue Queue;
+
+    if (!are_graphs_supported(Queue)) {
+      return 0;
+    }
+  }
+
   // Test with cycle checks
   CreateGraphWithCyclesTest(false);
   // Test without cycle checks
diff --git a/sycl/test-e2e/Graph/Explicit/executable_graph_update_ordering.cpp b/sycl/test-e2e/Graph/Explicit/executable_graph_update_ordering.cpp
@@ -11,4 +11,4 @@
 
 #define GRAPH_E2E_EXPLICIT
 
-#include "../Inputs/executable_graph_update_ordering"
+#include "../Inputs/executable_graph_update_ordering.cpp"
diff --git a/sycl/test-e2e/Graph/Inputs/buffer_ordering.cpp b/sycl/test-e2e/Graph/Inputs/buffer_ordering.cpp
@@ -12,7 +12,6 @@
 #include "../graph_common.hpp"
 
 int main() {
-
   queue Queue{{sycl::ext::intel::property::queue::no_immediate_command_list{}}};
 
   if (!are_graphs_supported(Queue)) {
diff --git a/sycl/test-e2e/Graph/RecordReplay/dotp_multiple_queues.cpp b/sycl/test-e2e/Graph/RecordReplay/dotp_multiple_queues.cpp
@@ -10,7 +10,6 @@
 #include "../graph_common.hpp"
 
 int main() {
-
   property_list Properties{
       property::queue::in_order{},
       sycl::ext::intel::property::queue::no_immediate_command_list{}};
diff --git a/sycl/test-e2e/Graph/RecordReplay/executable_graph_update_ordering.cpp b/sycl/test-e2e/Graph/RecordReplay/executable_graph_update_ordering.cpp
@@ -11,4 +11,4 @@
 
 #define GRAPH_E2E_RECORD_REPLAY
 
-#include "../Inputs/executable_graph_update_ordering"
+#include "../Inputs/executable_graph_update_ordering.cpp"
diff --git a/sycl/test-e2e/Graph/device_query.cpp b/sycl/test-e2e/Graph/device_query.cpp
@@ -1,4 +1,4 @@
-// REQUIRES: cuda || level_zero, gpu
+// REQUIRES: opencl || cuda || level_zero
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 
@@ -21,7 +21,7 @@ int main() {
   auto Backend = Device.get_backend();
 
   if ((Backend == backend::ext_oneapi_level_zero) ||
-      (Backend == backend::ext_oneapi_cuda)) {
+      (Backend == backend::ext_oneapi_cuda) || (Backend == backend::opencl)) {
     assert(SupportsGraphs == exp_ext::graph_support_level::native);
   } else {
     assert(SupportsGraphs == exp_ext::graph_support_level::unsupported);

Original file line number	Diff line number	Diff line change
`@@ -11,4 +11,4 @@`
`11`	`11`
`12`	`12`	`#define GRAPH_E2E_EXPLICIT`
`13`	`13`
`14`		`-#include "../Inputs/executable_graph_update_ordering"`
	`14`	`+#include "../Inputs/executable_graph_update_ordering.cpp"`