[UR][L0] Add multi-device-compile experimental feature

Expand upon the introduction of `urProgramBuildExp` and include
`urProgramCompileExp` and `urProgramLinkExp` which include a device-list
in place of a context. These more closely align with the PI/OpenCL
analogues but only to introduce device-lists, not all extant arguments
from those entry-points. This patch also moves the `urProgramBuildExp`
definition into an experimental feature file and introduces a brief
document containing motivation.

Author: kbenzie

include/ur.py

@@ -203,7 +203,9 @@ class ur_function_v(IntEnum):
     COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP = 195    ## Enumerator for ::urCommandBufferAppendUSMPrefetchExp
     COMMAND_BUFFER_APPEND_USM_ADVISE_EXP = 196      ## Enumerator for ::urCommandBufferAppendUSMAdviseExp
     PROGRAM_BUILD_EXP = 197                         ## Enumerator for ::urProgramBuildExp
-    LOADER_CONFIG_SET_CODE_LOCATION_CALLBACK = 198  ## Enumerator for ::urLoaderConfigSetCodeLocationCallback
+    PROGRAM_COMPILE_EXP = 198                       ## Enumerator for ::urProgramCompileExp
+    PROGRAM_LINK_EXP = 199                          ## Enumerator for ::urProgramLinkExp
+    LOADER_CONFIG_SET_CODE_LOCATION_CALLBACK = 200  ## Enumerator for ::urLoaderConfigSetCodeLocationCallback
 
 class ur_function_t(c_int):
     def __str__(self):
@@ -2316,6 +2318,11 @@ class ur_exp_command_buffer_handle_t(c_void_p):
 ##        which is returned when querying device extensions.
 UR_COOPERATIVE_KERNELS_EXTENSION_STRING_EXP = "ur_exp_cooperative_kernels"
 
+###############################################################################
+## @brief The extension string which defines support for test
+##        which is returned when querying device extensions.
+UR_MULTI_DEVICE_COMPILE_EXTENSION_STRING_EXP = "ur_exp_multi_device_compile"
+
 ###############################################################################
 ## @brief Supported peer info
 class ur_exp_peer_info_v(IntEnum):
@@ -2635,16 +2642,32 @@ class ur_program_dditable_t(Structure):
 ###############################################################################
 ## @brief Function-pointer for urProgramBuildExp
 if __use_win_types:
-    _urProgramBuildExp_t = WINFUNCTYPE( ur_result_t, ur_context_handle_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+    _urProgramBuildExp_t = WINFUNCTYPE( ur_result_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+else:
+    _urProgramBuildExp_t = CFUNCTYPE( ur_result_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+
+###############################################################################
+## @brief Function-pointer for urProgramCompileExp
+if __use_win_types:
+    _urProgramCompileExp_t = WINFUNCTYPE( ur_result_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+else:
+    _urProgramCompileExp_t = CFUNCTYPE( ur_result_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+
+###############################################################################
+## @brief Function-pointer for urProgramLinkExp
+if __use_win_types:
+    _urProgramLinkExp_t = WINFUNCTYPE( ur_result_t, ur_context_handle_t, c_ulong, POINTER(ur_device_handle_t), c_ulong, POINTER(ur_program_handle_t), c_char_p, POINTER(ur_program_handle_t) )
 else:
-    _urProgramBuildExp_t = CFUNCTYPE( ur_result_t, ur_context_handle_t, ur_program_handle_t, c_ulong, POINTER(ur_device_handle_t), c_char_p )
+    _urProgramLinkExp_t = CFUNCTYPE( ur_result_t, ur_context_handle_t, c_ulong, POINTER(ur_device_handle_t), c_ulong, POINTER(ur_program_handle_t), c_char_p, POINTER(ur_program_handle_t) )
 
 
 ###############################################################################
 ## @brief Table of ProgramExp functions pointers
 class ur_program_exp_dditable_t(Structure):
     _fields_ = [
-        ("pfnBuildExp", c_void_p)                                       ## _urProgramBuildExp_t
+        ("pfnBuildExp", c_void_p),                                      ## _urProgramBuildExp_t
+        ("pfnCompileExp", c_void_p),                                    ## _urProgramCompileExp_t
+        ("pfnLinkExp", c_void_p)                                        ## _urProgramLinkExp_t
     ]
 
 ###############################################################################
@@ -3992,6 +4015,8 @@ def __init__(self, version : ur_api_version_t):
 
         # attach function interface to function address
         self.urProgramBuildExp = _urProgramBuildExp_t(self.__dditable.ProgramExp.pfnBuildExp)
+        self.urProgramCompileExp = _urProgramCompileExp_t(self.__dditable.ProgramExp.pfnCompileExp)
+        self.urProgramLinkExp = _urProgramLinkExp_t(self.__dditable.ProgramExp.pfnLinkExp)
 
         # call driver to get function pointers
         Kernel = ur_kernel_dditable_t()

include/ur_api.h

@@ -212,7 +212,9 @@ typedef enum ur_function_t {
     UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP = 195,                  ///< Enumerator for ::urCommandBufferAppendUSMPrefetchExp
     UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP = 196,                    ///< Enumerator for ::urCommandBufferAppendUSMAdviseExp
     UR_FUNCTION_PROGRAM_BUILD_EXP = 197,                                       ///< Enumerator for ::urProgramBuildExp
-    UR_FUNCTION_LOADER_CONFIG_SET_CODE_LOCATION_CALLBACK = 198,                ///< Enumerator for ::urLoaderConfigSetCodeLocationCallback
+    UR_FUNCTION_PROGRAM_COMPILE_EXP = 198,                                     ///< Enumerator for ::urProgramCompileExp
+    UR_FUNCTION_PROGRAM_LINK_EXP = 199,                                        ///< Enumerator for ::urProgramLinkExp
+    UR_FUNCTION_LOADER_CONFIG_SET_CODE_LOCATION_CALLBACK = 200,                ///< Enumerator for ::urLoaderConfigSetCodeLocationCallback
     /// @cond
     UR_FUNCTION_FORCE_UINT32 = 0x7fffffff
     /// @endcond
@@ -4076,43 +4078,6 @@ urProgramBuild(
     const char *pOptions          ///< [in][optional] pointer to build options null-terminated string.
 );
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Produces an executable program from one program, negates need for the
-///        linking step.
-///
-/// @details
-///     - The application may call this function from simultaneous threads.
-///     - Following a successful call to this entry point, the program passed
-///       will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type
-///       for each device in `hContext`.
-///
-/// @remarks
-///   _Analogues_
-///     - **clBuildProgram**
-///
-/// @returns
-///     - ::UR_RESULT_SUCCESS
-///     - ::UR_RESULT_ERROR_UNINITIALIZED
-///     - ::UR_RESULT_ERROR_DEVICE_LOST
-///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
-///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
-///         + `NULL == hContext`
-///         + `NULL == hProgram`
-///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
-///         + `NULL == phDevices`
-///     - ::UR_RESULT_ERROR_INVALID_PROGRAM
-///         + If `hProgram` isn't a valid program object.
-///     - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE
-///         + If an error occurred when building `hProgram`.
-UR_APIEXPORT ur_result_t UR_APICALL
-urProgramBuildExp(
-    ur_context_handle_t hContext,  ///< [in] handle of the context instance.
-    ur_program_handle_t hProgram,  ///< [in] Handle of the program to build.
-    uint32_t numDevices,           ///< [in] number of devices
-    ur_device_handle_t *phDevices, ///< [in][range(0, numDevices)] pointer to array of device handles
-    const char *pOptions           ///< [in][optional] pointer to build options null-terminated string.
-);
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Produces an executable program from one or more programs.
 ///
@@ -8446,6 +8411,131 @@ urKernelSuggestMaxCooperativeGroupCountExp(
     uint32_t *pGroupCountRet    ///< [out] pointer to maximum number of groups
 );
 
+#if !defined(__GNUC__)
+#pragma endregion
+#endif
+// Intel 'oneAPI' Unified Runtime Experimental APIs for multi-device compile
+#if !defined(__GNUC__)
+#pragma region multi device compile(experimental)
+#endif
+///////////////////////////////////////////////////////////////////////////////
+#ifndef UR_MULTI_DEVICE_COMPILE_EXTENSION_STRING_EXP
+/// @brief The extension string which defines support for test
+///        which is returned when querying device extensions.
+#define UR_MULTI_DEVICE_COMPILE_EXTENSION_STRING_EXP "ur_exp_multi_device_compile"
+#endif // UR_MULTI_DEVICE_COMPILE_EXTENSION_STRING_EXP
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Produces an executable program from one program, negates need for the
+///        linking step.
+///
+/// @details
+///     - The application may call this function from simultaneous threads.
+///     - Following a successful call to this entry point, the program passed
+///       will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type
+///       for each device in `phDevices`.
+///
+/// @remarks
+///   _Analogues_
+///     - **clBuildProgram**
+///
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_UNINITIALIZED
+///     - ::UR_RESULT_ERROR_DEVICE_LOST
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
+///         + `NULL == hProgram`
+///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
+///         + `NULL == phDevices`
+///     - ::UR_RESULT_ERROR_INVALID_PROGRAM
+///         + If `hProgram` isn't a valid program object.
+///     - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE
+///         + If an error occurred when building `hProgram`.
+UR_APIEXPORT ur_result_t UR_APICALL
+urProgramBuildExp(
+    ur_program_handle_t hProgram,  ///< [in] Handle of the program to build.
+    uint32_t numDevices,           ///< [in] number of devices
+    ur_device_handle_t *phDevices, ///< [in][range(0, numDevices)] pointer to array of device handles
+    const char *pOptions           ///< [in][optional] pointer to build options null-terminated string.
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Produces an executable program from one or more programs.
+///
+/// @details
+///     - The application may call this function from simultaneous threads.
+///     - Following a successful call to this entry point `hProgram` will
+///       contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type
+///       for each device in `phDevices`.
+///
+/// @remarks
+///   _Analogues_
+///     - **clCompileProgram**
+///
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_UNINITIALIZED
+///     - ::UR_RESULT_ERROR_DEVICE_LOST
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
+///         + `NULL == hProgram`
+///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
+///         + `NULL == phDevices`
+///     - ::UR_RESULT_ERROR_INVALID_PROGRAM
+///         + If `hProgram` isn't a valid program object.
+///     - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE
+///         + If an error occurred while compiling `hProgram`.
+UR_APIEXPORT ur_result_t UR_APICALL
+urProgramCompileExp(
+    ur_program_handle_t hProgram,  ///< [in][out] handle of the program to compile.
+    uint32_t numDevices,           ///< [in] number of devices
+    ur_device_handle_t *phDevices, ///< [in][range(0, numDevices)] pointer to array of device handles
+    const char *pOptions           ///< [in][optional] pointer to build options null-terminated string.
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Produces an executable program from one or more programs.
+///
+/// @details
+///     - The application may call this function from simultaneous threads.
+///     - Following a successful call to this entry point the program returned
+///       in `phProgram` will contain a binary of the
+///       ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in
+///       `phDevices`.
+///
+/// @remarks
+///   _Analogues_
+///     - **clLinkProgram**
+///
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_UNINITIALIZED
+///     - ::UR_RESULT_ERROR_DEVICE_LOST
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
+///         + `NULL == hContext`
+///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
+///         + `NULL == phDevices`
+///         + `NULL == phPrograms`
+///         + `NULL == phProgram`
+///     - ::UR_RESULT_ERROR_INVALID_PROGRAM
+///         + If one of the programs in `phPrograms` isn't a valid program object.
+///     - ::UR_RESULT_ERROR_INVALID_SIZE
+///         + `count == 0`
+///     - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE
+///         + If an error occurred while linking `phPrograms`.
+UR_APIEXPORT ur_result_t UR_APICALL
+urProgramLinkExp(
+    ur_context_handle_t hContext,          ///< [in] handle of the context instance.
+    uint32_t numDevices,                   ///< [in] number of devices
+    ur_device_handle_t *phDevices,         ///< [in][range(0, numDevices)] pointer to array of device handles
+    uint32_t count,                        ///< [in] number of program handles in `phPrograms`.
+    const ur_program_handle_t *phPrograms, ///< [in][range(0, count)] pointer to array of program handles.
+    const char *pOptions,                  ///< [in][optional] pointer to linker options null-terminated string.
+    ur_program_handle_t *phProgram         ///< [out] pointer to handle of program object created.
+);
+
 #if !defined(__GNUC__)
 #pragma endregion
 #endif
@@ -8962,7 +9052,6 @@ typedef struct ur_program_build_params_t {
 /// @details Each entry is a pointer to the parameter passed to the function;
 ///     allowing the callback the ability to modify the parameter's value
 typedef struct ur_program_build_exp_params_t {
-    ur_context_handle_t *phContext;
     ur_program_handle_t *phProgram;
     uint32_t *pnumDevices;
     ur_device_handle_t **pphDevices;
@@ -8979,6 +9068,17 @@ typedef struct ur_program_compile_params_t {
     const char **ppOptions;
 } ur_program_compile_params_t;
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for urProgramCompileExp
+/// @details Each entry is a pointer to the parameter passed to the function;
+///     allowing the callback the ability to modify the parameter's value
+typedef struct ur_program_compile_exp_params_t {
+    ur_program_handle_t *phProgram;
+    uint32_t *pnumDevices;
+    ur_device_handle_t **pphDevices;
+    const char **ppOptions;
+} ur_program_compile_exp_params_t;
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urProgramLink
 /// @details Each entry is a pointer to the parameter passed to the function;
@@ -8991,6 +9091,20 @@ typedef struct ur_program_link_params_t {
     ur_program_handle_t **pphProgram;
 } ur_program_link_params_t;
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for urProgramLinkExp
+/// @details Each entry is a pointer to the parameter passed to the function;
+///     allowing the callback the ability to modify the parameter's value
+typedef struct ur_program_link_exp_params_t {
+    ur_context_handle_t *phContext;
+    uint32_t *pnumDevices;
+    ur_device_handle_t **pphDevices;
+    uint32_t *pcount;
+    const ur_program_handle_t **pphPrograms;
+    const char **ppOptions;
+    ur_program_handle_t **pphProgram;
+} ur_program_link_exp_params_t;
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urProgramRetain
 /// @details Each entry is a pointer to the parameter passed to the function;

include/ur_ddi.h

@@ -411,16 +411,36 @@ typedef ur_result_t(UR_APICALL *ur_pfnGetProgramProcAddrTable_t)(
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function-pointer for urProgramBuildExp
 typedef ur_result_t(UR_APICALL *ur_pfnProgramBuildExp_t)(
-    ur_context_handle_t,
     ur_program_handle_t,
     uint32_t,
     ur_device_handle_t *,
     const char *);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function-pointer for urProgramCompileExp
+typedef ur_result_t(UR_APICALL *ur_pfnProgramCompileExp_t)(
+    ur_program_handle_t,
+    uint32_t,
+    ur_device_handle_t *,
+    const char *);
+
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function-pointer for urProgramLinkExp
+typedef ur_result_t(UR_APICALL *ur_pfnProgramLinkExp_t)(
+    ur_context_handle_t,
+    uint32_t,
+    ur_device_handle_t *,
+    uint32_t,
+    const ur_program_handle_t *,
+    const char *,
+    ur_program_handle_t *);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Table of ProgramExp functions pointers
 typedef struct ur_program_exp_dditable_t {
     ur_pfnProgramBuildExp_t pfnBuildExp;
+    ur_pfnProgramCompileExp_t pfnCompileExp;
+    ur_pfnProgramLinkExp_t pfnLinkExp;
 } ur_program_exp_dditable_t;
 
 ///////////////////////////////////////////////////////////////////////////////