Skip to content

Commit f01edd3

Browse files
nrspruitnrspruit
authored
[UR] [L0] Update UR to link the Loader as static (#17104)
Migrated from oneapi-src/unified-runtime#2667 --------- Signed-off-by: Neil R. Spruit <[email protected]>
1 parent f36f787 commit f01edd3

File tree

9 files changed

+94
-31
lines changed

9 files changed

+94
-31
lines changed

‎.github/workflows/ur-build-hw.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,12 +136,18 @@ jobs:
136136
run: cmake --install ${{github.workspace}}/build
137137

138138
- name: Test adapter specific
139-
run: ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "adapter-specific" -E "memcheck" --timeout 600
139+
env:
140+
ZE_ENABLE_LOADER_DEBUG_TRACE: 1
141+
ZE_DEBUG: 1
142+
run: ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "adapter-specific" -E "memcheck" --timeout 600 -VV
140143
# Don't run adapter specific tests when building multiple adapters
141144
if: ${{ matrix.adapter.other_name == '' }}
142145

143146
- name: Test adapters
144-
run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "conformance" --timeout 600
147+
env:
148+
ZE_ENABLE_LOADER_DEBUG_TRACE: 1
149+
ZE_DEBUG: 1
150+
run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "conformance" --timeout 600 -VV
145151

146152
- name: Get information about platform
147153
if: ${{ always() }}

‎unified-runtime/cmake/FetchLevelZero.cmake

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,13 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
3737
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++98-compat-extra-semi")
3838
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option")
3939
endif()
40+
set(BUILD_STATIC ON)
4041

4142
if (UR_LEVEL_ZERO_LOADER_REPO STREQUAL "")
4243
set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
4344
endif()
4445
if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
45-
set(UR_LEVEL_ZERO_LOADER_TAG v1.19.2)
46+
set(UR_LEVEL_ZERO_LOADER_TAG v1.21.1)
4647
endif()
4748

4849
# Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104

‎unified-runtime/source/adapters/level_zero/CMakeLists.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,14 @@ if(UR_BUILD_ADAPTER_L0)
5555
)
5656
install_ur_library(ur_adapter_level_zero)
5757

58+
target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_LEVEL_ZERO)
5859
if(UR_STATIC_ADAPTER_L0)
5960
target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO)
6061

6162
# 'utils' target from 'level-zero-loader' includes path which is prefixed
6263
# in the source directory, this breaks the installation of 'utils' target.
63-
set_target_properties(utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "")
64-
install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader utils
64+
set_target_properties(level_zero_utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "")
65+
install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader level_zero_utils
6566
EXPORT ${PROJECT_NAME}-targets
6667
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
6768
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}

‎unified-runtime/source/adapters/level_zero/adapter.cpp

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
300300
ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED;
301301
ZesResult = ZE_RESULT_ERROR_UNINITIALIZED;
302302

303+
#ifdef UR_STATIC_LEVEL_ZERO
304+
// Given static linking of the L0 Loader, we must delay the loader's
305+
// destruction of its context until after the UR Adapter is destroyed.
306+
zelSetDelayLoaderContextTeardown();
307+
#endif
308+
303309
if (UrL0Debug & UR_L0_DEBUG_BASIC) {
304310
logger.setLegacySink(std::make_unique<ur_legacy_sink>());
305311
};
@@ -335,10 +341,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
335341
// Dynamically load the new L0 apis separately.
336342
// This must be done to avoid attempting to use symbols that do
337343
// not exist in older loader runtimes.
344+
#ifndef UR_STATIC_LEVEL_ZERO
338345
#ifdef _WIN32
339-
HMODULE processHandle = GetModuleHandle(NULL);
346+
GlobalAdapter->processHandle = GetModuleHandle(NULL);
340347
#else
341-
HMODULE processHandle = nullptr;
348+
GlobalAdapter->processHandle = nullptr;
349+
#endif
342350
#endif
343351

344352
// initialize level zero only once.
@@ -412,9 +420,13 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
412420
}
413421

414422
if (useInitDrivers) {
423+
#ifdef UR_STATIC_LEVEL_ZERO
424+
GlobalAdapter->initDriversFunctionPtr = zeInitDrivers;
425+
#else
415426
GlobalAdapter->initDriversFunctionPtr =
416427
(ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr(
417-
processHandle, "zeInitDrivers");
428+
GlobalAdapter->processHandle, "zeInitDrivers");
429+
#endif
418430
if (GlobalAdapter->initDriversFunctionPtr) {
419431
logger::debug("\nzeInitDrivers with flags value of {}\n",
420432
static_cast<int>(GlobalAdapter->InitDriversDesc.flags));
@@ -455,14 +467,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
455467

456468
return;
457469
}
458-
// Dynamically load the new L0 SysMan separate init and new EXP apis
459-
// separately. This must be done to avoid attempting to use symbols that do
460-
// not exist in older loader runtimes.
461-
#ifdef _WIN32
462-
GlobalAdapter->processHandle = GetModuleHandle(NULL);
463-
#else
464-
GlobalAdapter->processHandle = nullptr;
465-
#endif
466470

467471
// Check if the user has enabled the default L0 SysMan initialization.
468472
const int UrSysmanZesinitEnable = [&UserForcedSysManInit] {
@@ -484,6 +488,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
484488
ZesInitNeeded = true;
485489
}
486490
if (ZesInitNeeded) {
491+
#ifdef UR_STATIC_LEVEL_ZERO
492+
GlobalAdapter->getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp;
493+
GlobalAdapter->getSysManDriversFunctionPtr = zesDriverGet;
494+
GlobalAdapter->sysManInitFunctionPtr = zesInit;
495+
#else
487496
GlobalAdapter->getDeviceByUUIdFunctionPtr =
488497
(zes_pfnDriverGetDeviceByUuidExp_t)
489498
ur_loader::LibLoader::getFunctionPtr(
@@ -494,6 +503,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
494503
GlobalAdapter->sysManInitFunctionPtr =
495504
(zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr(
496505
GlobalAdapter->processHandle, "zesInit");
506+
#endif
497507
}
498508
if (GlobalAdapter->getDeviceByUUIdFunctionPtr &&
499509
GlobalAdapter->getSysManDriversFunctionPtr &&
@@ -668,7 +678,13 @@ ur_result_t urAdapterRelease(ur_adapter_handle_t) {
668678
if (GlobalAdapter) {
669679
std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
670680
if (--GlobalAdapter->RefCount == 0) {
671-
return adapterStateTeardown();
681+
auto result = adapterStateTeardown();
682+
#ifdef UR_STATIC_LEVEL_ZERO
683+
// Given static linking of the L0 Loader, we must delay the loader's
684+
// destruction of its context until after the UR Adapter is destroyed.
685+
zelLoaderContextTeardown();
686+
#endif
687+
return result;
672688
}
673689
}
674690

‎unified-runtime/source/adapters/level_zero/platform.cpp

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -354,52 +354,67 @@ ur_result_t ur_platform_handle_t_::initialize() {
354354
// Check if mutable command list extension is supported and initialize
355355
// function pointers.
356356
if (MutableCommandListSpecExtensionSupported) {
357+
#ifdef UR_STATIC_LEVEL_ZERO
358+
ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp =
359+
zeCommandListGetNextCommandIdExp;
360+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp =
361+
zeCommandListUpdateMutableCommandsExp;
362+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp =
363+
zeCommandListUpdateMutableCommandSignalEventExp;
364+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp =
365+
zeCommandListUpdateMutableCommandWaitEventsExp;
366+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp =
367+
zeCommandListUpdateMutableCommandKernelsExp;
368+
ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp =
369+
zeCommandListGetNextCommandIdWithKernelsExp;
370+
#else
357371
ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp =
358372
(ze_pfnCommandListGetNextCommandIdExp_t)
359373
ur_loader::LibLoader::getFunctionPtr(
360374
GlobalAdapter->processHandle,
361375
"zeCommandListGetNextCommandIdExp");
362-
ZeMutableCmdListExt.Supported |=
363-
ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr;
364376
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp =
365377
(ze_pfnCommandListUpdateMutableCommandsExp_t)
366378
ur_loader::LibLoader::getFunctionPtr(
367379
GlobalAdapter->processHandle,
368380
"zeCommandListUpdateMutableCommandsExp");
369-
ZeMutableCmdListExt.Supported |=
370-
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr;
371381
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp =
372382
(ze_pfnCommandListUpdateMutableCommandSignalEventExp_t)
373383
ur_loader::LibLoader::getFunctionPtr(
374384
GlobalAdapter->processHandle,
375385
"zeCommandListUpdateMutableCommandSignalEventExp");
376-
ZeMutableCmdListExt.Supported |=
377-
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp !=
378-
nullptr;
379386
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp =
380387
(ze_pfnCommandListUpdateMutableCommandWaitEventsExp_t)
381388
ur_loader::LibLoader::getFunctionPtr(
382389
GlobalAdapter->processHandle,
383390
"zeCommandListUpdateMutableCommandWaitEventsExp");
384-
ZeMutableCmdListExt.Supported |=
385-
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp !=
386-
nullptr;
387391
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp =
388392
(ze_pfnCommandListUpdateMutableCommandKernelsExp_t)
389393
ur_loader::LibLoader::getFunctionPtr(
390394
GlobalAdapter->processHandle,
391395
"zeCommandListUpdateMutableCommandKernelsExp");
392-
ZeMutableCmdListExt.Supported |=
393-
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp !=
394-
nullptr;
395396
ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp =
396397
(ze_pfnCommandListGetNextCommandIdWithKernelsExp_t)
397398
ur_loader::LibLoader::getFunctionPtr(
398399
GlobalAdapter->processHandle,
399400
"zeCommandListGetNextCommandIdWithKernelsExp");
401+
#endif
402+
ZeMutableCmdListExt.Supported |=
403+
ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr;
400404
ZeMutableCmdListExt.Supported |=
401405
ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp !=
402406
nullptr;
407+
ZeMutableCmdListExt.Supported |=
408+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp !=
409+
nullptr;
410+
ZeMutableCmdListExt.Supported |=
411+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp !=
412+
nullptr;
413+
ZeMutableCmdListExt.Supported |=
414+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp !=
415+
nullptr;
416+
ZeMutableCmdListExt.Supported |=
417+
ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr;
403418
ZeMutableCmdListExt.LoaderExtension = true;
404419
} else {
405420
ZeMutableCmdListExt.Supported |=
@@ -455,12 +470,18 @@ ur_result_t ur_platform_handle_t_::initialize() {
455470
// Check if ImmediateAppendCommandList is supported and initialize the
456471
// function pointer.
457472
if (ZeImmediateCommandListAppendExtensionFound) {
473+
#ifdef UR_STATIC_LEVEL_ZERO
474+
ZeCommandListImmediateAppendExt
475+
.zeCommandListImmediateAppendCommandListsExp =
476+
zeCommandListImmediateAppendCommandListsExp;
477+
#else
458478
ZeCommandListImmediateAppendExt
459479
.zeCommandListImmediateAppendCommandListsExp =
460480
(ze_pfnCommandListImmediateAppendCommandListsExp_t)
461481
ur_loader::LibLoader::getFunctionPtr(
462482
GlobalAdapter->processHandle,
463483
"zeCommandListImmediateAppendCommandListsExp");
484+
#endif
464485
ZeCommandListImmediateAppendExt.Supported =
465486
ZeCommandListImmediateAppendExt
466487
.zeCommandListImmediateAppendCommandListsExp != nullptr;

‎unified-runtime/test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) {
3434
moduleDesc.pBuildFlags = "";
3535
ze_module_handle_t module;
3636

37+
// Initialize Level Zero driver is required if this test is linked statically
38+
// with Level Zero loader, the driver will not be init otherwise.
39+
zeInit(ZE_INIT_FLAG_GPU_ONLY);
40+
3741
ASSERT_EQ(
3842
zeModuleCreate(native_context, native_device, &moduleDesc, &module, NULL),
3943
ZE_RESULT_SUCCESS);

‎unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,14 @@
1919
#include <unordered_set>
2020
#include <vector>
2121

22-
struct CommandListCacheTest : public uur::urContextTest {};
22+
struct CommandListCacheTest : public uur::urContextTest {
23+
void SetUp() override {
24+
// Initialize Level Zero driver is required if this test is linked
25+
// statically with Level Zero loader, the driver will not be init otherwise.
26+
zeInit(ZE_INIT_FLAG_GPU_ONLY);
27+
urContextTest::SetUp();
28+
}
29+
};
2330

2431
UUR_INSTANTIATE_DEVICE_TEST_SUITE(CommandListCacheTest);
2532

‎unified-runtime/test/adapters/level_zero/v2/deferred_kernel.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313

1414
struct urEnqueueKernelLaunchTest : uur::urKernelExecutionTest {
1515
void SetUp() override {
16+
// Initialize Level Zero driver is required if this test is linked
17+
// statically with Level Zero loader, the driver will not be init otherwise.
18+
zeInit(ZE_INIT_FLAG_GPU_ONLY);
1619
program_name = "fill";
1720
UUR_RETURN_ON_FATAL_FAILURE(urKernelExecutionTest::SetUp());
1821
}

‎unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ struct EventPoolTest : public uur::urQueueTestWithParam<ProviderParams> {
109109

110110
auto params = getParam();
111111

112+
// Initialize Level Zero driver is required if this test is linked
113+
// statically with Level Zero loader, the driver will not be init otherwise.
114+
zeInit(ZE_INIT_FLAG_GPU_ONLY);
115+
112116
mockVec.push_back(device);
113117

114118
cache = std::unique_ptr<event_pool_cache>(new event_pool_cache(

0 commit comments

Comments
 (0)