Skip to content

Commit ba4e49d

Browse files
pbalcerpbalcer
authored
Merge pull request #1770 from igchor/command_list_cache_v2
[L0] implement command list cache for queue
2 parents f040b1f + 81620ab commit ba4e49d

File tree

8 files changed

+438
-3
lines changed

8 files changed

+438
-3
lines changed

‎source/adapters/level_zero/common.cpp

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,6 @@ ur_result_t ze2urResult(ze_result_t ZeResult) {
6666
}
6767
}
6868

69-
usm::DisjointPoolAllConfigs DisjointPoolConfigInstance =
70-
InitializeDisjointPoolConfig();
71-
7269
// This function will ensure compatibility with both Linux and Windows for
7370
// setting environment variables.
7471
bool setEnvVar(const char *name, const char *value) {

‎source/adapters/level_zero/common.hpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,14 @@ ur_result_t ze2urResult(ze_result_t ZeResult);
325325
return ze2urResult(Result); \
326326
}
327327

328+
// Trace a call to Level-Zero RT, throw on error
329+
#define ZE2UR_CALL_THROWS(ZeName, ZeArgs) \
330+
{ \
331+
ze_result_t ZeResult = ZeName ZeArgs; \
332+
if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) \
333+
throw ze2urResult(Result); \
334+
}
335+
328336
// Perform traced call to L0 without checking for errors
329337
#define ZE_CALL_NOCHECK(ZeName, ZeArgs) \
330338
ZeCall().doCall(ZeName ZeArgs, #ZeName, #ZeArgs, false)

‎source/adapters/level_zero/usm.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121

2222
#include <umf_helpers.hpp>
2323

24+
usm::DisjointPoolAllConfigs DisjointPoolConfigInstance =
25+
InitializeDisjointPoolConfig();
26+
2427
ur_result_t umf2urResult(umf_result_t umfResult) {
2528
if (umfResult == UMF_RESULT_SUCCESS)
2629
return UR_RESULT_SUCCESS;

‎source/adapters/level_zero/v2/command_list_cache.cpp

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
//===--------- command_list_cache.cpp - Level Zero Adapter ---------------===//
2+
//
3+
// Copyright (C) 2024 Intel Corporation
4+
//
5+
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
6+
// Exceptions. See LICENSE.TXT
7+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8+
//
9+
//===----------------------------------------------------------------------===//
10+
11+
#include "command_list_cache.hpp"
12+
13+
#include "context.hpp"
14+
#include "device.hpp"
15+
16+
bool v2::immediate_command_list_descriptor_t::operator==(
17+
const immediate_command_list_descriptor_t &rhs) const {
18+
return ZeDevice == rhs.ZeDevice && IsInOrder == rhs.IsInOrder &&
19+
Mode == rhs.Mode && Priority == rhs.Priority && Index == rhs.Index;
20+
}
21+
22+
bool v2::regular_command_list_descriptor_t::operator==(
23+
const regular_command_list_descriptor_t &rhs) const {
24+
return ZeDevice == rhs.ZeDevice && Ordinal == rhs.Ordinal &&
25+
IsInOrder == rhs.IsInOrder;
26+
}
27+
28+
namespace v2 {
29+
inline size_t command_list_descriptor_hash_t::operator()(
30+
const command_list_descriptor_t &desc) const {
31+
if (auto ImmCmdDesc =
32+
std::get_if<immediate_command_list_descriptor_t>(&desc)) {
33+
return combine_hashes(0, ImmCmdDesc->ZeDevice, ImmCmdDesc->Ordinal,
34+
ImmCmdDesc->IsInOrder, ImmCmdDesc->Mode,
35+
ImmCmdDesc->Priority, ImmCmdDesc->Index);
36+
} else {
37+
auto RegCmdDesc = std::get<regular_command_list_descriptor_t>(desc);
38+
return combine_hashes(0, RegCmdDesc.ZeDevice, RegCmdDesc.IsInOrder,
39+
RegCmdDesc.Ordinal);
40+
}
41+
}
42+
43+
command_list_cache_t::command_list_cache_t(ze_context_handle_t ZeContext)
44+
: ZeContext{ZeContext} {}
45+
46+
raii::ze_command_list_t
47+
command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
48+
if (auto ImmCmdDesc =
49+
std::get_if<immediate_command_list_descriptor_t>(&desc)) {
50+
ze_command_list_handle_t ZeCommandList;
51+
ZeStruct<ze_command_queue_desc_t> QueueDesc;
52+
QueueDesc.ordinal = ImmCmdDesc->Ordinal;
53+
QueueDesc.mode = ImmCmdDesc->Mode;
54+
QueueDesc.priority = ImmCmdDesc->Priority;
55+
QueueDesc.flags =
56+
ImmCmdDesc->IsInOrder ? ZE_COMMAND_QUEUE_FLAG_IN_ORDER : 0;
57+
if (ImmCmdDesc->Index.has_value()) {
58+
QueueDesc.flags |= ZE_COMMAND_QUEUE_FLAG_EXPLICIT_ONLY;
59+
QueueDesc.index = ImmCmdDesc->Index.value();
60+
}
61+
ZE2UR_CALL_THROWS(
62+
zeCommandListCreateImmediate,
63+
(ZeContext, ImmCmdDesc->ZeDevice, &QueueDesc, &ZeCommandList));
64+
return raii::ze_command_list_t(ZeCommandList, &zeCommandListDestroy);
65+
} else {
66+
auto RegCmdDesc = std::get<regular_command_list_descriptor_t>(desc);
67+
ZeStruct<ze_command_list_desc_t> CmdListDesc;
68+
CmdListDesc.flags =
69+
RegCmdDesc.IsInOrder ? ZE_COMMAND_LIST_FLAG_IN_ORDER : 0;
70+
CmdListDesc.commandQueueGroupOrdinal = RegCmdDesc.Ordinal;
71+
72+
ze_command_list_handle_t ZeCommandList;
73+
ZE2UR_CALL_THROWS(zeCommandListCreate, (ZeContext, RegCmdDesc.ZeDevice,
74+
&CmdListDesc, &ZeCommandList));
75+
return raii::ze_command_list_t(ZeCommandList, &zeCommandListDestroy);
76+
}
77+
}
78+
79+
raii::ze_command_list_t command_list_cache_t::getImmediateCommandList(
80+
ze_device_handle_t ZeDevice, bool IsInOrder, uint32_t Ordinal,
81+
ze_command_queue_mode_t Mode, ze_command_queue_priority_t Priority,
82+
std::optional<uint32_t> Index) {
83+
immediate_command_list_descriptor_t Desc;
84+
Desc.ZeDevice = ZeDevice;
85+
Desc.Ordinal = Ordinal;
86+
Desc.IsInOrder = IsInOrder;
87+
Desc.Mode = Mode;
88+
Desc.Priority = Priority;
89+
Desc.Index = Index;
90+
return getCommandList(Desc);
91+
}
92+
93+
raii::ze_command_list_t
94+
command_list_cache_t::getRegularCommandList(ze_device_handle_t ZeDevice,
95+
bool IsInOrder, uint32_t Ordinal) {
96+
regular_command_list_descriptor_t Desc;
97+
Desc.ZeDevice = ZeDevice;
98+
Desc.IsInOrder = IsInOrder;
99+
Desc.Ordinal = Ordinal;
100+
return getCommandList(Desc);
101+
}
102+
103+
void command_list_cache_t::addImmediateCommandList(
104+
raii::ze_command_list_t cmdList, ze_device_handle_t ZeDevice,
105+
bool IsInOrder, uint32_t Ordinal, ze_command_queue_mode_t Mode,
106+
ze_command_queue_priority_t Priority, std::optional<uint32_t> Index) {
107+
immediate_command_list_descriptor_t Desc;
108+
Desc.ZeDevice = ZeDevice;
109+
Desc.Ordinal = Ordinal;
110+
Desc.IsInOrder = IsInOrder;
111+
Desc.Mode = Mode;
112+
Desc.Priority = Priority;
113+
Desc.Index = Index;
114+
addCommandList(Desc, std::move(cmdList));
115+
}
116+
117+
void command_list_cache_t::addRegularCommandList(
118+
raii::ze_command_list_t cmdList, ze_device_handle_t ZeDevice,
119+
bool IsInOrder, uint32_t Ordinal) {
120+
regular_command_list_descriptor_t Desc;
121+
Desc.ZeDevice = ZeDevice;
122+
Desc.IsInOrder = IsInOrder;
123+
Desc.Ordinal = Ordinal;
124+
addCommandList(Desc, std::move(cmdList));
125+
}
126+
127+
raii::ze_command_list_t
128+
command_list_cache_t::getCommandList(const command_list_descriptor_t &desc) {
129+
std::unique_lock<ur_mutex> Lock(ZeCommandListCacheMutex);
130+
auto it = ZeCommandListCache.find(desc);
131+
if (it == ZeCommandListCache.end()) {
132+
Lock.unlock();
133+
return createCommandList(desc);
134+
}
135+
136+
assert(!it->second.empty());
137+
138+
raii::ze_command_list_t CommandListHandle = std::move(it->second.top());
139+
it->second.pop();
140+
141+
if (it->second.empty())
142+
ZeCommandListCache.erase(it);
143+
144+
return CommandListHandle;
145+
}
146+
147+
void command_list_cache_t::addCommandList(const command_list_descriptor_t &desc,
148+
raii::ze_command_list_t cmdList) {
149+
// TODO: add a limit?
150+
std::unique_lock<ur_mutex> Lock(ZeCommandListCacheMutex);
151+
auto [it, _] = ZeCommandListCache.try_emplace(desc);
152+
it->second.emplace(std::move(cmdList));
153+
}
154+
} // namespace v2

‎source/adapters/level_zero/v2/command_list_cache.hpp

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
//===--------- command_list_cache.hpp - Level Zero Adapter ---------------===//
2+
//
3+
// Copyright (C) 2024 Intel Corporation
4+
//
5+
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
6+
// Exceptions. See LICENSE.TXT
7+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8+
//
9+
//===----------------------------------------------------------------------===//
10+
#pragma once
11+
12+
#include <stack>
13+
14+
#include <ur/ur.hpp>
15+
#include <ur_api.h>
16+
#include <ze_api.h>
17+
18+
#include "common.hpp"
19+
20+
namespace v2 {
21+
namespace raii {
22+
using ze_command_list_t = std::unique_ptr<::_ze_command_list_handle_t,
23+
decltype(&zeCommandListDestroy)>;
24+
}
25+
26+
struct immediate_command_list_descriptor_t {
27+
ze_device_handle_t ZeDevice;
28+
bool IsInOrder;
29+
uint32_t Ordinal;
30+
ze_command_queue_mode_t Mode;
31+
ze_command_queue_priority_t Priority;
32+
std::optional<uint32_t> Index;
33+
bool operator==(const immediate_command_list_descriptor_t &rhs) const;
34+
};
35+
36+
struct regular_command_list_descriptor_t {
37+
ze_device_handle_t ZeDevice;
38+
bool IsInOrder;
39+
uint32_t Ordinal;
40+
bool operator==(const regular_command_list_descriptor_t &rhs) const;
41+
};
42+
43+
using command_list_descriptor_t =
44+
std::variant<immediate_command_list_descriptor_t,
45+
regular_command_list_descriptor_t>;
46+
47+
struct command_list_descriptor_hash_t {
48+
inline size_t operator()(const command_list_descriptor_t &desc) const;
49+
};
50+
51+
struct command_list_cache_t {
52+
command_list_cache_t(ze_context_handle_t ZeContext);
53+
54+
raii::ze_command_list_t
55+
getImmediateCommandList(ze_device_handle_t ZeDevice, bool IsInOrder,
56+
uint32_t Ordinal, ze_command_queue_mode_t Mode,
57+
ze_command_queue_priority_t Priority,
58+
std::optional<uint32_t> Index = std::nullopt);
59+
raii::ze_command_list_t getRegularCommandList(ze_device_handle_t ZeDevice,
60+
bool IsInOrder,
61+
uint32_t Ordinal);
62+
63+
void addImmediateCommandList(raii::ze_command_list_t cmdList,
64+
ze_device_handle_t ZeDevice, bool IsInOrder,
65+
uint32_t Ordinal, ze_command_queue_mode_t Mode,
66+
ze_command_queue_priority_t Priority,
67+
std::optional<uint32_t> Index = std::nullopt);
68+
void addRegularCommandList(raii::ze_command_list_t cmdList,
69+
ze_device_handle_t ZeDevice, bool IsInOrder,
70+
uint32_t Ordinal);
71+
72+
private:
73+
ze_context_handle_t ZeContext;
74+
std::unordered_map<command_list_descriptor_t,
75+
std::stack<raii::ze_command_list_t>,
76+
command_list_descriptor_hash_t>
77+
ZeCommandListCache;
78+
ur_mutex ZeCommandListCacheMutex;
79+
80+
raii::ze_command_list_t getCommandList(const command_list_descriptor_t &desc);
81+
void addCommandList(const command_list_descriptor_t &desc,
82+
raii::ze_command_list_t cmdList);
83+
raii::ze_command_list_t
84+
createCommandList(const command_list_descriptor_t &desc);
85+
};
86+
} // namespace v2

‎test/adapters/level_zero/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,5 @@ if(NOT WIN32)
6868

6969
target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE zeCallMap)
7070
endif()
71+
72+
add_subdirectory(v2)

‎test/adapters/level_zero/v2/CMakeLists.txt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
3+
# See LICENSE.TXT
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
6+
function(add_unittest name)
7+
set(target test-adapter-${name})
8+
add_adapter_test(${name}
9+
FIXTURE DEVICES
10+
ENVIRONMENT
11+
"UR_ADAPTERS_FORCE_LOAD=\"$<TARGET_FILE:ur_adapter_level_zero>\""
12+
SOURCES
13+
${PROJECT_SOURCE_DIR}/source/adapters/level_zero/common.cpp
14+
${PROJECT_SOURCE_DIR}/source/adapters/level_zero/ur_level_zero.cpp
15+
${ARGN})
16+
17+
target_include_directories(${target} PUBLIC
18+
${PROJECT_SOURCE_DIR}/source
19+
${PROJECT_SOURCE_DIR}/source/adapters/level_zero
20+
${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2
21+
LevelZeroLoader-Headers)
22+
23+
target_link_libraries(${target} PRIVATE
24+
${PROJECT_NAME}::common
25+
LevelZeroLoader
26+
LevelZeroLoader-Headers
27+
)
28+
endfunction()
29+
30+
add_unittest(level_zero_command_list_cache
31+
command_list_cache_test.cpp
32+
${PROJECT_SOURCE_DIR}/source/adapters/level_zero/v2/command_list_cache.cpp)

0 commit comments

Comments
 (0)