Skip to content

Commit 6b2fb66

Browse files
aelovikov-intelaelovikov-intel
authored
[SYCL] Fix Level-Zero's sycl::make_device interop (#13483)
The device returned by that API for Level-Zero devops must be a copy of a device in a fixed device hierarchy, meaning it must be equally comparable and result in the same hash value.
1 parent 646db9c commit 6b2fb66

File tree

5 files changed

+296
-25
lines changed

5 files changed

+296
-25
lines changed

‎sycl/include/sycl/backend.hpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,16 @@ std::enable_if_t<detail::InteropFeatureSupportMap<Backend>::MakeDevice == true,
312312
device>
313313
make_device(const typename backend_traits<Backend>::template input_type<device>
314314
&BackendObject) {
315+
for (auto p : platform::get_platforms()) {
316+
if (p.get_backend() != Backend)
317+
continue;
318+
319+
for (auto d : p.get_devices()) {
320+
if (get_native<Backend>(d) == BackendObject)
321+
return d;
322+
}
323+
}
324+
315325
return detail::make_device(detail::pi::cast<pi_native_handle>(BackendObject),
316326
Backend);
317327
}

‎sycl/include/sycl/ext/oneapi/backend/level_zero.hpp

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,67 @@ inline context make_context<backend::ext_oneapi_level_zero>(
125125
BackendObject.Ownership == ext::oneapi::level_zero::ownership::keep);
126126
}
127127

128+
namespace detail {
129+
inline std::optional<sycl::device> find_matching_descendent_device(
130+
sycl::device d,
131+
const backend_input_t<backend::ext_oneapi_level_zero, device>
132+
&BackendObject) {
133+
if (get_native<backend::ext_oneapi_level_zero>(d) == BackendObject)
134+
return d;
135+
std::vector<info::partition_property> partition_props =
136+
d.get_info<info::device::partition_properties>();
137+
138+
for (auto pp : partition_props) {
139+
if (pp == info::partition_property::partition_by_affinity_domain) {
140+
auto sub_devices = d.create_sub_devices<
141+
info::partition_property::partition_by_affinity_domain>(
142+
info::partition_affinity_domain::next_partitionable);
143+
for (auto sub_dev : sub_devices) {
144+
if (auto maybe_device =
145+
find_matching_descendent_device(sub_dev, BackendObject))
146+
return maybe_device;
147+
}
148+
}
149+
150+
assert(false && "Unexpected partitioning scheme for a Level-Zero device!");
151+
}
152+
153+
return {};
154+
}
155+
} // namespace detail
156+
157+
// Specialization of sycl::make_device for Level-Zero backend.
158+
// Level-Zero backend specification says:
159+
//
160+
// > The SYCL execution environment for the Level Zero backend contains a fixed
161+
// > number of devices that are enumerated via sycl::device::get_devices() and
162+
// > a fixed number of sub-devices that are enumerated via
163+
// > sycl::device::create_sub_devices(...). Calling this function does not
164+
// > create a new device. Rather it merely creates a sycl::device object that
165+
// > is a copy of one of the devices from those enumerations.
166+
//
167+
// Per SYCL 2020 specification, device and it's copy should be equally
168+
// comparable and its hashes must be equal. As such, we cannot simply create a
169+
// new `detail::device_impl` and then a `sycl::device` out of it and have to
170+
// iterate over the existing device hierarchy and make a copy.
171+
template <>
172+
inline device make_device<backend::ext_oneapi_level_zero>(
173+
const backend_input_t<backend::ext_oneapi_level_zero, device>
174+
&BackendObject) {
175+
for (auto p : platform::get_platforms()) {
176+
if (p.get_backend() != backend::ext_oneapi_level_zero)
177+
continue;
178+
179+
for (auto d : p.get_devices()) {
180+
if (auto maybe_device = find_matching_descendent_device(d, BackendObject))
181+
return *maybe_device;
182+
}
183+
}
184+
185+
throw sycl::exception(make_error_code(errc::invalid),
186+
"Native device isn't exposed to SYCL.");
187+
}
188+
128189
// Specialization of sycl::make_queue for Level-Zero backend.
129190
template <>
130191
inline queue make_queue<backend::ext_oneapi_level_zero>(

‎sycl/test-e2e/Plugin/interop-device.cpp

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// RUN: %{build} -o %t.out
2+
// RUN: %{run} %t.out
3+
4+
// SYCL 2020:
5+
// > The execution environment for a SYCL application has a fixed number of root
6+
// > devices which does not vary as the application executes.
7+
//
8+
// Verify that round-robin conversion of a root device (SYCL->native->SYCL)
9+
// doesn't create a "new" SYCL device that isn't equally comparable to one of
10+
// the root devices in the pre-existin fixed hierarchy.
11+
12+
#include <sycl/sycl.hpp>
13+
14+
int main() {
15+
auto root_devices = sycl::device::get_devices();
16+
17+
for (auto d : root_devices) {
18+
// TODO: No sycl::device interop support for
19+
// sycl::backend::ext_oneapi_native_cpu, sycl::backend::ext_oneapi_cuda,
20+
// sycl::backend::ext_oneapi_hip.
21+
constexpr sycl::backend backends[] = {sycl::backend::opencl,
22+
sycl::backend::ext_oneapi_level_zero};
23+
sycl::detail::loop<std::size(backends)>([&](auto be_idx) {
24+
constexpr auto be = backends[be_idx];
25+
if (d.get_backend() != be)
26+
return;
27+
28+
auto native = sycl::get_native<be>(d);
29+
auto from_native = sycl::make_device<be>(native);
30+
assert(d == from_native);
31+
std::hash<sycl::device> hash;
32+
assert(hash(d) == hash(from_native));
33+
});
34+
}
35+
36+
return 0;
37+
}

‎sycl/test-e2e/Plugin/interop-level-zero-device.cpp

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// REQUIRES: level_zero
2+
// RUN: %{build} -o %t.out
3+
// RUN: %{run} %t.out
4+
5+
#include <sycl/sycl.hpp>
6+
7+
template <typename FuncTy>
8+
void for_each_descendent_dev(sycl::device dev, FuncTy F) {
9+
F(dev);
10+
11+
std::vector<sycl::info::partition_property> partition_props =
12+
dev.get_info<sycl::info::device::partition_properties>();
13+
14+
auto supports = [&](auto prop) {
15+
return std::find(partition_props.begin(), partition_props.end(), prop) !=
16+
partition_props.end();
17+
};
18+
19+
if (supports(sycl::info::partition_property::partition_by_affinity_domain)) {
20+
std::cout << "Affinity" << std::endl;
21+
auto sub_devices = dev.create_sub_devices<
22+
sycl::info::partition_property::partition_by_affinity_domain>(
23+
sycl::info::partition_affinity_domain::next_partitionable);
24+
for (auto d : sub_devices)
25+
for_each_descendent_dev(d, F);
26+
}
27+
28+
// I'm not sure if remaining partitioning schems are actually supported by any
29+
// of the existing Level-Zero devices. Make sure we still cover that
30+
// possibility in this test to accomodate any future situation.
31+
32+
if (supports(sycl::info::partition_property::partition_equally)) {
33+
std::cout << "Equally" << std::endl;
34+
auto max_compute_units =
35+
dev.get_info<sycl::info::device::max_compute_units>();
36+
for (int count = 1; count < max_compute_units; ++count) {
37+
auto sub_devices = dev.create_sub_devices<
38+
sycl::info::partition_property::partition_equally>(count);
39+
for (auto d : sub_devices)
40+
for_each_descendent_dev(d, F);
41+
}
42+
}
43+
44+
if (supports(sycl::info::partition_property::partition_by_counts)) {
45+
std::cout << "By counts" << std::endl;
46+
auto max_compute_units =
47+
dev.get_info<sycl::info::device::max_compute_units>();
48+
49+
// Iterating over all possible sub-devices with this partitioning scheme
50+
// explodes combinatorially, yet Level-Zero backend specificaiton states
51+
// that device produced by `make_device` has to be a copy of a device from
52+
// the existing fixed number of (sub-)devices enumerated by
53+
// get_devices/create_sub_devices. As such, it wouldn't be practical to face
54+
// Level-Zero devices going through code path unless the specification is
55+
// changed.
56+
assert(max_compute_units <= 8 && "Don't expect L0 devices like that.");
57+
58+
auto fill_counts_and_invoke = [&](auto self, std::vector<size_t> counts) {
59+
size_t used = std::accumulate(counts.begin(), counts.end(), 0);
60+
61+
if (used == max_compute_units) {
62+
std::cout << "counts:";
63+
for (auto c : counts)
64+
std::cout << " " << c;
65+
std::cout << ", total: " << used << std::endl;
66+
67+
auto sub_devices = dev.create_sub_devices<
68+
sycl::info::partition_property::partition_by_counts>(counts);
69+
for (auto d : sub_devices)
70+
for_each_descendent_dev(d, F);
71+
return;
72+
}
73+
for (size_t i = 1; i <= max_compute_units - used; ++i) {
74+
std::vector<size_t> new_counts{counts};
75+
new_counts.push_back(i);
76+
self(self, new_counts);
77+
}
78+
};
79+
fill_counts_and_invoke(fill_counts_and_invoke, {});
80+
}
81+
}
82+
83+
int main() {
84+
auto root_devices = sycl::device::get_devices();
85+
86+
for (auto d : root_devices)
87+
for_each_descendent_dev(d, [](sycl::device d) {
88+
int level = 0;
89+
sycl::device tmp = d;
90+
while (tmp.get_info<sycl::info::device::partition_type_property>() !=
91+
sycl::info::partition_property::no_partition) {
92+
++level;
93+
tmp = tmp.template get_info<sycl::info::device::parent_device>();
94+
}
95+
std::cout << "Device at level " << level << std::endl;
96+
97+
constexpr auto be = sycl::backend::ext_oneapi_level_zero;
98+
99+
auto native = sycl::get_native<be>(d);
100+
auto from_native = sycl::make_device<be>(native);
101+
assert(d == from_native);
102+
std::hash<sycl::device> hash;
103+
assert(hash(d) == hash(from_native));
104+
});
105+
106+
return 0;
107+
}

‎sycl/test-e2e/Regression/cache_test.cpp

Lines changed: 81 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -91,35 +91,91 @@ int queryFromNativeHandle(std::vector<sycl::platform> *platform_list,
9191
zeDeviceGet(l0_drivers[0], &l0_device_count, l0_devices.data());
9292

9393
// Create the platform and device objects using the native handle.
94-
auto plt = level_zero::make<sycl::platform>(l0_drivers[0]);
95-
auto dev = level_zero::make<sycl::device>(plt, l0_devices[0]);
96-
97-
// Check to see if this platform is in the platform list.
98-
std::cout << "Platform created with native handle: "
99-
<< plt.get_info<sycl::info::platform::name>() << std::endl;
100-
auto plt_result = std::find_if(platform_list->begin(), platform_list->end(),
101-
[&](sycl::platform &p) { return p == plt; });
102-
if (plt_result != platform_list->end()) {
103-
std::cout << "The platform list contains: "
94+
{
95+
// Using deprecated level_zero-specific interface. Intentionally copy-pasted
96+
// and not outlined into a helper because the deprecated interface will be
97+
// removed in a few months.
98+
auto plt = level_zero::make<sycl::platform>(l0_drivers[0]);
99+
auto dev = level_zero::make<sycl::device>(plt, l0_devices[0]);
100+
101+
// Check to see if this platform is in the platform list.
102+
std::cout << "Platform created with native handle: "
104103
<< plt.get_info<sycl::info::platform::name>() << std::endl;
105-
} else {
106-
std::cout << plt.get_info<sycl::info::platform::name>()
107-
<< " was not in the platform list.\n";
108-
failures++;
104+
auto plt_result = std::find_if(platform_list->begin(), platform_list->end(),
105+
[&](sycl::platform &p) { return p == plt; });
106+
if (plt_result != platform_list->end()) {
107+
std::cout << "The platform list contains: "
108+
<< plt.get_info<sycl::info::platform::name>() << std::endl;
109+
} else {
110+
std::cout << plt.get_info<sycl::info::platform::name>()
111+
<< " was not in the platform list.\n";
112+
failures++;
113+
}
114+
115+
// Check to see if this device is in the device list.
116+
std::cout << "Device created with native handle: "
117+
<< dev.get_info<sycl::info::device::name>() << std::endl;
118+
auto dev_result = std::find_if(device_list->begin(), device_list->end(),
119+
[&](sycl::device &d) { return d == dev; });
120+
if (dev_result != device_list->end()) {
121+
std::cout << "The device list contains: "
122+
<< dev.get_info<sycl::info::device::name>() << std::endl;
123+
} else {
124+
std::cout << dev.get_info<sycl::info::device::name>()
125+
<< " was not in the device list.\n";
126+
failures++;
127+
}
109128
}
129+
{
130+
// Using SYCL2020 interface.
131+
auto plt = sycl::make_platform<sycl::backend::ext_oneapi_level_zero>(
132+
l0_drivers[0]);
133+
auto dev =
134+
sycl::make_device<sycl::backend::ext_oneapi_level_zero>(l0_devices[0]);
135+
136+
// Check to see if this platform is in the platform list.
137+
std::cout << "Platform created with native handle: "
138+
<< plt.get_info<sycl::info::platform::name>() << std::endl;
139+
auto plt_result = std::find_if(platform_list->begin(), platform_list->end(),
140+
[&](sycl::platform &p) { return p == plt; });
141+
if (plt_result != platform_list->end()) {
142+
std::cout << "The platform list contains: "
143+
<< plt.get_info<sycl::info::platform::name>() << std::endl;
144+
} else {
145+
std::cout << plt.get_info<sycl::info::platform::name>()
146+
<< " was not in the platform list.\n";
147+
failures++;
148+
}
110149

111-
// Check to see if this device is in the device list.
112-
std::cout << "Device created with native handle: "
113-
<< dev.get_info<sycl::info::device::name>() << std::endl;
114-
auto dev_result = std::find_if(device_list->begin(), device_list->end(),
115-
[&](sycl::device &d) { return d == dev; });
116-
if (dev_result != device_list->end()) {
117-
std::cout << "The device list contains: "
150+
// Check to see if this device is in the device list.
151+
std::cout << "Device created with native handle: "
118152
<< dev.get_info<sycl::info::device::name>() << std::endl;
119-
} else {
120-
std::cout << dev.get_info<sycl::info::device::name>()
121-
<< " was not in the device list.\n";
122-
failures++;
153+
auto dev_result = std::find_if(device_list->begin(), device_list->end(),
154+
[&](sycl::device &d) { return d == dev; });
155+
if (dev_result != device_list->end()) {
156+
// Level-Zero backend specification for sycl::make_device:
157+
//
158+
// > Constructs a SYCL device instance from a Level-Zero
159+
// > ze_device_handle_t. The SYCL execution environment for the Level
160+
// > Zero backend contains a fixed number of devices that are enumerated
161+
// > via sycl::device::get_devices() and a fixed number of sub-devices
162+
// > that are enumerated via sycl::device::create_sub_devices(...).
163+
// > Calling this function does not create a new device. Rather it
164+
// > merely creates a sycl::device object that is a copy of one of the
165+
// > devices from those enumerations.
166+
//
167+
// SYCL 2020's common reference semantics says that such a copy must
168+
// result in the same hash value.
169+
auto hash = std::hash<sycl::device>{};
170+
assert(hash(*dev_result) == hash(dev));
171+
172+
std::cout << "The device list contains: "
173+
<< dev.get_info<sycl::info::device::name>() << std::endl;
174+
} else {
175+
std::cout << dev.get_info<sycl::info::device::name>()
176+
<< " was not in the device list.\n";
177+
failures++;
178+
}
123179
}
124180
return failures;
125181
}

0 commit comments

Comments
 (0)