Skip to content

Commit 9d49c56

Browse files
Jaime ArteagaCompute-Runtime-Automation
Jaime Arteaga
authored and
Compute-Runtime-Automation
committed
Include domain when ordering with ZE_ENABLE_PCI_ID_DEVICE_ORDER
Signed-off-by: Jaime Arteaga <[email protected]>
1 parent 279181a commit 9d49c56

File tree

7 files changed

+92
-24
lines changed

7 files changed

+92
-24
lines changed

‎level_zero/core/test/unit_tests/sources/driver/linux/test_driver_handle_imp_linux.cpp

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ class DriverLinuxFixture : public ::testing::Test {
5353
static constexpr uint32_t numRootDevices = 5u;
5454
static constexpr uint32_t numSubDevices = 2u;
5555
std::vector<std::unique_ptr<NEO::Device>> devices;
56-
std::string bdf[numRootDevices] = {"03:04.0", "08:02.0", "08:03.1", "10:03.0", "02:01.0"};
57-
std::string sortedBdf[numRootDevices] = {"02:01.0", "03:04.0", "08:02.0", "08:03.1", "10:03.0"};
56+
std::string bdf[numRootDevices] = {"0000:03:04.0", "0000:08:02.0", "0000:08:03.1", "0000:10:03.0", "0000:02:01.0"};
57+
std::string sortedBdf[numRootDevices] = {"0000:02:01.0", "0000:03:04.0", "0000:08:02.0", "0000:08:03.1", "0000:10:03.0"};
5858
std::unique_ptr<UltDeviceFactory> deviceFactory;
5959
};
6060

@@ -147,8 +147,8 @@ class DriverPciOrderWitSimilarBusLinuxFixture : public ::testing::Test {
147147
static constexpr uint32_t numRootDevices = 4u;
148148
static constexpr uint32_t numSubDevices = 2u;
149149
std::vector<std::unique_ptr<NEO::Device>> devices;
150-
std::string bdf[numRootDevices] = {"03:04.0", "03:05.0", "03:06.0", "03:01.0"};
151-
std::string sortedBdf[numRootDevices] = {"03:01.0", "03:04.0", "03:05.0", "03:06.0"};
150+
std::string bdf[numRootDevices] = {"0000:03:04.0", "0000:03:05.0", "0000:03:06.0", "0000:03:01.0"};
151+
std::string sortedBdf[numRootDevices] = {"0000:03:01.0", "0000:03:04.0", "0000:03:05.0", "0000:03:06.0"};
152152
std::unique_ptr<UltDeviceFactory> deviceFactory;
153153
};
154154

@@ -198,8 +198,8 @@ class DriverPciOrderWitDifferentDeviceLinuxFixture : public ::testing::Test {
198198
static constexpr uint32_t numRootDevices = 2u;
199199
static constexpr uint32_t numSubDevices = 2u;
200200
std::vector<std::unique_ptr<NEO::Device>> devices;
201-
std::string bdf[numRootDevices] = {"03:05.0", "03:04.0"};
202-
std::string sortedBdf[numRootDevices] = {"03:04.0", "03:05.0"};
201+
std::string bdf[numRootDevices] = {"0000:03:05.0", "0000:03:04.0"};
202+
std::string sortedBdf[numRootDevices] = {"0000:03:04.0", "0000:03:05.0"};
203203
std::unique_ptr<UltDeviceFactory> deviceFactory;
204204
};
205205

@@ -249,8 +249,8 @@ class DriverPciOrderWitSimilarBusAndDeviceLinuxFixture : public ::testing::Test
249249
static constexpr uint32_t numRootDevices = 2u;
250250
static constexpr uint32_t numSubDevices = 2u;
251251
std::vector<std::unique_ptr<NEO::Device>> devices;
252-
std::string bdf[numRootDevices] = {"03:04.1", "03:04.0"};
253-
std::string sortedBdf[numRootDevices] = {"03:04.0", "03:04.1"};
252+
std::string bdf[numRootDevices] = {"0000:03:04.1", "0000:03:04.0"};
253+
std::string sortedBdf[numRootDevices] = {"0000:03:04.0", "0000:03:04.1"};
254254
std::unique_ptr<UltDeviceFactory> deviceFactory;
255255
};
256256

@@ -271,5 +271,56 @@ TEST_F(DriverPciOrderWitSimilarBusAndDeviceLinuxFixture, GivenEnvironmentVariabl
271271
delete driverHandle;
272272
}
273273

274+
class DriverPciOrderWitSimilarBDFLinuxFixture : public ::testing::Test {
275+
public:
276+
void SetUp() override {
277+
DebugManagerStateRestore restorer;
278+
DebugManager.flags.ZE_ENABLE_PCI_ID_DEVICE_ORDER.set(1);
279+
280+
NEO::MockCompilerEnableGuard mock(true);
281+
auto executionEnvironment = new NEO::ExecutionEnvironment();
282+
executionEnvironment->prepareRootDeviceEnvironments(numRootDevices);
283+
NEO::HardwareInfo hwInfo = *NEO::defaultHwInfo.get();
284+
for (auto i = 0u; i < executionEnvironment->rootDeviceEnvironments.size(); i++) {
285+
executionEnvironment->rootDeviceEnvironments[i]->setHwInfo(&hwInfo);
286+
}
287+
deviceFactory = std::make_unique<UltDeviceFactory>(numRootDevices, numSubDevices, *executionEnvironment);
288+
for (auto i = 0u; i < executionEnvironment->rootDeviceEnvironments.size(); i++) {
289+
devices.push_back(std::unique_ptr<NEO::Device>(deviceFactory->rootDevices[i]));
290+
}
291+
for (auto i = 0u; i < devices.size(); i++) {
292+
devices[i]->getExecutionEnvironment()->rootDeviceEnvironments[i]->osInterface = std::make_unique<NEO::OSInterface>();
293+
auto osInterface = devices[i]->getExecutionEnvironment()->rootDeviceEnvironments[i]->osInterface.get();
294+
osInterface->setDriverModel(std::make_unique<TestDriverMockDrm>(bdf[i], const_cast<NEO::RootDeviceEnvironment &>(devices[i]->getRootDeviceEnvironment())));
295+
}
296+
executionEnvironment->sortNeoDevices();
297+
}
298+
void TearDown() override {}
299+
300+
static constexpr uint32_t numRootDevices = 2u;
301+
static constexpr uint32_t numSubDevices = 2u;
302+
std::vector<std::unique_ptr<NEO::Device>> devices;
303+
std::string bdf[numRootDevices] = {"0001:03:04.0", "0000:03:04.0"};
304+
std::string sortedBdf[numRootDevices] = {"0000:03:04.0", "0001:03:04.0"};
305+
std::unique_ptr<UltDeviceFactory> deviceFactory;
306+
};
307+
308+
TEST_F(DriverPciOrderWitSimilarBDFLinuxFixture, GivenEnvironmentVariableForDeviceOrderAccordingToPciSetWhenRetrievingNeoDevicesThenNeoDevicesAccordingToDomainOrderRetrieved) {
309+
NEO::MockCompilerEnableGuard mock(true);
310+
DriverHandleImp *driverHandle = new DriverHandleImp;
311+
312+
EXPECT_EQ(ZE_RESULT_SUCCESS, driverHandle->initialize(std::move(devices)));
313+
314+
for (uint32_t i = 0; i < numRootDevices; i++) {
315+
auto L0Device = driverHandle->devices[i];
316+
if (L0Device != nullptr) {
317+
auto pDrm = L0Device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[L0Device->getRootDeviceIndex()]->osInterface->getDriverModel()->as<Drm>();
318+
EXPECT_NE(pDrm, nullptr);
319+
EXPECT_TRUE(!pDrm->getPciPath().compare(sortedBdf[i]));
320+
}
321+
}
322+
delete driverHandle;
323+
}
324+
274325
} // namespace ult
275326
} // namespace L0

‎opencl/test/unit_test/linux/main_linux_dll.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -146,16 +146,16 @@ TEST_F(DrmSimpleTests, GivenSelectedExistingDeviceWhenOpenDirSuccedsThenHwDevice
146146
auto hwDeviceIds = OSInterface::discoverDevices(executionEnvironment);
147147
EXPECT_EQ(1u, hwDeviceIds.size());
148148
EXPECT_NE(nullptr, hwDeviceIds[0].get());
149-
EXPECT_STREQ("test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
149+
EXPECT_STREQ("0000:test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
150150

151151
entryIndex = 0;
152152
openCounter = 2;
153153
hwDeviceIds = OSInterface::discoverDevices(executionEnvironment);
154154
EXPECT_EQ(2u, hwDeviceIds.size());
155155
EXPECT_NE(nullptr, hwDeviceIds[0].get());
156-
EXPECT_STREQ("test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
156+
EXPECT_STREQ("0000:test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
157157
EXPECT_NE(nullptr, hwDeviceIds[1].get());
158-
EXPECT_STREQ("test2", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
158+
EXPECT_STREQ("0000:test2", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
159159
}
160160

161161
TEST_F(DrmSimpleTests, GivenSelectedExistingDeviceWhenOpenDirFailsThenRetryOpeningRenderDevices) {
@@ -170,14 +170,14 @@ TEST_F(DrmSimpleTests, GivenSelectedExistingDeviceWhenOpenDirFailsThenRetryOpeni
170170
EXPECT_STREQ("/dev/dri/renderD128", lastOpenedPath.c_str());
171171
EXPECT_EQ(1u, hwDeviceIds.size());
172172
EXPECT_NE(nullptr, hwDeviceIds[0].get());
173-
EXPECT_STREQ("00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
173+
EXPECT_STREQ("0000:00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
174174

175175
openCounter = 2;
176176
hwDeviceIds = OSInterface::discoverDevices(executionEnvironment);
177177
EXPECT_STREQ("/dev/dri/renderD129", lastOpenedPath.c_str());
178178
EXPECT_EQ(2u, hwDeviceIds.size());
179179
EXPECT_NE(nullptr, hwDeviceIds[0].get());
180-
EXPECT_STREQ("00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
180+
EXPECT_STREQ("0000:00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
181181
EXPECT_NE(nullptr, hwDeviceIds[1].get());
182182
EXPECT_STREQ("00:03.0", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
183183
}
@@ -352,7 +352,7 @@ TEST_F(DrmSimpleTests, GivenFailingOpenDirAndMultipleAvailableDevicesWhenCreateM
352352
EXPECT_STREQ("/dev/dri/renderD129", lastOpenedPath.c_str());
353353
EXPECT_EQ(requestedNumRootDevices, hwDeviceIds.size());
354354
EXPECT_NE(nullptr, hwDeviceIds[0].get());
355-
EXPECT_STREQ("00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
355+
EXPECT_STREQ("0000:00:02.0", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
356356
EXPECT_NE(nullptr, hwDeviceIds[1].get());
357357
EXPECT_STREQ("00:03.0", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
358358
}
@@ -370,9 +370,9 @@ TEST_F(DrmSimpleTests, GivenMultipleAvailableDevicesWhenCreateMultipleRootDevice
370370
EXPECT_STREQ("/dev/dri/by-path/pci-0000:test2-render", lastOpenedPath.c_str());
371371
EXPECT_EQ(requestedNumRootDevices, hwDeviceIds.size());
372372
EXPECT_NE(nullptr, hwDeviceIds[0].get());
373-
EXPECT_STREQ("test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
373+
EXPECT_STREQ("0000:test1", hwDeviceIds[0]->as<HwDeviceIdDrm>()->getPciPath());
374374
EXPECT_NE(nullptr, hwDeviceIds[1].get());
375-
EXPECT_STREQ("test2", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
375+
EXPECT_STREQ("0000:test2", hwDeviceIds[1]->as<HwDeviceIdDrm>()->getPciPath());
376376
}
377377

378378
TEST_F(DrmSimpleTests, GivenSelectedIncorectDeviceWhenGetDeviceFdThenFail) {

‎opencl/test/unit_test/linux/os_interface_linux_tests.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ TEST(OsInterfaceTest, whenOsInterfaceSetupsGmmInputArgsThenFileDescriptorIsSetWi
2626
rootDeviceEnvironment->osInterface.reset(osInterface);
2727

2828
auto drm = new DrmMock(fakeFd, *rootDeviceEnvironment);
29-
drm->setPciPath("01:23.4");
29+
drm->setPciPath("0000:01:23.4");
3030
EXPECT_EQ(0, drm->queryAdapterBDF());
3131

3232
osInterface->setDriverModel(std::unique_ptr<DriverModel>(drm));

‎opencl/test/unit_test/os_interface/linux/drm_tests.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ TEST(DrmTest, GivenValidPciPathWhenGettingAdapterBdfThenCorrectValuesAreReturned
4545
DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]};
4646

4747
{
48-
drm.setPciPath("ab:cd.e");
48+
drm.setPciPath("0000:ab:cd.e");
4949
EXPECT_EQ(0, drm.queryAdapterBDF());
5050
auto adapterBdf = drm.getAdapterBDF();
5151
EXPECT_EQ(0xabu, adapterBdf.Bus);
@@ -60,7 +60,7 @@ TEST(DrmTest, GivenValidPciPathWhenGettingAdapterBdfThenCorrectValuesAreReturned
6060
}
6161

6262
{
63-
drm.setPciPath("01:23.4");
63+
drm.setPciPath("0000:01:23.4");
6464
EXPECT_EQ(0, drm.queryAdapterBDF());
6565
auto adapterBdf = drm.getAdapterBDF();
6666
EXPECT_EQ(0x1u, adapterBdf.Bus);

‎shared/source/execution_environment/linux/execution_environment_linux.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,15 @@ bool comparePciIdBusNumber(std::unique_ptr<RootDeviceEnvironment> &rootDeviceEnv
1919
// BDF sample format is : 00:02.0
2020
rootDeviceEnvironment1.get()->osInterface->getDriverModel()->as<NEO::Drm>()->queryAdapterBDF();
2121
auto bdfDevice1 = rootDeviceEnvironment1.get()->osInterface->getDriverModel()->as<NEO::Drm>()->getAdapterBDF();
22+
auto domain1 = rootDeviceEnvironment1.get()->osInterface->getDriverModel()->as<NEO::Drm>()->getPciDomain();
2223

2324
rootDeviceEnvironment2.get()->osInterface->getDriverModel()->as<NEO::Drm>()->queryAdapterBDF();
2425
auto bdfDevice2 = rootDeviceEnvironment2.get()->osInterface->getDriverModel()->as<NEO::Drm>()->getAdapterBDF();
26+
auto domain2 = rootDeviceEnvironment2.get()->osInterface->getDriverModel()->as<NEO::Drm>()->getPciDomain();
27+
28+
if (domain1 != domain2) {
29+
return (domain1 < domain2);
30+
}
2531

2632
if (bdfDevice1.Bus != bdfDevice2.Bus) {
2733
return (bdfDevice1.Bus < bdfDevice2.Bus);

‎shared/source/os_interface/linux/drm_neo.cpp

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,7 @@ void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int
514514
std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironment &executionEnvironment) {
515515
std::vector<std::unique_ptr<HwDeviceId>> hwDeviceIds;
516516
executionEnvironment.osEnvironment = std::make_unique<OsEnvironment>();
517-
std::string devicePrefix = std::string(Os::pciDevicesDirectory) + "/pci-0000:";
517+
std::string devicePrefix = std::string(Os::pciDevicesDirectory) + "/pci-";
518518
const char *renderDeviceSuffix = "-render";
519519
size_t numRootDevices = 0u;
520520
if (DebugManager.flags.CreateMultipleRootDevices.get()) {
@@ -534,7 +534,7 @@ std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironme
534534

535535
auto pciPath = NEO::getPciPath(fileDescriptor);
536536

537-
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("00:02.0").c_str());
537+
appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str());
538538
if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
539539
break;
540540
}
@@ -743,13 +743,15 @@ Drm::~Drm() {
743743
}
744744

745745
int Drm::queryAdapterBDF() {
746-
constexpr int pciBusInfoTokensNum = 3;
747-
uint32_t bus, device, function;
746+
constexpr int pciBusInfoTokensNum = 4;
747+
uint32_t domain, bus, device, function;
748748

749-
if (std::sscanf(hwDeviceId->getPciPath(), "%02x:%02x.%01x", &bus, &device, &function) != pciBusInfoTokensNum) {
749+
if (std::sscanf(hwDeviceId->getPciPath(), "%04x:%02x:%02x.%01x",
750+
&domain, &bus, &device, &function) != pciBusInfoTokensNum) {
750751
adapterBDF.Data = std::numeric_limits<uint32_t>::max();
751752
return 1;
752753
}
754+
setPciDomain(domain);
753755
adapterBDF.Bus = bus;
754756
adapterBDF.Function = function;
755757
adapterBDF.Device = device;

‎shared/source/os_interface/linux/drm_neo.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,14 @@ class Drm : public DriverModel {
234234

235235
std::unique_lock<std::mutex> lockBindFenceMutex();
236236

237+
void setPciDomain(uint32_t domain) {
238+
pciDomain = domain;
239+
}
240+
241+
uint32_t getPciDomain() {
242+
return pciDomain;
243+
}
244+
237245
protected:
238246
Drm(std::unique_ptr<HwDeviceIdDrm> hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment);
239247

@@ -275,6 +283,7 @@ class Drm : public DriverModel {
275283

276284
drm_i915_gem_context_param_sseu sseu{};
277285
ADAPTER_BDF adapterBDF{};
286+
uint32_t pciDomain = 0;
278287

279288
TopologyMap topologyMap;
280289
struct IoctlStatisticsEntry {

0 commit comments

Comments
 (0)