Skip to content

Commit c4e802f

Browse files
jchodorCompute-Runtime-Automation
jchodor
authored and
Compute-Runtime-Automation
committed
WSL - fixing resource cleanup on process shutdown
Resolves issues with coexistance of NEO L0 and NEO OCL libraries within a single process running in WSL and using WDDM GPU PV Signed-off-by: Jaroslaw Chodor <[email protected]>
1 parent c77fe0b commit c4e802f

File tree

13 files changed

+120
-26
lines changed

13 files changed

+120
-26
lines changed

‎opencl/test/unit_test/os_interface/linux/drm_tests.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,3 +875,11 @@ TEST(DrmQueryTest, GivenRpsMaxFreqFileDoesntExistWhenFrequencyIsQueriedThenFallb
875875

876876
EXPECT_EQ(expectedMaxFrequency, maxFrequency);
877877
}
878+
879+
TEST(DrmTest, whenCheckedIfResourcesCleanupCanBeSkippedThenReturnsFalse) {
880+
auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
881+
executionEnvironment->prepareRootDeviceEnvironments(1);
882+
DrmMock *pDrm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]);
883+
EXPECT_FALSE(pDrm->skipResourceCleanup());
884+
delete pDrm;
885+
}

‎shared/source/command_stream/command_stream_receiver.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,15 @@ bool CommandStreamReceiver::isRcs() const {
211211
return this->osContext->getEngineType() == aub_stream::ENGINE_RCS;
212212
}
213213

214+
bool CommandStreamReceiver::skipResourceCleanup() const {
215+
return this->getOSInterface() && this->getOSInterface()->getDriverModel() && this->getOSInterface()->getDriverModel()->skipResourceCleanup();
216+
}
217+
214218
void CommandStreamReceiver::cleanupResources() {
219+
if (this->skipResourceCleanup()) {
220+
return;
221+
}
222+
215223
waitForTaskCountAndCleanAllocationList(this->latestFlushedTaskCount, TEMPORARY_ALLOCATION);
216224
waitForTaskCountAndCleanAllocationList(this->latestFlushedTaskCount, REUSABLE_ALLOCATION);
217225

‎shared/source/command_stream/command_stream_receiver.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,8 @@ class CommandStreamReceiver {
288288
return activePartitions;
289289
}
290290

291+
bool skipResourceCleanup() const;
292+
291293
std::unique_ptr<GmmPageTableMngr> pageTableManager;
292294

293295
protected:

‎shared/source/device/device.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,15 @@ Device::Device(ExecutionEnvironment *executionEnvironment)
3636
}
3737

3838
Device::~Device() {
39-
getMemoryManager()->freeGraphicsMemory(rtMemoryBackedBuffer);
40-
rtMemoryBackedBuffer = nullptr;
39+
if (false == commandStreamReceivers.empty()) {
40+
if (commandStreamReceivers[0]->skipResourceCleanup()) {
41+
return;
42+
}
43+
}
4144

4245
DEBUG_BREAK_IF(nullptr == executionEnvironment->memoryManager.get());
46+
getMemoryManager()->freeGraphicsMemory(rtMemoryBackedBuffer);
47+
rtMemoryBackedBuffer = nullptr;
4348

4449
if (performanceCounters) {
4550
performanceCounters->shutdown();

‎shared/source/os_interface/os_interface.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ class DriverModel : public NonCopyableClass {
8383
return std::numeric_limits<size_t>::max();
8484
}
8585

86+
virtual bool skipResourceCleanup() const {
87+
return false;
88+
}
89+
8690
protected:
8791
DriverModelType driverModelType;
8892
};

‎shared/source/os_interface/windows/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ set(NEO_CORE_OS_INTERFACE_WDDM
8787
${CMAKE_CURRENT_SOURCE_DIR}/wddm/set_gmm_input_args_${DRIVER_MODEL}.cpp
8888
${CMAKE_CURRENT_SOURCE_DIR}/wddm/max_mem_alloc_size_${DRIVER_MODEL}.cpp
8989
${CMAKE_CURRENT_SOURCE_DIR}/wddm/helper_${DRIVER_MODEL}.cpp
90+
${CMAKE_CURRENT_SOURCE_DIR}/wddm/skip_resource_cleanup_${DRIVER_MODEL}.cpp
9091
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp
9192
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h
9293
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h

‎shared/source/os_interface/windows/os_context_win.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ void OsContextWin::initializeContext() {
3939
};
4040

4141
OsContextWin::~OsContextWin() {
42-
if (contextInitialized) {
42+
if (contextInitialized && (false == this->wddm.skipResourceCleanup())) {
4343
wddm.getWddmInterface()->destroyHwQueue(hardwareQueue.handle);
4444
wddm.getWddmInterface()->destroyMonitorFence(residencyController.getMonitoredFence());
4545
wddm.destroyContext(wddmContextHandle);

‎shared/source/os_interface/windows/wddm/configure_device_address_space_drm_or_wddm.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ bool ensureGpuAddressRangeIsReserved(uint64_t address, size_t size, D3DKMT_HANDL
159159
rangeDesc.MaximumAddress = alignUp(address + size, MemoryConstants::pageSize64k);
160160
rangeDesc.Size = MemoryConstants::pageSize64k;
161161
status = gdi.reserveGpuVirtualAddress(&rangeDesc);
162-
if (status != STATUS_SUCCESS) {
162+
if (status == STATUS_SUCCESS) {
163163
DEBUG_BREAK_IF(true);
164164
return false;
165165
}

‎shared/source/os_interface/windows/wddm/skip_resource_cleanup_drm_or_wddm.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Copyright (C) 2021 Intel Corporation
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*
6+
*/
7+
8+
#include "shared/source/os_interface/windows/gdi_interface.h"
9+
#include "shared/source/os_interface/windows/wddm/wddm.h"
10+
11+
namespace NEO {
12+
13+
bool Wddm::skipResourceCleanup() const {
14+
D3DKMT_GETDEVICESTATE deviceState = {};
15+
deviceState.hDevice = device;
16+
deviceState.StateType = D3DKMT_DEVICESTATE_PRESENT;
17+
18+
NTSTATUS status = STATUS_SUCCESS;
19+
status = getGdi()->getDeviceState(&deviceState);
20+
return status != STATUS_SUCCESS;
21+
}
22+
23+
} // namespace NEO

‎shared/source/os_interface/windows/wddm/skip_resource_cleanup_wddm.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright (C) 2021 Intel Corporation
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*
6+
*/
7+
8+
#include "shared/source/os_interface/windows/wddm/wddm.h"
9+
10+
namespace NEO {
11+
12+
bool Wddm::skipResourceCleanup() const {
13+
return false;
14+
}
15+
16+
} // namespace NEO

‎shared/source/os_interface/windows/wddm/wddm.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ class Wddm : public DriverModel {
189189
PhysicalDevicePciBusInfo getPciBusInfo() const override;
190190

191191
size_t getMaxMemAllocSize() const override;
192+
bool skipResourceCleanup() const override;
192193

193194
static std::vector<std::unique_ptr<HwDeviceId>> discoverDevices(ExecutionEnvironment &executionEnvironment);
194195

‎shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp

Lines changed: 43 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,9 @@ struct GdiMockConfig {
118118
D3DKMT_LOCK2 receivedLock2Args = {};
119119
D3DKMT_DESTROYALLOCATION2 receivedDestroyAllocation2Args = {};
120120
uint32_t mockAllocationHandle = 7U;
121+
122+
D3DKMT_GETDEVICESTATE receivedGetDeviceStateArgs = {};
123+
GdiMockCallbackWithReturn getDeviceStateClb = {};
121124
} gdiMockConfig;
122125

123126
NTSTATUS __stdcall reserveDeviceAddressSpaceMock(D3DDDI_RESERVEGPUVIRTUALADDRESS *arg) {
@@ -185,6 +188,15 @@ NTSTATUS __stdcall escapeMock(const D3DKMT_ESCAPE *arg) {
185188
return gdiMockConfig.escapeClb.returnValue;
186189
}
187190

191+
NTSTATUS __stdcall getDeviceStateMock(D3DKMT_GETDEVICESTATE *arg) {
192+
gdiMockConfig.receivedGetDeviceStateArgs = *arg;
193+
gdiMockConfig.getDeviceStateClb.callCount += 1;
194+
if (gdiMockConfig.getDeviceStateClb.callback) {
195+
gdiMockConfig.getDeviceStateClb.callback();
196+
}
197+
return gdiMockConfig.getDeviceStateClb.returnValue;
198+
}
199+
188200
struct WddmLinuxTest : public ::testing::Test {
189201
void SetUp() override {
190202
mockRootDeviceEnvironment = std::make_unique<NEO::MockRootDeviceEnvironment>(mockExecEnv);
@@ -248,9 +260,16 @@ TEST_F(WddmLinuxConfigureDeviceAddressSpaceTest, givenPreReservedSvmAddressSpace
248260
}
249261

250262
this->osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock;
251-
gdiMockConfig.reserveGpuVaClb.returnValue = -2;
252-
gdiMockConfig.reserveGpuVaClb.callback = []() { gdiMockConfig.reserveGpuVaClb.returnValue += 1; };
263+
auto cantReserveWholeGpuVAButCanReservePortion = []() {
264+
gdiMockConfig.reserveGpuVaClb.returnValue = (gdiMockConfig.reserveGpuVaClb.callCount == 1) ? -1 : 0;
265+
};
266+
gdiMockConfig.reserveGpuVaClb.callback = cantReserveWholeGpuVAButCanReservePortion;
253267
bool success = this->wddm->configureDeviceAddressSpace();
268+
EXPECT_FALSE(success);
269+
270+
auto cantReserveWholeGpuVAAndCantReservePortion = []() { gdiMockConfig.reserveGpuVaClb.returnValue = -1; };
271+
gdiMockConfig.reserveGpuVaClb.callback = cantReserveWholeGpuVAAndCantReservePortion;
272+
success = this->wddm->configureDeviceAddressSpace();
254273
EXPECT_TRUE(success);
255274

256275
auto svmSize = this->getMaxSvmSize();
@@ -261,19 +280,6 @@ TEST_F(WddmLinuxConfigureDeviceAddressSpaceTest, givenPreReservedSvmAddressSpace
261280
EXPECT_EQ(wddm->getAdapter(), gdiMockConfig.receivedReserveGpuVaArgs.hAdapter);
262281
}
263282

264-
TEST_F(WddmLinuxConfigureDeviceAddressSpaceTest, givenSvmAddressSpaceWhenCouldNotReserveGpuVAForSvmThenFail) {
265-
if (NEO::hardwareInfoTable[productFamily]->capabilityTable.gpuAddressSpace < MemoryConstants::max64BitAppAddress) {
266-
GTEST_SKIP();
267-
}
268-
269-
osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock;
270-
this->osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock;
271-
272-
gdiMockConfig.reserveGpuVaClb.returnValue = -1;
273-
bool success = this->wddm->configureDeviceAddressSpace();
274-
EXPECT_FALSE(success);
275-
}
276-
277283
TEST_F(WddmLinuxConfigureDeviceAddressSpaceTest, givenNonSvmAddressSpaceThenReserveGpuVAForUSMIsNotCalled) {
278284
if (NEO::hardwareInfoTable[productFamily]->capabilityTable.gpuAddressSpace >= MemoryConstants::max64BitAppAddress) {
279285
GTEST_SKIP();
@@ -531,10 +537,13 @@ TEST_F(WddmLinuxConfigureReduced48bitDeviceAddressSpaceTest, givenTwoSvmAddressS
531537
}
532538

533539
this->wddm->featureTable->ftrCCSRing = 1;
534-
gdiMockConfig.reserveGpuVaClb.returnValue = -1;
535540

536541
osEnvironment->gdi->escape = escapeMock;
537542
osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock;
543+
auto cantReserveWholeGpuVAButCanReservePortion = []() {
544+
gdiMockConfig.reserveGpuVaClb.returnValue = (gdiMockConfig.reserveGpuVaClb.callCount == 1) ? -1 : 0;
545+
};
546+
gdiMockConfig.reserveGpuVaClb.callback = cantReserveWholeGpuVAButCanReservePortion;
538547
bool success = this->wddm->configureDeviceAddressSpace();
539548
EXPECT_FALSE(success);
540549
EXPECT_EQ(1U, this->wddm->validAddressRangeReservations.size());
@@ -549,14 +558,12 @@ TEST_F(WddmLinuxConfigureReduced48bitDeviceAddressSpaceTest, givenTwoSvmAddressS
549558
}
550559

551560
this->wddm->featureTable->ftrCCSRing = 1;
552-
gdiMockConfig.reserveGpuVaClb.callback = []() {
553-
if (gdiMockConfig.reserveGpuVaClb.callCount > 1) {
554-
gdiMockConfig.reserveGpuVaClb.returnValue = -1;
555-
};
556-
};
557-
558561
osEnvironment->gdi->escape = escapeMock;
559562
osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock;
563+
auto cantReserveWholeGpuVAOfSecondButCanReservePortionOfSecont = []() {
564+
gdiMockConfig.reserveGpuVaClb.returnValue = (gdiMockConfig.reserveGpuVaClb.callCount == 2) ? -1 : 0;
565+
};
566+
gdiMockConfig.reserveGpuVaClb.callback = cantReserveWholeGpuVAOfSecondButCanReservePortionOfSecont;
560567
bool success = this->wddm->configureDeviceAddressSpace();
561568
EXPECT_FALSE(success);
562569
EXPECT_EQ(1U, this->wddm->validAddressRangeReservations.size());
@@ -605,6 +612,20 @@ TEST_F(WddmLinuxTest, givenRequestFor32bitAllocationWithoutPreexistingHostPtrWhe
605612
memoryManager.freeGraphicsMemoryImpl(alloc);
606613
}
607614

615+
TEST_F(WddmLinuxTest, whenCheckedIfResourcesCleanupCanBeSkippedAndDeviceIsAliveThenReturnsFalse) {
616+
osEnvironment->gdi->getDeviceState = getDeviceStateMock;
617+
gdiMockConfig.getDeviceStateClb.returnValue = STATUS_SUCCESS;
618+
EXPECT_FALSE(this->wddm->skipResourceCleanup());
619+
EXPECT_EQ(1, gdiMockConfig.getDeviceStateClb.callCount);
620+
}
621+
622+
TEST_F(WddmLinuxTest, whenCheckedIfResourcesCleanupCanBeSkippedAndDeviceIsLostThenReturnsTrue) {
623+
osEnvironment->gdi->getDeviceState = getDeviceStateMock;
624+
gdiMockConfig.getDeviceStateClb.returnValue = -1;
625+
EXPECT_TRUE(this->wddm->skipResourceCleanup());
626+
EXPECT_EQ(1, gdiMockConfig.getDeviceStateClb.callCount);
627+
}
628+
608629
class MockOsTimeLinux : public NEO::OSTimeLinux {
609630
public:
610631
MockOsTimeLinux(NEO::OSInterface *osInterface, std::unique_ptr<NEO::DeviceTime> deviceTime) : NEO::OSTimeLinux(osInterface, std::move(deviceTime)) {}

‎shared/test/unit_test/os_interface/windows/wddm_tests.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,9 @@ TEST_F(WddmTests, givenWddmWhenPassesIncorrectHandleToVerifyNTHandleThenReturnFa
7070
EXPECT_FALSE(wddm->verifyNTHandle(handle));
7171
}
7272

73+
TEST_F(WddmTests, whenCheckedIfResourcesCleanupCanBeSkippedThenReturnsFalse) {
74+
init();
75+
EXPECT_FALSE(wddm->skipResourceCleanup());
76+
}
77+
7378
} // namespace NEO

0 commit comments

Comments
 (0)