Skip to content

Commit 7149743

Browse files
bibrakCompute-Runtime-Automation
bibrak
authored and
Compute-Runtime-Automation
committed
fix: Set vmbind user fence when makeMemoryResident
Related-To: NEO-11977, GSD-10293 Signed-off-by: Chandio, Bibrak Qamar <[email protected]>
1 parent 18d7a22 commit 7149743

File tree

50 files changed

+616
-336
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+616
-336
lines changed

‎level_zero/core/source/context/context_imp.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ ze_result_t ContextImp::makeMemoryResident(ze_device_handle_t hDevice, void *ptr
513513
}
514514

515515
NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
516-
auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), true);
516+
auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), true, true);
517517
ze_result_t res = changeMemoryOperationStatusToL0ResultType(success);
518518

519519
if (ZE_RESULT_SUCCESS == res) {
@@ -557,9 +557,9 @@ ze_result_t ContextImp::makeImageResident(ze_device_handle_t hDevice, ze_image_h
557557

558558
NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice();
559559
NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
560-
auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&alloc, 1), true);
560+
auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&alloc, 1), true, false);
561561
if (implicitArgsAlloc) {
562-
memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&implicitArgsAlloc, 1), true);
562+
memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&implicitArgsAlloc, 1), true, false);
563563
}
564564
return changeMemoryOperationStatusToL0ResultType(success);
565565
}
@@ -1312,7 +1312,7 @@ ze_result_t ContextImp::mapVirtualMem(const void *ptr,
13121312
virtualMemoryReservation->mappedAllocations.insert(std::pair<void *, NEO::MemoryMappedRange *>(const_cast<void *>(ptr), mappedRange));
13131313
this->driverHandle->getSvmAllocsManager()->insertSVMAlloc(allocData);
13141314
NEO::MemoryOperationsHandler *memoryOperationsIface = allocationNode->device->getRootDeviceEnvironment().memoryOperationsInterface.get();
1315-
auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef<NEO::GraphicsAllocation *>(&allocationNode->allocation, 1), false);
1315+
auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef<NEO::GraphicsAllocation *>(&allocationNode->allocation, 1), false, false);
13161316
ze_result_t res = changeMemoryOperationStatusToL0ResultType(success);
13171317
return res;
13181318
} else {

‎level_zero/core/source/module/module_imp.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1048,7 +1048,7 @@ void ModuleImp::copyPatchedSegments(const NEO::Linker::PatchableSegments &isaSeg
10481048
NEO::MemoryOperationsHandler *memoryOperationsIface = rootDeviceEnvironment.memoryOperationsInterface.get();
10491049
auto allocation = kernelImmData->getIsaGraphicsAllocation();
10501050
if (memoryOperationsIface) {
1051-
memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), false);
1051+
memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), false, false);
10521052
}
10531053
}
10541054
}

‎opencl/source/mem_obj/buffer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -575,7 +575,7 @@ Buffer *Buffer::create(Context *context,
575575
auto device = context->getDevice(deviceNum);
576576
auto graphicsAllocation = pBuffer->getGraphicsAllocation(device->getRootDeviceIndex());
577577
auto rootDeviceEnvironment = pBuffer->executionEnvironment->rootDeviceEnvironments[device->getRootDeviceIndex()].get();
578-
rootDeviceEnvironment->memoryOperationsInterface->makeResident(&device->getDevice(), ArrayRef<GraphicsAllocation *>(&graphicsAllocation, 1), false);
578+
rootDeviceEnvironment->memoryOperationsInterface->makeResident(&device->getDevice(), ArrayRef<GraphicsAllocation *>(&graphicsAllocation, 1), false, false);
579579
}
580580
}
581581

‎opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ struct WddmMemoryOperationsHandlerTest : public WddmTest {
5858

5959
TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenMakingResidentAllocationThenMakeResidentIsCalledAndAllocationIsMarkedAsExplicitlyResident) {
6060
wddmAllocation->setExplicitlyMadeResident(false);
61-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false), MemoryOperationsStatus::success);
61+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false, false), MemoryOperationsStatus::success);
6262
EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success);
6363
EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident());
6464
}
@@ -67,7 +67,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenMakingResid
6767
allocationPtr = wddmFragmentedAllocation.get();
6868
allocationPtr->setExplicitlyMadeResident(false);
6969

70-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false), MemoryOperationsStatus::success);
70+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false, false), MemoryOperationsStatus::success);
7171
EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success);
7272
EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident());
7373
}
@@ -78,7 +78,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenMakingResiden
7878
allocation->setExplicitlyMadeResident(false);
7979
}
8080

81-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false), MemoryOperationsStatus::success);
81+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false, false), MemoryOperationsStatus::success);
8282
EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success);
8383
EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success);
8484

@@ -91,7 +91,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenEvictingReside
9191
wddm->callBaseEvict = true;
9292
allocationPtr->setExplicitlyMadeResident(false);
9393

94-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false), MemoryOperationsStatus::success);
94+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false, false), MemoryOperationsStatus::success);
9595
EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident());
9696

9797
EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success);
@@ -105,7 +105,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenEvictingRes
105105
allocationPtr = wddmFragmentedAllocation.get();
106106
allocationPtr->setExplicitlyMadeResident(false);
107107

108-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false), MemoryOperationsStatus::success);
108+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1), false, false), MemoryOperationsStatus::success);
109109
EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident());
110110

111111
EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success);
@@ -119,7 +119,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenEvictingResid
119119
allocation->setExplicitlyMadeResident(false);
120120
}
121121
wddm->evictResult.called = 0;
122-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false), MemoryOperationsStatus::success);
122+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false, false), MemoryOperationsStatus::success);
123123
EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident());
124124
EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident());
125125

@@ -143,7 +143,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenFreeResidentA
143143
allocation->setExplicitlyMadeResident(false);
144144
}
145145
wddm->evictResult.called = 0;
146-
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false), MemoryOperationsStatus::success);
146+
EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef<GraphicsAllocation *>(allocationData), false, false), MemoryOperationsStatus::success);
147147
EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident());
148148
EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident());
149149

‎shared/source/debugger/debugger_l0.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,11 @@ void DebuggerL0::initialize() {
9898

9999
NEO::MemoryOperationsHandler *memoryOperationsIface = rootDeviceEnvironment.memoryOperationsInterface.get();
100100
if (memoryOperationsIface) {
101-
memoryOperationsIface->makeResident(device, ArrayRef<NEO::GraphicsAllocation *>(&moduleDebugArea, 1), false);
101+
memoryOperationsIface->makeResident(device, ArrayRef<NEO::GraphicsAllocation *>(&moduleDebugArea, 1), false, false);
102102
auto numSubDevices = device->getNumSubDevices();
103103
for (uint32_t i = 0; i < numSubDevices; i++) {
104104
auto subDevice = device->getSubDevice(i);
105-
memoryOperationsIface->makeResident(subDevice, ArrayRef<NEO::GraphicsAllocation *>(&moduleDebugArea, 1), false);
105+
memoryOperationsIface->makeResident(subDevice, ArrayRef<NEO::GraphicsAllocation *>(&moduleDebugArea, 1), false, false);
106106
}
107107
}
108108

@@ -142,7 +142,7 @@ void DebuggerL0::notifyModuleLoadAllocations(Device *device, const StackVec<NEO:
142142
NEO::MemoryOperationsHandler *memoryOperationsIface = device->getRootDeviceEnvironment().memoryOperationsInterface.get();
143143
if (memoryOperationsIface) {
144144
for (auto gfxAlloc : allocs) {
145-
memoryOperationsIface->makeResident(device, ArrayRef<NEO::GraphicsAllocation *>(&gfxAlloc, 1), false);
145+
memoryOperationsIface->makeResident(device, ArrayRef<NEO::GraphicsAllocation *>(&gfxAlloc, 1), false, false);
146146
}
147147
}
148148
}

‎shared/source/direct_submission/direct_submission_hw.inl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,7 @@ bool DirectSubmissionHw<GfxFamily, Dispatcher>::allocateResources() {
420420

421421
template <typename GfxFamily, typename Dispatcher>
422422
bool DirectSubmissionHw<GfxFamily, Dispatcher>::makeResourcesResident(DirectSubmissionAllocations &allocations) {
423-
auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(allocations), false) == MemoryOperationsStatus::success;
423+
auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(allocations), false, false) == MemoryOperationsStatus::success;
424424

425425
return ret;
426426
}
@@ -1140,7 +1140,7 @@ inline GraphicsAllocation *DirectSubmissionHw<GfxFamily, Dispatcher>::switchRing
11401140
nextAllocation = memoryManager->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties);
11411141
this->currentRingBuffer = static_cast<uint32_t>(this->ringBuffers.size());
11421142
this->ringBuffers.emplace_back(0ull, nextAllocation);
1143-
auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(&nextAllocation, 1u), false) == MemoryOperationsStatus::success;
1143+
auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef<GraphicsAllocation *>(&nextAllocation, 1u), false, false) == MemoryOperationsStatus::success;
11441144
UNRECOVERABLE_IF(!ret);
11451145
}
11461146
}

‎shared/source/helpers/bindless_heaps_helper.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size
153153

154154
GraphicsAllocation *allocation = memManager->allocateGraphicsMemoryWithProperties(properties);
155155
MemoryOperationsHandler *memoryOperationsIface = rootDevice->getRootDeviceEnvironmentRef().memoryOperationsInterface.get();
156-
auto result = memoryOperationsIface->makeResident(rootDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), false);
156+
auto result = memoryOperationsIface->makeResident(rootDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1), false, false);
157157
if (result != NEO::MemoryOperationsStatus::success) {
158158
memManager->freeGraphicsMemory(allocation);
159159
return nullptr;

‎shared/source/memory_manager/memory_operations_handler.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ class MemoryOperationsHandler {
2020
MemoryOperationsHandler() = default;
2121
virtual ~MemoryOperationsHandler() = default;
2222

23-
virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded) = 0;
23+
virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) = 0;
2424
virtual MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) = 0;
2525
virtual MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) = 0;
2626
virtual MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) = 0;
2727
virtual MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) { return MemoryOperationsStatus::success; }
2828

29-
virtual MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) = 0;
29+
virtual MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable, const bool forcePagingFence) = 0;
3030
virtual MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) = 0;
3131
virtual void processFlushResidency(CommandStreamReceiver *csr) {}
3232
};

‎shared/source/os_interface/aub_memory_operations_handler.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ AubMemoryOperationsHandler::AubMemoryOperationsHandler(aub_stream::AubManager *a
2626
this->aubManager = aubManager;
2727
}
2828

29-
MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded) {
29+
MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) {
3030
if (!aubManager) {
3131
return MemoryOperationsStatus::deviceUninitialized;
3232
}
@@ -76,7 +76,7 @@ MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device,
7676
}
7777

7878
MemoryOperationsStatus AubMemoryOperationsHandler::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
79-
return makeResident(device, gfxAllocations, false);
79+
return makeResident(device, gfxAllocations, false, false);
8080
}
8181

8282
MemoryOperationsStatus AubMemoryOperationsHandler::evict(Device *device, GraphicsAllocation &gfxAllocation) {
@@ -98,8 +98,8 @@ MemoryOperationsStatus AubMemoryOperationsHandler::free(Device *device, Graphics
9898
return MemoryOperationsStatus::success;
9999
}
100100

101-
MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) {
102-
return makeResident(nullptr, gfxAllocations, false);
101+
MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable, const bool forcePagingFence) {
102+
return makeResident(nullptr, gfxAllocations, false, forcePagingFence);
103103
}
104104

105105
MemoryOperationsStatus AubMemoryOperationsHandler::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) {

‎shared/source/os_interface/aub_memory_operations_handler.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ class AubMemoryOperationsHandler : public MemoryOperationsHandler {
2222
AubMemoryOperationsHandler(aub_stream::AubManager *aubManager);
2323
~AubMemoryOperationsHandler() override = default;
2424

25-
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded) override;
25+
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override;
2626
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
2727
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;
2828
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;
2929
MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override;
3030

31-
MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) override;
31+
MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable, const bool forcePagingFence) override;
3232
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
3333

3434
void processFlushResidency(CommandStreamReceiver *csr) override;

0 commit comments

Comments
 (0)