Revert "Move and extend local memory DRM allocations"

This reverts commit b84b001.

Signed-off-by: Compute-Runtime-Validation <[email protected]>

Author: Compute-Runtime-Validation

level_zero/core/source/event/event.cpp

@@ -104,14 +104,8 @@ ze_result_t EventPoolImp::initialize(DriverHandle *driver, Context *context, uin
         }
 
     } else {
-        const bool isShareable = (eventPoolFlags & ZE_EVENT_POOL_FLAG_IPC);
-        if (isShareable) {
-            allocationType = NEO::AllocationType::TIMESTAMP_PACKET_TAG_BUFFER;
-        }
-
         NEO::AllocationProperties allocationProperties{*rootDeviceIndices.begin(), alignedSize, allocationType, systemMemoryBitfield};
         allocationProperties.alignment = eventAlignment;
-        allocationProperties.flags.shareable = isShareable;
 
         std::vector<uint32_t> rootDeviceIndicesVector = {rootDeviceIndices.begin(), rootDeviceIndices.end()};
         eventPoolPtr = driver->getMemoryManager()->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector,

level_zero/core/test/unit_tests/sources/event/test_event.cpp

@@ -258,7 +258,7 @@ TEST_F(EventPoolIPCHandleTests, whenGettingIpcHandleForEventPoolThenHandleAndNum
     ze_event_pool_desc_t eventPoolDesc = {
         ZE_STRUCTURE_TYPE_EVENT_POOL_DESC,
         nullptr,
-        ZE_EVENT_POOL_FLAG_HOST_VISIBLE | ZE_EVENT_POOL_FLAG_IPC,
+        ZE_EVENT_POOL_FLAG_HOST_VISIBLE,
         numEvents};
 
     auto deviceHandle = device->toHandle();

opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp

@@ -1240,34 +1240,6 @@ TEST_F(DrmMemoryManagerTest, GivenShareableEnabledWhenAskedToCreateGraphicsAlloc
     memoryManager->freeGraphicsMemory(allocation);
 }
 
-TEST_F(DrmMemoryManagerTest, GivenAllocationTypeThatRequiresCpuAccessForKmdAllocationThenLockTheResourceIsCalled) {
-    struct DrmMemoryManagerToTestLockResource : public DrmMemoryManager {
-        using DrmMemoryManager::allocateMemoryByKMD;
-
-        DrmMemoryManagerToTestLockResource(ExecutionEnvironment &executionEnvironment)
-            : DrmMemoryManager(gemCloseWorkerMode::gemCloseWorkerInactive, false, false, executionEnvironment) {
-        }
-        void *lockResourceImpl(GraphicsAllocation &allocation) override {
-            return reinterpret_cast<void *>(0xDEADBEEF);
-        }
-    };
-
-    DrmMemoryManagerToTestLockResource drmMemoryManager(*executionEnvironment);
-
-    mock->ioctl_expected.gemWait = 1;
-    mock->ioctl_expected.gemCreate = 1;
-    mock->ioctl_expected.gemClose = 1;
-
-    allocationData.type = AllocationType::TIMESTAMP_PACKET_TAG_BUFFER;
-
-    auto allocation = drmMemoryManager.allocateMemoryByKMD(allocationData);
-    EXPECT_NE(nullptr, allocation);
-    EXPECT_NE(0u, allocation->getGpuAddress());
-    EXPECT_EQ(0xDEADBEEF, reinterpret_cast<uintptr_t>(allocation->getUnderlyingBuffer()));
-
-    memoryManager->freeGraphicsMemory(allocation);
-}
-
 TEST_F(DrmMemoryManagerTest, GivenMisalignedHostPtrAndMultiplePagesSizeWhenAskedForGraphicsAllocationThenItContainsAllFragmentsWithProperGpuAdrresses) {
     mock->ioctl_expected.gemUserptr = 3;
     mock->ioctl_expected.gemWait = 3;

opencl/test/unit_test/test_files/igdrcl.config

@@ -389,4 +389,3 @@ AccessCountersGranularity = -1
 OverridePatIndex = -1
 UseTileMemoryBankInVirtualMemoryCreation = -1
 DisableScratchPages = 0
-SetVmAdviseAtomicAttribute = -1

shared/source/debug_settings/debug_variables_base.inl

@@ -192,7 +192,6 @@ DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersTrigger, -1, "-1: default - disabl
 DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersGranularity, -1, "-1: default - ACG_2MB, >= 0: granularites - 0: ACG_128K, 1: ACG_2M, 2: ACG_16M, 3: ACG_16M")
 DECLARE_DEBUG_VARIABLE(int32_t, OverridePatIndex, -1, "-1: default, >=0: PatIndex to override")
 DECLARE_DEBUG_VARIABLE(int32_t, UseTileMemoryBankInVirtualMemoryCreation, -1, "-1: default - on, 0: do not assign tile memory bank to virtual memory space, 1: assign tile memory bank to virtual memory space")
-DECLARE_DEBUG_VARIABLE(int32_t, SetVmAdviseAtomicAttribute, -1, "-1: default - atomic system, 0: atomic none, 1: atomic device, 2: atomic system)")
 DECLARE_DEBUG_VARIABLE(bool, DisableScratchPages, false, "Disable scratch pages during VM creations")
 /*LOGGING FLAGS*/
 DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level")

shared/source/generated/gen12lp/hw_cmds_generated_gen12lp.inl

@@ -5062,7 +5062,7 @@ typedef struct tagMI_STORE_DATA_IMM {
         ADDRESS_ALIGN_SIZE = 0x4,
     } ADDRESS;
     inline void setAddress(const uint64_t value) {
-        UNRECOVERABLE_IF((value >> ADDRESS_BIT_SHIFT) > 0x3fffffffffffffffL);
+        UNRECOVERABLE_IF(value > 0x3fffffffffffffffL);
         TheStructure.Common.Address = value >> ADDRESS_BIT_SHIFT;
     }
     inline uint64_t getAddress() const {

shared/source/os_interface/linux/CMakeLists.txt

@@ -37,10 +37,11 @@ set(NEO_CORE_OS_INTERFACE_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler.h
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_manager_create_multi_host_allocation.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_local_memory.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_query.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_drm.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -21,8 +21,6 @@
 #include "shared/source/helpers/string.h"
 #include "shared/source/helpers/surface_format_info.h"
 #include "shared/source/memory_manager/host_ptr_manager.h"
-#include "shared/source/memory_manager/memory_banks.h"
-#include "shared/source/memory_manager/memory_pool.h"
 #include "shared/source/memory_manager/residency.h"
 #include "shared/source/os_interface/linux/allocator_helper.h"
 #include "shared/source/os_interface/linux/drm_memory_operations_handler.h"
@@ -506,12 +504,6 @@ GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData &
     allocation->setDefaultGmm(gmm.release());
 
     allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuRange), bufferSize);
-
-    if (GraphicsAllocation::isCpuAccessRequired(allocationData.type)) {
-        auto cpuAddress = lockResource(allocation);
-        allocation->setCpuPtrAndGpuAddress(cpuAddress, gpuRange);
-    }
-
     bo.release();
     return allocation;
 }
@@ -1624,160 +1616,4 @@ void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) {
     return bo->peekLockedAddress();
 }
 
-void createMemoryRegionsForSharedAllocation(const HardwareInfo &hwInfo, MemoryInfo &memoryInfo, const AllocationData &allocationData, MemRegionsVec &memRegions) {
-    auto memoryBanks = allocationData.storageInfo.memoryBanks;
-
-    if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) {
-        //System memory region
-        auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
-        memRegions.push_back(regionClassAndInstance);
-    }
-
-    //All local memory regions
-    size_t currentBank = 0;
-    size_t i = 0;
-
-    while (i < memoryBanks.count()) {
-        if (memoryBanks.test(currentBank)) {
-            auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(1u << currentBank, hwInfo);
-            memRegions.push_back(regionClassAndInstance);
-            i++;
-        }
-        currentBank++;
-    }
-
-    if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::GPU) {
-        //System memory region
-        auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
-        memRegions.push_back(regionClassAndInstance);
-    }
-}
-
-GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
-    auto &drm = this->getDrm(allocationData.rootDeviceIndex);
-
-    const auto vmAdviseAttribute = drm.getIoctlHelper()->getVmAdviseAtomicAttribute();
-    if (vmAdviseAttribute == 0) {
-        return nullptr;
-    }
-
-    auto memoryInfo = drm.getMemoryInfo();
-    if (!memoryInfo) {
-        return nullptr;
-    }
-
-    auto size = allocationData.size;
-    auto alignment = allocationData.alignment;
-
-    auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo();
-
-    MemRegionsVec memRegions;
-    createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memRegions);
-
-    uint32_t handle = 0;
-    auto ret = memoryInfo->createGemExt(&drm, memRegions, size, handle);
-
-    if (ret) {
-        return nullptr;
-    }
-
-    std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(&drm, handle, size, maxOsContextCount));
-
-    if (!drm.getIoctlHelper()->setVmBoAdvise(&drm, bo->peekHandle(), vmAdviseAttribute, nullptr)) {
-        return nullptr;
-    }
-
-    uint64_t offset = 0;
-    if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
-        return nullptr;
-    }
-
-    auto totalSizeToAlloc = size + alignment;
-    auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-
-    auto cpuBasePointer = cpuPointer;
-    cpuPointer = alignUp(cpuPointer, alignment);
-
-    this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast<off_t>(offset));
-
-    bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
-
-    auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages);
-    allocation->setMmapPtr(cpuBasePointer);
-    allocation->setMmapSize(totalSizeToAlloc);
-    if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast<CacheRegion>(allocationData.cacheRegion))) {
-        this->munmapFunction(cpuPointer, totalSizeToAlloc);
-        return nullptr;
-    }
-
-    bo.release();
-
-    return allocation.release();
-}
-
-DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
-    drm_prime_handle openFd = {0, 0, 0};
-    openFd.fd = handle;
-
-    auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
-    if (ret != 0) {
-        int err = this->getDrm(properties.rootDeviceIndex).getErrno();
-        PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
-        DEBUG_BREAK_IF(ret != 0);
-        return nullptr;
-    }
-
-    if (hasMappedPtr) {
-        auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
-        bo->setAddress(properties.gpuAddress);
-
-        return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
-                                 handle, MemoryPool::SystemCpuInaccessible);
-    }
-
-    auto boHandle = openFd.handle;
-    auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex);
-
-    void *cpuPointer = nullptr;
-    size_t size = lseekFunction(handle, 0, SEEK_END);
-
-    if (bo == nullptr) {
-        bo = new BufferObject(&getDrm(properties.rootDeviceIndex), boHandle, size, maxOsContextCount);
-        cpuPointer = this->mmapFunction(0, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-        bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
-
-        uint64_t offset = 0;
-        if (!retrieveMmapOffsetForBufferObject(properties.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
-            this->munmapFunction(cpuPointer, size);
-            delete bo;
-            return nullptr;
-        }
-
-        [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(properties.rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
-        DEBUG_BREAK_IF(retPtr != cpuPointer);
-
-        AllocationData allocationData = {};
-        allocationData.rootDeviceIndex = properties.rootDeviceIndex;
-        allocationData.size = size;
-        emitPinningRequest(bo, allocationData);
-
-        bo->setUnmapSize(size);
-        bo->setRootDeviceIndex(properties.rootDeviceIndex);
-
-        pushSharedBufferObject(bo);
-
-        DrmAllocation *drmAllocation = nullptr;
-        drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages);
-        drmAllocation->setMmapPtr(cpuPointer);
-        drmAllocation->setMmapSize(size);
-        drmAllocation->setReservedAddressRange(reinterpret_cast<void *>(cpuPointer), size);
-        drmAllocation->setCacheRegion(&this->getDrm(properties.rootDeviceIndex), static_cast<CacheRegion>(properties.cacheRegion));
-
-        return drmAllocation;
-    }
-
-    return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
-                             handle, MemoryPool::SystemCpuInaccessible);
-}
-
 } // namespace NEO

shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2019-2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/helpers/basic_math.h"
+#include "shared/source/os_interface/linux/drm_memory_manager.h"
+
+namespace NEO {
+
+DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
+    drm_prime_handle openFd = {0, 0, 0};
+    openFd.fd = handle;
+
+    auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
+
+    if (ret != 0) {
+        int err = this->getDrm(properties.rootDeviceIndex).getErrno();
+        PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
+        DEBUG_BREAK_IF(ret != 0);
+        return nullptr;
+    }
+
+    auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
+    bo->setAddress(properties.gpuAddress);
+
+    return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
+                             handle, MemoryPool::SystemCpuInaccessible);
+}
+
+GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
+    return nullptr;
+}
+
+} // namespace NEO

shared/source/os_interface/linux/ioctl_helper.h

@@ -110,7 +110,6 @@ class IoctlHelper {
     virtual void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) = 0;
     virtual std::optional<uint64_t> getCopyClassSaturatePCIECapability() = 0;
     virtual std::optional<uint64_t> getCopyClassSaturateLinkCapability() = 0;
-    virtual uint32_t getVmAdviseAtomicAttribute() = 0;
     virtual int vmBind(Drm *drm, const VmBindParams &vmBindParams) = 0;
     virtual int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) = 0;
     virtual bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) = 0;
@@ -156,7 +155,6 @@ class IoctlHelperUpstream : public IoctlHelper {
     void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override;
     std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
     std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
-    uint32_t getVmAdviseAtomicAttribute() override;
     int vmBind(Drm *drm, const VmBindParams &vmBindParams) override;
     int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override;
     bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override;
@@ -215,7 +213,6 @@ class IoctlHelperPrelim20 : public IoctlHelper {
     void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override;
     std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
     std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
-    uint32_t getVmAdviseAtomicAttribute() override;
     int vmBind(Drm *drm, const VmBindParams &vmBindParams) override;
     int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override;
     bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override;

shared/source/os_interface/linux/ioctl_helper_prelim.cpp

@@ -451,17 +451,6 @@ std::optional<uint64_t> IoctlHelperPrelim20::getCopyClassSaturateLinkCapability(
     return PRELIM_I915_COPY_CLASS_CAP_SATURATE_LINK;
 }
 
-uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() {
-    switch (NEO::DebugManager.flags.SetVmAdviseAtomicAttribute.get()) {
-    case 0:
-        return PRELIM_I915_VM_ADVISE_ATOMIC_NONE;
-    case 1:
-        return PRELIM_I915_VM_ADVISE_ATOMIC_DEVICE;
-    default:
-        return PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM;
-    }
-}
-
 prelim_drm_i915_gem_vm_bind translateVmBindParamsToPrelimStruct(const VmBindParams &vmBindParams) {
     prelim_drm_i915_gem_vm_bind vmBind{};
     vmBind.vm_id = vmBindParams.vmId;

shared/source/os_interface/linux/ioctl_helper_upstream.cpp

@@ -204,10 +204,6 @@ std::optional<uint64_t> IoctlHelperUpstream::getCopyClassSaturateLinkCapability(
     return std::nullopt;
 }
 
-uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() {
-    return 0;
-}
-
 int IoctlHelperUpstream::vmBind(Drm *drm, const VmBindParams &vmBindParams) {
     return 0;
 }