Add new colouring policy

Signed-off-by: Lukasz Jobczyk <[email protected]>

Author: LukaszJobczyk

opencl/test/unit_test/memory_manager/storage_info_tests.cpp

@@ -246,6 +246,8 @@ TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForBufferCompressedThe
     EXPECT_EQ(allTilesMask, storageInfo.memoryBanks);
     EXPECT_EQ(allTilesMask, storageInfo.pageTablesVisibility);
     EXPECT_TRUE(storageInfo.multiStorage);
+    EXPECT_EQ(storageInfo.colouringGranularity, MemoryConstants::pageSize64k);
+    EXPECT_EQ(storageInfo.colouringPolicy, ColouringPolicy::DeviceCountBased);
 }
 
 TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForBufferThenAllMemoryBanksAreOnAndPageTableClonningIsRequired) {
@@ -255,6 +257,8 @@ TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForBufferThenAllMemory
     EXPECT_EQ(allTilesMask, storageInfo.memoryBanks);
     EXPECT_EQ(allTilesMask, storageInfo.pageTablesVisibility);
     EXPECT_TRUE(storageInfo.multiStorage);
+    EXPECT_EQ(storageInfo.colouringGranularity, MemoryConstants::pageSize64k);
+    EXPECT_EQ(storageInfo.colouringPolicy, ColouringPolicy::DeviceCountBased);
 }
 
 TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForSVMGPUThenAllMemoryBanksAreOnAndPageTableClonningIsRequired) {
@@ -264,6 +268,22 @@ TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForSVMGPUThenAllMemory
     EXPECT_EQ(allTilesMask, storageInfo.memoryBanks);
     EXPECT_EQ(allTilesMask, storageInfo.pageTablesVisibility);
     EXPECT_TRUE(storageInfo.multiStorage);
+    EXPECT_EQ(storageInfo.colouringGranularity, MemoryConstants::pageSize64k);
+    EXPECT_EQ(storageInfo.colouringPolicy, ColouringPolicy::DeviceCountBased);
+}
+
+TEST_F(MultiDeviceStorageInfoTest, givenMultiStorageGranularityWhenCreatingStorageInfoThenProperGranularityIsSet) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.MultiStorageGranularity.set(128);
+
+    AllocationProperties properties{mockRootDeviceIndex, false, 10 * MemoryConstants::pageSize64k, GraphicsAllocation::AllocationType::SVM_GPU, true, allTilesMask};
+    auto storageInfo = memoryManager->createStorageInfoFromProperties(properties);
+    EXPECT_TRUE(storageInfo.cloningOfPageTables);
+    EXPECT_EQ(allTilesMask, storageInfo.memoryBanks);
+    EXPECT_EQ(allTilesMask, storageInfo.pageTablesVisibility);
+    EXPECT_TRUE(storageInfo.multiStorage);
+    EXPECT_EQ(storageInfo.colouringGranularity, MemoryConstants::kiloByte * 128);
+    EXPECT_EQ(storageInfo.colouringPolicy, ColouringPolicy::ChunkSizeBased);
 }
 
 TEST_F(MultiDeviceStorageInfoTest, givenTwoPagesAllocationSizeWhenCreatingStorageInfoForBufferThenSingleMemoryBankIsOnAndPageTableClonningIsRequired) {

opencl/test/unit_test/test_files/igdrcl.config

@@ -190,6 +190,7 @@ OverridePreemptionSurfaceSizeInMb = -1
 OverrideLeastOccupiedBank = -1
 UseAsyncDrmExec = -1
 EnableMultiStorageResources = -1
+MultiStorageGranularity = -1
 PrintExecutionBuffer = 0
 PrintBOsForSubmit = 0
 EnableCrossDeviceAccess = -1

shared/source/debug_settings/debug_variables_base.inl

@@ -94,6 +94,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, PauseOnEnqueue, -1, "-1: default, -2: always, x:
 DECLARE_DEBUG_VARIABLE(int32_t, PauseOnBlitCopy, -1, "-1: default, -2: always, x: pause on blit enqueue number x and ask for user confirmation before and after execution, counted from 0. Note that single blit enqueue may have multiple copy instructions")
 DECLARE_DEBUG_VARIABLE(int32_t, PauseOnGpuMode, -1, "-1: default (before and after), 0: before only, 1: after only")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableMultiStorageResources, -1, "-1: default, 0: Disable, 1: Enable")
+DECLARE_DEBUG_VARIABLE(int32_t, MultiStorageGranularity, -1, "Forces chunk size based resource colouring with given granularity: -1 - disabled (default, subdevice count based colouring), >0 - enabled chunk size based colouring with granularity in kb (should be multiple of 64kb page size, because of alignment restrictions)")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitBlitterMaxWidth, -1, "-1: default, >=0: Max width")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitBlitterMaxHeight, -1, "-1: default, >=0: Max height")
 DECLARE_DEBUG_VARIABLE(int32_t, PostBlitCommand, -1, "-1: default, 0: MI_ARB_CHECK, 1: MI_FLUSH, 2: Nothing")

shared/source/helpers/common_types.h

@@ -26,6 +26,11 @@ enum class DebugPauseState : uint32_t {
     terminate
 };
 
+enum class ColouringPolicy : uint32_t {
+    ChunkSizeBased,
+    DeviceCountBased
+};
+
 class TagTypeBase {
 };
 

shared/source/memory_manager/allocation_properties.h

@@ -39,6 +39,8 @@ struct AllocationProperties {
     GraphicsAllocation::AllocationType allocationType = GraphicsAllocation::AllocationType::UNKNOWN;
     ImageInfo *imgInfo = nullptr;
     bool multiStorageResource = false;
+    ColouringPolicy colouringPolicy = ColouringPolicy::DeviceCountBased;
+    size_t colouringGranularity = MemoryConstants::pageSize64k;
     DeviceBitfield subDevicesBitfield{};
     uint64_t gpuAddress = 0;
     OsContext *osContext = nullptr;

shared/source/memory_manager/definitions/storage_info.cpp

@@ -111,13 +111,25 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
         break;
     case GraphicsAllocation::AllocationType::BUFFER:
     case GraphicsAllocation::AllocationType::BUFFER_COMPRESSED:
-    case GraphicsAllocation::AllocationType::SVM_GPU:
+    case GraphicsAllocation::AllocationType::SVM_GPU: {
+        auto colouringPolicy = properties.colouringPolicy;
+        auto granularity = properties.colouringGranularity;
+
+        if (DebugManager.flags.MultiStorageGranularity.get() != -1) {
+            colouringPolicy = ColouringPolicy::ChunkSizeBased;
+            granularity = DebugManager.flags.MultiStorageGranularity.get() * MemoryConstants::kiloByte;
+        }
+
+        DEBUG_BREAK_IF(colouringPolicy == ColouringPolicy::DeviceCountBased && granularity != MemoryConstants::pageSize64k);
+
         if (this->supportsMultiStorageResources &&
             properties.multiStorageResource &&
-            properties.size >= deviceCount * MemoryConstants::pageSize64k &&
+            properties.size >= deviceCount * granularity &&
             properties.subDevicesBitfield.count() != 1u) {
             storageInfo.memoryBanks = allTilesValue;
             storageInfo.multiStorage = true;
+            storageInfo.colouringPolicy = colouringPolicy;
+            storageInfo.colouringGranularity = granularity;
             if (DebugManager.flags.OverrideMultiStoragePlacement.get() != -1) {
                 storageInfo.memoryBanks = DebugManager.flags.OverrideMultiStoragePlacement.get();
             }
@@ -130,6 +142,7 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
         }
         storageInfo.localOnlyRequired = true;
         break;
+    }
     case GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY:
         storageInfo.memoryBanks = allTilesValue;
         break;

shared/source/memory_manager/definitions/storage_info.h

@@ -7,7 +7,9 @@
 
 #pragma once
 #include "shared/source/helpers/app_resource_defines.h"
+#include "shared/source/helpers/basic_math.h"
 #include "shared/source/helpers/common_types.h"
+#include "shared/source/helpers/constants.h"
 
 #include <cstdint>
 
@@ -23,11 +25,14 @@ struct StorageInfo {
     bool cloningOfPageTables = true;
     bool tileInstanced = false;
     bool multiStorage = false;
+    ColouringPolicy colouringPolicy = ColouringPolicy::DeviceCountBased;
+    size_t colouringGranularity = MemoryConstants::pageSize64k;
     bool readOnlyMultiStorage = false;
     bool cpuVisibleSegment = false;
     bool isLockable = false;
     bool localOnlyRequired = false;
     char resourceTag[AppResourceDefines::maxStrLen + 1] = "";
     uint32_t getMemoryBanks() const { return static_cast<uint32_t>(memoryBanks.to_ulong()); }
+    uint32_t getTotalBanksCnt() const { return Math::log2(getMemoryBanks()) + 1; }
 };
 } // namespace NEO

shared/source/memory_manager/graphics_allocation.h

@@ -268,6 +268,9 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
     void setGmm(Gmm *gmm, uint32_t handleId) {
         gmms[handleId] = gmm;
     }
+    void resizeGmms(uint32_t size) {
+        gmms.resize(size);
+    }
 
     uint32_t getNumGmms() const {
         return static_cast<uint32_t>(gmms.size());

shared/source/os_interface/linux/drm_allocation.h

@@ -63,6 +63,10 @@ class DrmAllocation : public GraphicsAllocation {
         return bufferObjects[handleIndex];
     }
 
+    void resizeBufferObjects(uint32_t size) {
+        this->bufferObjects.resize(size);
+    }
+
     uint64_t peekInternalHandle(MemoryManager *memoryManager) override;
 
     bool setCacheRegion(Drm *drm, CacheRegion regionIndex);

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -1120,8 +1120,16 @@ void DrmMemoryManager::unlockResourceInLocalMemoryImpl(BufferObject *bo) {
 }
 
 void createColouredGmms(GmmClientContext *clientContext, DrmAllocation &allocation, const StorageInfo &storageInfo, bool compression) {
-    auto remainingSize = alignUp(allocation.getUnderlyingBufferSize(), MemoryConstants::pageSize64k);
+    DEBUG_BREAK_IF(storageInfo.colouringPolicy == ColouringPolicy::DeviceCountBased && storageInfo.colouringGranularity != MemoryConstants::pageSize64k);
+
+    auto remainingSize = alignUp(allocation.getUnderlyingBufferSize(), storageInfo.colouringGranularity);
     auto handles = storageInfo.getNumBanks();
+    auto banksCnt = storageInfo.getTotalBanksCnt();
+
+    if (storageInfo.colouringPolicy == ColouringPolicy::ChunkSizeBased) {
+        handles = static_cast<uint32_t>(remainingSize / storageInfo.colouringGranularity);
+        allocation.resizeGmms(handles);
+    }
     /* This logic is to colour resource as equally as possible.
     Divide size by number of devices and align result up to 64kb page, then subtract it from whole size and allocate it on the first tile. First tile has it's chunk.
     In the following iteration divide rest of a size by remaining devices and again subtract it.
@@ -1136,10 +1144,10 @@ void createColouredGmms(GmmClientContext *clientContext, DrmAllocation &allocati
 
     It was tested and doesn't require any debug*/
     for (auto handleId = 0u; handleId < handles; handleId++) {
-        auto currentSize = alignUp(remainingSize / (handles - handleId), MemoryConstants::pageSize64k);
+        auto currentSize = alignUp(remainingSize / (handles - handleId), storageInfo.colouringGranularity);
         remainingSize -= currentSize;
         StorageInfo limitedStorageInfo = storageInfo;
-        limitedStorageInfo.memoryBanks &= 1u << handleId;
+        limitedStorageInfo.memoryBanks &= (1u << (handleId % banksCnt));
         auto gmm = new Gmm(clientContext,
                            nullptr,
                            currentSize,
@@ -1324,11 +1332,25 @@ BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(Drm *drm,
 }
 
 bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation, uint64_t gpuAddress, size_t maxOsContextCount) {
-    std::array<std::unique_ptr<BufferObject>, EngineLimits::maxHandleCount> bos{};
+    BufferObjects bos{};
     auto &storageInfo = allocation->storageInfo;
     auto boAddress = gpuAddress;
     auto currentBank = 0u;
-    for (auto handleId = 0u; handleId < storageInfo.getNumBanks(); handleId++, currentBank++) {
+    auto iterationOffset = 0u;
+    auto banksCnt = storageInfo.getTotalBanksCnt();
+
+    auto handles = storageInfo.getNumBanks();
+    if (storageInfo.colouringPolicy == ColouringPolicy::ChunkSizeBased) {
+        handles = allocation->getNumGmms();
+        allocation->resizeBufferObjects(handles);
+        bos.resize(handles);
+    }
+
+    for (auto handleId = 0u; handleId < handles; handleId++, currentBank++) {
+        if (currentBank == banksCnt) {
+            currentBank = 0;
+            iterationOffset += banksCnt;
+        }
         uint32_t memoryBanks = static_cast<uint32_t>(storageInfo.memoryBanks.to_ulong());
         if (storageInfo.getNumBanks() > 1) {
             //check if we have this bank, if not move to next one
@@ -1339,18 +1361,15 @@ bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation,
             memoryBanks &= 1u << currentBank;
         }
         auto boSize = alignUp(allocation->getGmm(handleId)->gmmResourceInfo->getSizeAllocation(), MemoryConstants::pageSize64k);
-        bos[handleId] = std::unique_ptr<BufferObject>(createBufferObjectInMemoryRegion(drm, boAddress, boSize, memoryBanks, maxOsContextCount));
+        bos[handleId] = createBufferObjectInMemoryRegion(drm, boAddress, boSize, memoryBanks, maxOsContextCount);
         if (nullptr == bos[handleId]) {
             return false;
         }
-        allocation->getBufferObjectToModify(currentBank) = bos[handleId].get();
+        allocation->getBufferObjectToModify(currentBank + iterationOffset) = bos[handleId];
         if (storageInfo.multiStorage) {
             boAddress += boSize;
         }
     }
-    for (auto &bo : bos) {
-        bo.release();
-    }
     return true;
 }