Extend PhysicalAddressAllocator with page size and alignement

- this allows for reserving 64k pages or bigger with specified alignement
if required

Change-Id: I256d6c0d9e7fee0e2bac5f4ab5e4fd49ea9d8d50

Author: HoppeMateusz

runtime/memory_manager/memory_constants.h

@@ -8,6 +8,7 @@
 #pragma once
 #include <cstdint>
 #include <cstddef>
+#include <limits>
 
 namespace MemoryConstants {
 static const uint64_t zoneHigh = ~(uint64_t)0xFFFFFFFF;
@@ -30,6 +31,8 @@ static const uint64_t max64BitAppAddress = ((1ULL << 47) - 1);
 static const uint32_t sizeOf4GBinPageEntities = (MemoryConstants::gigaByte * 4 - MemoryConstants::pageSize) / MemoryConstants::pageSize;
 static const uint64_t max32BitAddress = ((1ULL << 32) - 1);
 static const uint64_t max48BitAddress = ((1ULL << 48) - 1);
+static const uintptr_t page4kEntryMask = std::numeric_limits<uintptr_t>::max() & ~MemoryConstants::pageMask;
+static const uintptr_t page64kEntryMask = std::numeric_limits<uintptr_t>::max() & ~MemoryConstants::page64kMask;
 } // namespace MemoryConstants
 
 const bool is32bit = (sizeof(void *) == 4) ? true : false;

runtime/memory_manager/page_table.cpp

@@ -18,18 +18,17 @@ uintptr_t PTE::map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memor
     size_t indexEnd = ((vm + size - 1) >> shift) & mask;
     uintptr_t res = -1;
     bool updateEntryBits = entryBits != PageTableEntry::nonValidBits;
-    uint64_t newEntryBits = entryBits & 0xfff;
-    auto entriesMask = std::numeric_limits<uintptr_t>::max() & ~MemoryConstants::pageMask;
+    uint64_t newEntryBits = entryBits & MemoryConstants::pageMask;
     newEntryBits |= 0x1;
 
     for (size_t index = indexStart; index <= indexEnd; index++) {
         if (entries[index] == 0x0) {
-            uint64_t tmp = allocator->reservePage(memoryBank);
+            uint64_t tmp = allocator->reserve4kPage(memoryBank);
             entries[index] = reinterpret_cast<void *>(tmp | newEntryBits);
         } else if (updateEntryBits) {
-            entries[index] = reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(entries[index]) & entriesMask) | newEntryBits);
+            entries[index] = reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(entries[index]) & MemoryConstants::page4kEntryMask) | newEntryBits);
         }
-        res = std::min(reinterpret_cast<uintptr_t>(entries[index]) & entriesMask, res);
+        res = std::min(reinterpret_cast<uintptr_t>(entries[index]) & MemoryConstants::page4kEntryMask, res);
     }
     return (res & ~newEntryBits) + (vm & (pageSize - 1));
 }
@@ -43,21 +42,20 @@ void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits,
     uint64_t res = -1;
     uintptr_t rem = vm & (pageSize - 1);
     bool updateEntryBits = entryBits != PageTableEntry::nonValidBits;
-    uint64_t newEntryBits = entryBits & 0xfff;
-    auto entriesMask = std::numeric_limits<uintptr_t>::max() & ~MemoryConstants::pageMask;
+    uint64_t newEntryBits = entryBits & MemoryConstants::pageMask;
     newEntryBits |= 0x1;
 
     for (size_t index = indexStart; index <= indexEnd; index++) {
         if (entries[index] == 0x0) {
-            uint64_t tmp = allocator->reservePage(memoryBank);
+            uint64_t tmp = allocator->reserve4kPage(memoryBank);
             entries[index] = reinterpret_cast<void *>(tmp | newEntryBits);
         } else if (updateEntryBits) {
-            entries[index] = reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(entries[index]) & entriesMask) | newEntryBits);
+            entries[index] = reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(entries[index]) & MemoryConstants::page4kEntryMask) | newEntryBits);
         }
-        res = reinterpret_cast<uintptr_t>(entries[index]) & entriesMask;
+        res = reinterpret_cast<uintptr_t>(entries[index]) & MemoryConstants::page4kEntryMask;
 
         size_t lSize = std::min(pageSize - rem, size);
-        pageWalker((res & ~0x1) + rem, lSize, offset, reinterpret_cast<uintptr_t>(entries[index]) & 0xfffu);
+        pageWalker((res & ~0x1) + rem, lSize, offset, reinterpret_cast<uintptr_t>(entries[index]) & MemoryConstants::pageMask);
 
         size -= lSize;
         offset += lSize;

runtime/memory_manager/physical_address_allocator.h

@@ -6,10 +6,12 @@
  */
 
 #pragma once
+#include "runtime/helpers/aligned_memory.h"
 #include "runtime/helpers/debug_helpers.h"
 #include "runtime/memory_manager/memory_banks.h"
 #include "runtime/memory_manager/memory_constants.h"
 #include <atomic>
+#include <mutex>
 
 namespace OCLRT {
 
@@ -21,13 +23,28 @@ class PhysicalAddressAllocator {
 
     virtual ~PhysicalAddressAllocator() = default;
 
-    virtual uint64_t reservePage(uint32_t memoryBank) {
+    uint64_t reserve4kPage(uint32_t memoryBank) {
+        return reservePage(memoryBank, MemoryConstants::pageSize, MemoryConstants::pageSize);
+    }
+
+    uint64_t reserve64kPage(uint32_t memoryBank) {
+        return reservePage(memoryBank, MemoryConstants::pageSize64k, MemoryConstants::pageSize64k);
+    }
+
+    virtual uint64_t reservePage(uint32_t memoryBank, size_t pageSize, size_t alignement) {
         UNRECOVERABLE_IF(memoryBank != MemoryBanks::MainBank);
-        return mainAllocator.fetch_add(MemoryConstants::pageSize);
+
+        std::unique_lock<std::mutex> lock(pageReserveMutex);
+
+        auto currentAddress = mainAllocator.load();
+        auto alignmentSize = alignUp(currentAddress, alignement) - currentAddress;
+        mainAllocator += alignmentSize;
+        return mainAllocator.fetch_add(pageSize);
     }
 
   protected:
     std::atomic<uint64_t> mainAllocator;
+    std::mutex pageReserveMutex;
     const uint64_t initialPageAddress = 0x1000;
 };
 

runtime/os_interface/windows/hw_info_config.cpp

@@ -9,11 +9,12 @@
 #include "runtime/gen_common/hw_cmds.h"
 #include "runtime/helpers/hw_info.h"
 #include "runtime/helpers/hw_helper.h"
-#include "instrumentation.h"
 #include "runtime/memory_manager/memory_constants.h"
 #include "runtime/os_interface/hw_info_config.h"
 #include "runtime/os_interface/debug_settings_manager.h"
 
+#include "instrumentation.h"
+
 namespace OCLRT {
 
 HwInfoConfig *hwInfoConfigFactory[IGFX_MAX_PRODUCT] = {};

unit_tests/memory_manager/physical_address_allocator_tests.cpp

@@ -14,17 +14,65 @@ using namespace OCLRT;
 
 TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingFirstPageThenNonZeroAddressIsReturned) {
     MockPhysicalAddressAllocator allocator;
-    auto physAddress = allocator.reservePage(MemoryBanks::MainBank);
+    auto physAddress = allocator.reserve4kPage(MemoryBanks::MainBank);
     EXPECT_NE(0u, physAddress);
 }
 
-TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingConsecutivePagesThenReturnedAddressesAreDifferent) {
+TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingConsecutive4kPagesThenReturnedAddressesAreDifferentAndAligned) {
     MockPhysicalAddressAllocator allocator;
 
-    auto physAddress = allocator.reservePage(MemoryBanks::MainBank);
+    auto physAddress = allocator.reserve4kPage(MemoryBanks::MainBank);
     EXPECT_NE(0u, physAddress);
-    auto physAddress1 = allocator.reservePage(MemoryBanks::MainBank);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::pageMask);
+
+    auto physAddress1 = allocator.reserve4kPage(MemoryBanks::MainBank);
+    EXPECT_NE(physAddress, physAddress1);
+    EXPECT_EQ(0u, physAddress1 & MemoryConstants::pageMask);
+
+    auto physAddress2 = allocator.reserve4kPage(MemoryBanks::MainBank);
+    EXPECT_NE(physAddress1, physAddress2);
+    EXPECT_EQ(0u, physAddress2 & MemoryConstants::pageMask);
+}
+
+TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingFirst64kPageThen64kAlignedIsReturned) {
+    MockPhysicalAddressAllocator allocator;
+    auto physAddress = allocator.reserve64kPage(MemoryBanks::MainBank);
+    EXPECT_NE(0u, physAddress);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::page64kMask);
+}
+
+TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingConsecutive64kPagesThenReturnedAddressesAreDifferentAndAligned) {
+    MockPhysicalAddressAllocator allocator;
+
+    auto physAddress = allocator.reserve64kPage(MemoryBanks::MainBank);
+    EXPECT_NE(0u, physAddress);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::page64kMask);
+
+    auto physAddress1 = allocator.reserve64kPage(MemoryBanks::MainBank);
     EXPECT_NE(physAddress, physAddress1);
-    auto physAddress2 = allocator.reservePage(MemoryBanks::MainBank);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::page64kMask);
+
+    auto physAddress2 = allocator.reserve64kPage(MemoryBanks::MainBank);
     EXPECT_NE(physAddress1, physAddress2);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::page64kMask);
+}
+
+TEST(PhysicalAddressAllocator, givenPhysicalAddressesAllocatorWhenReservingInterleaving4kPagesAnd64kPagesThenReturnedAddressesAreCorrectlyAligned) {
+    MockPhysicalAddressAllocator allocator;
+
+    auto physAddress = allocator.reserve4kPage(MemoryBanks::MainBank);
+    EXPECT_NE(0u, physAddress);
+    EXPECT_EQ(0u, physAddress & MemoryConstants::pageMask);
+
+    auto physAddress1 = allocator.reserve64kPage(MemoryBanks::MainBank);
+    EXPECT_NE(physAddress, physAddress1);
+    EXPECT_EQ(0u, physAddress1 & MemoryConstants::page64kMask);
+
+    auto physAddress2 = allocator.reserve4kPage(MemoryBanks::MainBank);
+    EXPECT_NE(physAddress1, physAddress2);
+    EXPECT_EQ(0u, physAddress2 & MemoryConstants::pageMask);
+
+    auto physAddress3 = allocator.reserve64kPage(MemoryBanks::MainBank);
+    EXPECT_NE(physAddress, physAddress1);
+    EXPECT_EQ(0u, physAddress3 & MemoryConstants::page64kMask);
 }

unit_tests/os_interface/windows/ult_dxgi_factory.cpp

@@ -1,27 +1,12 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (C) 2017-2018 Intel Corporation
  *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
+ * SPDX-License-Identifier: MIT
  *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "unit_tests/os_interface/windows/ult_dxgi_factory.h"
 #include "runtime/memory_manager/memory_constants.h"
+#include "unit_tests/os_interface/windows/ult_dxgi_factory.h"
 
 namespace OCLRT {