Fix Read/WriteBuffer for unaligned offsets

Change-Id: I08d33e80243f41174f4629c8a611e286629d2e10

Author: ivvenevt

runtime/built_ins/built_ins.cpp

@@ -243,14 +243,14 @@ class BuiltInOp<HWFamily, EBuiltInOps::CopyBufferToBuffer> : public BuiltinDispa
         } else if (operationParams.srcMemObj) {
             kernelSplit1DBuilder.setArg(0, operationParams.srcMemObj);
         } else {
-            kernelSplit1DBuilder.setArgSvm(0, operationParams.size.x, operationParams.srcPtr, nullptr, CL_MEM_READ_ONLY);
+            kernelSplit1DBuilder.setArgSvm(0, operationParams.size.x + operationParams.srcOffset.x, operationParams.srcPtr, nullptr, CL_MEM_READ_ONLY);
         }
         if (operationParams.dstSvmAlloc) {
             kernelSplit1DBuilder.setArgSvmAlloc(1, operationParams.dstPtr, operationParams.dstSvmAlloc);
         } else if (operationParams.dstMemObj) {
             kernelSplit1DBuilder.setArg(1, operationParams.dstMemObj);
         } else {
-            kernelSplit1DBuilder.setArgSvm(1, operationParams.size.x, operationParams.dstPtr);
+            kernelSplit1DBuilder.setArgSvm(1, operationParams.size.x + operationParams.dstOffset.x, operationParams.dstPtr);
         }
 
         // Set-up srcOffset

runtime/command_queue/enqueue_read_buffer.h

@@ -88,21 +88,30 @@ cl_int CommandQueueHw<GfxFamily>::enqueueReadBuffer(
     BuiltInOwnershipWrapper builtInLock(builder, this->context);
 
     void *dstPtr = ptr;
+    void *alignedDstPtr = dstPtr;
+    size_t dstPtrOffset = 0;
+
+    if (!isAligned<4>(dstPtr)) {
+        alignedDstPtr = alignDown(dstPtr, 4);
+        dstPtrOffset = ptrDiff(dstPtr, alignedDstPtr);
+    }
 
     MemObjSurface bufferSurf(buffer);
-    HostPtrSurface hostPtrSurf(dstPtr, size);
+    HostPtrSurface hostPtrSurf(alignedDstPtr, size + dstPtrOffset);
     Surface *surfaces[] = {&bufferSurf, &hostPtrSurf};
 
     if (size != 0) {
         bool status = getCommandStreamReceiver().createAllocationForHostSurface(hostPtrSurf, getDevice(), true);
         if (!status) {
             return CL_OUT_OF_RESOURCES;
         }
-        dstPtr = reinterpret_cast<void *>(hostPtrSurf.getAllocation()->getGpuAddressToPatch());
+
+        hostPtrSurf.getAllocation()->allocationOffset += dstPtrOffset;
     }
 
     BuiltinDispatchInfoBuilder::BuiltinOpParams dc;
-    dc.dstPtr = dstPtr;
+    dc.dstPtr = alignedDstPtr;
+    dc.dstOffset = {dstPtrOffset, 0, 0};
     dc.srcMemObj = buffer;
     dc.srcOffset = {offset, 0, 0};
     dc.size = {size, 0, 0};

runtime/command_queue/enqueue_write_buffer.h

@@ -89,8 +89,15 @@ cl_int CommandQueueHw<GfxFamily>::enqueueWriteBuffer(
     BuiltInOwnershipWrapper builtInLock(builder, this->context);
 
     void *srcPtr = const_cast<void *>(ptr);
+    void *alignedSrcPtr = srcPtr;
+    size_t srcPtrOffset = 0;
 
-    HostPtrSurface hostPtrSurf(srcPtr, size, true);
+    if (!isAligned<4>(srcPtr)) {
+        alignedSrcPtr = alignDown(srcPtr, 4);
+        srcPtrOffset = ptrDiff(srcPtr, alignedSrcPtr);
+    }
+
+    HostPtrSurface hostPtrSurf(alignedSrcPtr, size + srcPtrOffset, true);
     MemObjSurface bufferSurf(buffer);
     Surface *surfaces[] = {&bufferSurf, &hostPtrSurf};
 
@@ -99,11 +106,13 @@ cl_int CommandQueueHw<GfxFamily>::enqueueWriteBuffer(
         if (!status) {
             return CL_OUT_OF_RESOURCES;
         }
-        srcPtr = reinterpret_cast<void *>(hostPtrSurf.getAllocation()->getGpuAddressToPatch());
+
+        hostPtrSurf.getAllocation()->allocationOffset += srcPtrOffset;
     }
 
     BuiltinDispatchInfoBuilder::BuiltinOpParams dc;
-    dc.srcPtr = srcPtr;
+    dc.srcPtr = alignedSrcPtr;
+    dc.srcOffset = {srcPtrOffset, 0, 0};
     dc.dstMemObj = buffer;
     dc.dstOffset = {offset, 0, 0};
     dc.size = {size, 0, 0};

runtime/command_stream/aub_command_stream_receiver_hw.inl

@@ -623,7 +623,7 @@ void AUBCommandStreamReceiverHw<GfxFamily>::writeMemory(uint64_t gpuAddress, voi
 
 template <typename GfxFamily>
 bool AUBCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxAllocation) {
-    auto cpuAddress = gfxAllocation.getUnderlyingBuffer();
+    auto cpuAddress = ptrOffset(gfxAllocation.getUnderlyingBuffer(), static_cast<size_t>(gfxAllocation.allocationOffset));
     auto gpuAddress = GmmHelper::decanonize(gfxAllocation.getGpuAddress());
     auto size = gfxAllocation.getUnderlyingBufferSize();
     if (gfxAllocation.gmm && gfxAllocation.gmm->isRenderCompressed) {

unit_tests/aub_tests/command_queue/enqueue_read_buffer_aub_tests.cpp

@@ -143,3 +143,66 @@ HWTEST_F(AUBReadBuffer, reserveCanonicalGpuAddress) {
 
     AUBCommandStreamFixture::expectMemory<FamilyType>(dstGpuAddress, srcMemory, sizeof(dstMemory));
 }
+
+struct AUBReadBufferUnaligned
+    : public CommandEnqueueAUBFixture,
+      public ::testing::Test {
+
+    void SetUp() override {
+        CommandEnqueueAUBFixture::SetUp();
+    }
+
+    void TearDown() override {
+        CommandEnqueueAUBFixture::TearDown();
+    }
+
+    template <typename FamilyType>
+    void testReadBufferUnaligned(size_t offset, size_t size) {
+        MockContext context(&pCmdQ->getDevice());
+
+        char srcMemory[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        const auto bufferSize = sizeof(srcMemory);
+        char dstMemory[bufferSize] = {0};
+
+        auto retVal = CL_INVALID_VALUE;
+
+        auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+            &context,
+            CL_MEM_USE_HOST_PTR,
+            bufferSize,
+            srcMemory,
+            retVal));
+        ASSERT_NE(nullptr, buffer);
+
+        buffer->forceDisallowCPUCopy = true;
+
+        // Map destination memory to GPU
+        GraphicsAllocation *allocation = createResidentAllocationAndStoreItInCsr(dstMemory, bufferSize);
+        auto dstMemoryGPUPtr = reinterpret_cast<char *>(allocation->getGpuAddress());
+
+        // Do unaligned read
+        retVal = pCmdQ->enqueueReadBuffer(
+            buffer.get(),
+            CL_TRUE,
+            offset,
+            size,
+            ptrOffset(dstMemory, offset),
+            0,
+            nullptr,
+            nullptr);
+        EXPECT_EQ(CL_SUCCESS, retVal);
+
+        // Check the memory
+        AUBCommandStreamFixture::expectMemory<FamilyType>(ptrOffset(dstMemoryGPUPtr, offset), ptrOffset(srcMemory, offset), size);
+    }
+};
+
+HWTEST_F(AUBReadBufferUnaligned, all) {
+    const std::vector<size_t> offsets = {0, 1, 2, 3};
+    const std::vector<size_t> sizes = {4, 3, 2, 1};
+    for (auto offset : offsets) {
+        for (auto size : sizes) {
+            testReadBufferUnaligned<FamilyType>(offset, size);
+        }
+    }
+}

unit_tests/aub_tests/command_queue/enqueue_write_buffer_aub_tests.cpp

@@ -108,3 +108,63 @@ INSTANTIATE_TEST_CASE_P(AUBWriteBuffer_simple,
                             1 * sizeof(cl_float),
                             2 * sizeof(cl_float),
                             3 * sizeof(cl_float)));
+
+struct AUBWriteBufferUnaligned
+    : public CommandEnqueueAUBFixture,
+      public ::testing::Test {
+
+    void SetUp() override {
+        CommandEnqueueAUBFixture::SetUp();
+    }
+
+    void TearDown() override {
+        CommandEnqueueAUBFixture::TearDown();
+    }
+
+    template <typename FamilyType>
+    void testWriteBufferUnaligned(size_t offset, size_t size) {
+        MockContext context(&pCmdQ->getDevice());
+
+        char srcMemory[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        const auto bufferSize = sizeof(srcMemory);
+        char dstMemory[bufferSize] = {0};
+
+        auto retVal = CL_INVALID_VALUE;
+
+        auto buffer = std::unique_ptr<Buffer>(Buffer::create(
+            &context,
+            CL_MEM_USE_HOST_PTR,
+            bufferSize,
+            dstMemory,
+            retVal));
+        ASSERT_NE(nullptr, buffer);
+
+        buffer->forceDisallowCPUCopy = true;
+
+        // Do unaligned write
+        retVal = pCmdQ->enqueueWriteBuffer(
+            buffer.get(),
+            CL_TRUE,
+            offset,
+            size,
+            ptrOffset(srcMemory, offset),
+            0,
+            nullptr,
+            nullptr);
+        EXPECT_EQ(CL_SUCCESS, retVal);
+
+        // Check the memory
+        auto bufferGPUPtr = reinterpret_cast<char *>((buffer->getGraphicsAllocation()->getGpuAddress()));
+        AUBCommandStreamFixture::expectMemory<FamilyType>(ptrOffset(bufferGPUPtr, offset), ptrOffset(srcMemory, offset), size);
+    }
+};
+
+HWTEST_F(AUBWriteBufferUnaligned, all) {
+    const std::vector<size_t> offsets = {0, 1, 2, 3};
+    const std::vector<size_t> sizes = {4, 3, 2, 1};
+    for (auto offset : offsets) {
+        for (auto size : sizes) {
+            testWriteBufferUnaligned<FamilyType>(offset, size);
+        }
+    }
+}

unit_tests/command_queue/enqueue_read_buffer_tests.cpp

@@ -10,6 +10,7 @@
 #include "reg_configs_common.h"
 #include "runtime/helpers/cache_policy.h"
 #include "runtime/helpers/dispatch_info.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/command_queue/enqueue_fixture.h"
 #include "unit_tests/gen_common/gen_commands_common_validation.h"
 #include "unit_tests/command_queue/enqueue_read_buffer_fixture.h"
@@ -326,6 +327,30 @@ HWTEST_F(EnqueueReadBufferTypeTest, givenNotAlignedPointerAndAlignedSizeWhenRead
     EXPECT_FALSE(csr.disableL3Cache);
 }
 
+HWTEST_F(EnqueueReadBufferTypeTest, givenNotAlignedPointerAndAlignedSizeWhenReadBufferIsCalledThenHostGraphicsAllocationHasCorrectOffset) {
+    void *ptr = (void *)0x1039;
+
+    cl_int retVal = pCmdQ->enqueueReadBuffer(srcBuffer.get(),
+                                             CL_FALSE,
+                                             0,
+                                             MemoryConstants::cacheLineSize,
+                                             ptr,
+                                             0,
+                                             nullptr,
+                                             nullptr);
+
+    EXPECT_EQ(CL_SUCCESS, retVal);
+    auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
+
+    auto allocation = csr.getTemporaryAllocations().peekHead();
+    while (allocation && allocation->getUnderlyingBuffer() != alignDown(ptr, 4)) {
+        allocation = allocation->next;
+    }
+
+    ASSERT_NE(allocation, nullptr);
+    EXPECT_EQ((void *)allocation->getGpuAddressToPatch(), ptr);
+}
+
 HWTEST_F(EnqueueReadBufferTypeTest, givenOOQWithEnabledSupportCpuCopiesAndDstPtrEqualSrcPtrAndZeroCopyBufferWhenReadBufferIsExecutedThenTaskLevelNotIncreased) {
     DebugManagerStateRestore dbgRestore;
     DebugManager.flags.DoCpuCopyOnReadBuffer.set(true);

unit_tests/command_queue/enqueue_write_buffer_tests.cpp

@@ -10,6 +10,7 @@
 #include "runtime/built_ins/builtins_dispatch_builder.h"
 #include "reg_configs_common.h"
 #include "runtime/helpers/dispatch_info.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/command_queue/enqueue_fixture.h"
 #include "unit_tests/gen_common/gen_commands_common_validation.h"
 #include "unit_tests/helpers/debug_manager_state_restore.h"
@@ -433,3 +434,27 @@ HWTEST_F(NegativeFailAllocationTest, givenEnqueueWriteBufferWhenHostPtrAllocatio
 
     EXPECT_EQ(CL_OUT_OF_RESOURCES, retVal);
 }
+
+HWTEST_F(EnqueueWriteBufferTypeTest, givenNotAlignedPointerAndAlignedSizeWhenWriteBufferIsCalledThenHostGraphicsAllocationHasCorrectOffset) {
+    void *ptr = (void *)0x1039;
+
+    cl_int retVal = pCmdQ->enqueueWriteBuffer(srcBuffer.get(),
+                                              CL_FALSE,
+                                              0,
+                                              MemoryConstants::cacheLineSize,
+                                              ptr,
+                                              0,
+                                              nullptr,
+                                              nullptr);
+
+    EXPECT_EQ(CL_SUCCESS, retVal);
+    auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
+
+    auto allocation = csr.getTemporaryAllocations().peekHead();
+    while (allocation && allocation->getUnderlyingBuffer() != alignDown(ptr, 4)) {
+        allocation = allocation->next;
+    }
+
+    ASSERT_NE(allocation, nullptr);
+    EXPECT_EQ((void *)allocation->getGpuAddressToPatch(), ptr);
+}