Add experimental command buffer

This code is an infrastructure for special debug purpose that allow measure
execution time of any hardware command.

Change-Id: Id12a7979d204734a0c4a6c4700e427b65ac2397f

Author: zzdanowicz

runtime/command_stream/CMakeLists.txt

@@ -36,6 +36,9 @@ set(RUNTIME_SRCS_COMMAND_STREAM
   ${CMAKE_CURRENT_SOURCE_DIR}/create_command_stream_impl.h
   ${CMAKE_CURRENT_SOURCE_DIR}/csr_definitions.h
   ${CMAKE_CURRENT_SOURCE_DIR}/device_command_stream.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/experimental_command_buffer.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/experimental_command_buffer.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/experimental_command_buffer.inl
   ${CMAKE_CURRENT_SOURCE_DIR}/linear_stream.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/linear_stream.h
   ${CMAKE_CURRENT_SOURCE_DIR}/submissions_aggregator.cpp

runtime/command_stream/command_stream_receiver.cpp

@@ -22,6 +22,7 @@
 
 #include "runtime/built_ins/built_ins.h"
 #include "runtime/command_stream/command_stream_receiver.h"
+#include "runtime/command_stream/experimental_command_buffer.h"
 #include "runtime/command_stream/preemption.h"
 #include "runtime/device/device.h"
 #include "runtime/gtpin/gtpin_notify.h"
@@ -203,6 +204,7 @@ void CommandStreamReceiver::cleanupResources() {
         commandStream.replaceGraphicsAllocation(nullptr);
         commandStream.replaceBuffer(nullptr, 0);
     }
+    experimentalCmdBuffer.reset(nullptr);
 }
 
 bool CommandStreamReceiver::waitForCompletionWithTimeout(bool enableTimeout, int64_t timeoutMicroseconds, uint32_t taskCountToWait) {
@@ -338,4 +340,8 @@ void CommandStreamReceiver::releaseIndirectHeap(IndirectHeap::Type heapType) {
     }
 }
 
+void CommandStreamReceiver::setExperimentalCmdBuffer(std::unique_ptr<ExperimentalCommandBuffer> &&cmdBuffer) {
+    experimentalCmdBuffer = std::move(cmdBuffer);
+}
+
 } // namespace OCLRT

runtime/command_stream/command_stream_receiver.h

@@ -37,11 +37,12 @@
 namespace OCLRT {
 class Device;
 class EventBuilder;
-class LinearStream;
+class ExperimentalCommandBuffer;
+class GraphicsAllocation;
 class IndirectHeap;
+class LinearStream;
 class MemoryManager;
 class OSInterface;
-class GraphicsAllocation;
 
 enum class DispatchMode {
     DeviceDefault = 0,          //default for given device
@@ -139,6 +140,7 @@ class CommandStreamReceiver {
     void releaseIndirectHeap(IndirectHeap::Type heapType);
 
     virtual enum CommandStreamReceiverType getType() = 0;
+    void setExperimentalCmdBuffer(std::unique_ptr<ExperimentalCommandBuffer> &&cmdBuffer);
 
   protected:
     void setDisableL3Cache(bool val) {
@@ -190,6 +192,7 @@ class CommandStreamReceiver {
     SamplerCacheFlushState samplerCacheFlushRequired = SamplerCacheFlushState::samplerCacheFlushNotRequired;
     IndirectHeap *indirectHeap[IndirectHeap::NUM_TYPES];
     std::unique_ptr<FlatBatchBufferHelper> flatBatchBufferHelper;
+    std::unique_ptr<ExperimentalCommandBuffer> experimentalCmdBuffer;
 };
 
 typedef CommandStreamReceiver *(*CommandStreamReceiverCreateFunc)(const HardwareInfo &hwInfoIn, bool withAubDump);

runtime/command_stream/command_stream_receiver_hw.h

@@ -55,7 +55,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     int getRequiredPipeControlSize() const;
 
     static void addBatchBufferEnd(LinearStream &commandStream, void **patchLocation);
-    void addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress);
+    void addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress, bool secondary);
     static void alignToCacheLine(LinearStream &commandStream);
 
     size_t getRequiredCmdStreamSize(const DispatchFlags &dispatchFlags);

runtime/command_stream/command_stream_receiver_hw.inl

@@ -21,6 +21,7 @@
  */
 
 #include "runtime/command_stream/command_stream_receiver_hw.h"
+#include "runtime/command_stream/experimental_command_buffer.h"
 #include "runtime/command_stream/linear_stream.h"
 #include "runtime/device/device.h"
 #include "runtime/gtpin/gtpin_notify.h"
@@ -62,10 +63,13 @@ inline void CommandStreamReceiverHw<GfxFamily>::addBatchBufferEnd(LinearStream &
 }
 
 template <typename GfxFamily>
-inline void CommandStreamReceiverHw<GfxFamily>::addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress) {
+inline void CommandStreamReceiverHw<GfxFamily>::addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress, bool secondary) {
     *commandBufferMemory = GfxFamily::cmdInitBatchBufferStart;
     commandBufferMemory->setBatchBufferStartAddressGraphicsaddress472(startAddress);
     commandBufferMemory->setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT);
+    if (secondary) {
+        commandBufferMemory->setSecondLevelBatchBuffer(MI_BATCH_BUFFER_START::SECOND_LEVEL_BATCH_BUFFER_SECOND_LEVEL_BATCH);
+    }
     if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
         flatBatchBufferHelper->registerBatchBufferStartAddress(reinterpret_cast<uint64_t>(commandBufferMemory), startAddress);
     }
@@ -328,6 +332,12 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
             }
         }
     }
+
+    if (experimentalCmdBuffer.get() != nullptr) {
+        size_t startingOffset = experimentalCmdBuffer->programExperimentalCommandBuffer<GfxFamily>();
+        experimentalCmdBuffer->injectBufferStart<GfxFamily>(commandStreamCSR, startingOffset);
+    }
+
     // Add a PC if we have a dependency on a previous walker to avoid concurrency issues.
     if (taskLevel > this->taskLevel) {
         addPipeControl(commandStreamCSR, false);
@@ -358,6 +368,10 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
         makeResident(*BuiltIns::getInstance().getSipKernel(sipType, *device).getSipAllocation());
     }
 
+    if (experimentalCmdBuffer.get() != nullptr) {
+        experimentalCmdBuffer->makeResidentAllocations();
+    }
+
     // If the CSR has work in its CS, flush it before the task
     bool submitTask = commandStreamStartTask != commandStreamTask.getUsed();
     bool submitCSR = commandStreamStartCSR != commandStreamCSR.getUsed();
@@ -377,7 +391,7 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
             chainedBatchBuffer = commandStreamTask.getGraphicsAllocation();
             // Add MI_BATCH_BUFFER_START to chain from CSR -> Task
             auto pBBS = reinterpret_cast<MI_BATCH_BUFFER_START *>(commandStreamCSR.getSpace(sizeof(MI_BATCH_BUFFER_START)));
-            addBatchBufferStart(pBBS, ptrOffset(commandStreamTask.getGraphicsAllocation()->getGpuAddress(), commandStreamStartTask));
+            addBatchBufferStart(pBBS, ptrOffset(commandStreamTask.getGraphicsAllocation()->getGpuAddress(), commandStreamStartTask), false);
             if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
                 flatBatchBufferHelper->registerCommandChunk(commandStreamTask.getGraphicsAllocation()->getGpuAddress(),
                                                             reinterpret_cast<uint64_t>(commandStreamTask.getCpuBase()),
@@ -510,7 +524,7 @@ inline void CommandStreamReceiverHw<GfxFamily>::flushBatchedSubmissions() {
                 flushStampUpdateHelper.insert(nextCommandBuffer->flushStamp->getStampReference());
                 auto nextCommandBufferAddress = nextCommandBuffer->batchBuffer.commandBufferAllocation->getUnderlyingBuffer();
                 auto offsetedCommandBuffer = (uint64_t)ptrOffset(nextCommandBufferAddress, nextCommandBuffer->batchBuffer.startOffset);
-                addBatchBufferStart((MI_BATCH_BUFFER_START *)currentBBendLocation, offsetedCommandBuffer);
+                addBatchBufferStart((MI_BATCH_BUFFER_START *)currentBBendLocation, offsetedCommandBuffer, false);
                 if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
                     flatBatchBufferHelper->registerCommandChunk(nextCommandBuffer->batchBuffer, sizeof(MI_BATCH_BUFFER_START));
                 }
@@ -610,6 +624,9 @@ size_t CommandStreamReceiverHw<GfxFamily>::getRequiredCmdStreamSize(const Dispat
             size += sizeof(typename GfxFamily::PIPE_CONTROL);
         }
     }
+    if (experimentalCmdBuffer.get() != nullptr) {
+        size += experimentalCmdBuffer->getRequiredInjectionSize<GfxFamily>();
+    }
     return size;
 }
 

runtime/command_stream/experimental_command_buffer.cpp

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "runtime/command_stream/command_stream_receiver.h"
+#include "runtime/command_stream/experimental_command_buffer.h"
+#include "runtime/command_stream/linear_stream.h"
+#include "runtime/device/device.h"
+#include "runtime/memory_manager/memory_constants.h"
+#include "runtime/memory_manager/memory_manager.h"
+#include <cstring>
+#include <type_traits>
+
+namespace OCLRT {
+
+ExperimentalCommandBuffer::ExperimentalCommandBuffer(CommandStreamReceiver *csr) : commandStreamReceiver(csr),
+                                                                                   currentStream(nullptr),
+                                                                                   timestampsOffset(0),
+                                                                                   experimentalAllocationOffset(0),
+                                                                                   defaultPrint(true) {
+    timestamps = csr->getMemoryManager()->allocateGraphicsMemory(MemoryConstants::pageSize);
+    memset(timestamps->getUnderlyingBuffer(), 0, timestamps->getUnderlyingBufferSize());
+    experimentalAllocation = csr->getMemoryManager()->allocateGraphicsMemory(MemoryConstants::pageSize);
+    memset(experimentalAllocation->getUnderlyingBuffer(), 0, experimentalAllocation->getUnderlyingBufferSize());
+    timerResolution = commandStreamReceiver->getMemoryManager()->device->getDeviceInfo().profilingTimerResolution;
+}
+
+ExperimentalCommandBuffer::~ExperimentalCommandBuffer() {
+    auto timestamp = static_cast<uint64_t *>(timestamps->getUnderlyingBuffer());
+    for (uint32_t i = 0; i < timestampsOffset / (2 * sizeof(uint64_t)); i++) {
+        auto stop = static_cast<uint64_t>(*(timestamp + 1) * timerResolution);
+        auto start = static_cast<uint64_t>(*timestamp * timerResolution);
+        auto delta = stop - start;
+        printDebugString(defaultPrint, stdout, "#%u: delta %llu start %llu stop %llu\n", i, delta, start, stop);
+        timestamp += 2;
+    }
+    MemoryManager *memManager = commandStreamReceiver->getMemoryManager();
+    if (memManager) {
+        memManager->freeGraphicsMemory(timestamps);
+        memManager->freeGraphicsMemory(experimentalAllocation);
+
+        if (currentStream.get()) {
+            memManager->storeAllocation(std::unique_ptr<GraphicsAllocation>(currentStream->getGraphicsAllocation()), REUSABLE_ALLOCATION);
+            currentStream->replaceGraphicsAllocation(nullptr);
+        }
+    }
+}
+
+void ExperimentalCommandBuffer::getCS(size_t minRequiredSize) {
+    if (!currentStream) {
+        currentStream.reset(new LinearStream(nullptr));
+    }
+    minRequiredSize += CSRequirements::minCommandQueueCommandStreamSize;
+    if (currentStream->getAvailableSpace() < minRequiredSize) {
+        MemoryManager *memManager = commandStreamReceiver->getMemoryManager();
+        // If not, allocate a new block. allocate full pages
+        minRequiredSize = alignUp(minRequiredSize, MemoryConstants::pageSize);
+
+        auto requiredSize = minRequiredSize + CSRequirements::csOverfetchSize;
+
+        GraphicsAllocation *allocation = memManager->obtainReusableAllocation(requiredSize, false).release();
+        if (!allocation) {
+            allocation = memManager->allocateGraphicsMemory(requiredSize);
+        }
+        allocation->setAllocationType(GraphicsAllocation::AllocationType::LINEAR_STREAM);
+        // Deallocate the old block, if not null
+        auto oldAllocation = currentStream->getGraphicsAllocation();
+        if (oldAllocation) {
+            memManager->storeAllocation(std::unique_ptr<GraphicsAllocation>(oldAllocation), REUSABLE_ALLOCATION);
+        }
+        currentStream->replaceBuffer(allocation->getUnderlyingBuffer(), minRequiredSize - CSRequirements::minCommandQueueCommandStreamSize);
+        currentStream->replaceGraphicsAllocation(allocation);
+    }
+}
+
+void ExperimentalCommandBuffer::makeResidentAllocations() {
+    commandStreamReceiver->makeResident(*currentStream->getGraphicsAllocation());
+    commandStreamReceiver->makeResident(*timestamps);
+    commandStreamReceiver->makeResident(*experimentalAllocation);
+}
+
+} // namespace OCLRT

runtime/command_stream/experimental_command_buffer.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+#include <memory>
+#include <stdint.h>
+
+namespace OCLRT {
+
+class CommandStreamReceiver;
+class GraphicsAllocation;
+class LinearStream;
+class MemoryManager;
+
+class ExperimentalCommandBuffer {
+  public:
+    virtual ~ExperimentalCommandBuffer();
+    ExperimentalCommandBuffer(CommandStreamReceiver *csr);
+
+    template <typename GfxFamily>
+    void injectBufferStart(LinearStream &parentStream, size_t cmdBufferOffset);
+
+    template <typename GfxFamily>
+    size_t getRequiredInjectionSize() noexcept;
+
+    template <typename GfxFamily>
+    size_t programExperimentalCommandBuffer();
+
+    void makeResidentAllocations();
+
+  protected:
+    template <typename GfxFamily>
+    size_t getTotalExperimentalSize() noexcept;
+
+    void getCS(size_t minRequiredSize);
+
+    template <typename GfxFamily>
+    void addTimeStampPipeControl();
+
+    template <typename GfxFamily>
+    size_t getTimeStampPipeControlSize() noexcept;
+
+    template <typename GfxFamily>
+    void addExperimentalCommands();
+
+    template <typename GfxFamily>
+    size_t getExperimentalCommandsSize() noexcept;
+
+    CommandStreamReceiver *commandStreamReceiver;
+    std::unique_ptr<LinearStream> currentStream;
+
+    GraphicsAllocation *timestamps;
+    uint32_t timestampsOffset;
+
+    GraphicsAllocation *experimentalAllocation;
+    uint32_t experimentalAllocationOffset;
+
+    bool defaultPrint;
+    double timerResolution;
+};
+
+} // namespace OCLRT