Initial implementation of Timestamp Packet write

Change-Id: Ic498bcf9795f54fbb5fb5a8d07ed17fa70dc4f1a
Signed-off-by: Dunajski, Bartosz <[email protected]>

Author: BartoszDunajski

runtime/command_queue/command_queue.h

@@ -331,7 +331,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
     Context &getContext() { return *context; }
     Context *getContextPtr() { return context; }
 
-    LinearStream &getCS(size_t minRequiredSize = 1024u);
+    MOCKABLE_VIRTUAL LinearStream &getCS(size_t minRequiredSize);
     IndirectHeap &getIndirectHeap(IndirectHeap::Type heapType,
                                   size_t minRequiredSize);
 

runtime/command_queue/enqueue_common.h

@@ -269,6 +269,7 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
             &blockedCommandsData,
             hwTimeStamps,
             hwPerfCounter,
+            nullptr,
             preemption,
             blockQueue,
             commandType);

runtime/command_queue/gpgpu_walker.h

@@ -33,6 +33,7 @@
 #include "runtime/helpers/dispatch_info.h"
 #include "runtime/helpers/kernel_commands.h"
 #include "runtime/helpers/task_information.h"
+#include "runtime/helpers/timestamp_packet.h"
 #include "runtime/indirect_heap/indirect_heap.h"
 #include "runtime/kernel/kernel.h"
 #include "runtime/program/kernel_info.h"
@@ -124,6 +125,8 @@ inline cl_uint computeDimensions(const size_t workItems[3]) {
 template <typename GfxFamily>
 class GpgpuWalkerHelper {
   public:
+    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
+
     static void addAluReadModifyWriteRegister(
         LinearStream *pCommandStream,
         uint32_t aluRegister,
@@ -199,10 +202,17 @@ class GpgpuWalkerHelper {
         KernelOperation **blockedCommandsData,
         HwTimeStamps *hwTimeStamps,
         OCLRT::HwPerfCounter *hwPerfCounter,
+        TimestampPacket *timestampPacket,
         PreemptionMode preemptionMode,
         bool blockQueue,
         uint32_t commandType = 0);
 
+    static void setupTimestampPacket(
+        LinearStream *cmdStream,
+        WALKER_HANDLE walkerHandle,
+        TimestampPacket *timestampPacket,
+        TimestampPacket::WriteOperationType writeOperationType);
+
     static void dispatchScheduler(
         CommandQueue &commandQueue,
         DeviceQueueHw<GfxFamily> &devQueueHw,
@@ -214,6 +224,7 @@ class GpgpuWalkerHelper {
 
 template <typename GfxFamily>
 struct EnqueueOperation {
+    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
     static size_t getTotalSizeRequiredCS(bool reserveProfilingCmdsSpace, bool reservePerfCounters, CommandQueue &commandQueue, const MultiDispatchInfo &multiDispatchInfo);
     static size_t getSizeRequiredCS(uint32_t cmdType, bool reserveProfilingCmdsSpace, bool reservePerfCounters, CommandQueue &commandQueue, const Kernel *pKernel);
 
@@ -239,6 +250,9 @@ LinearStream &getCommandStream(CommandQueue &commandQueue, bool reserveProfiling
         SchedulerKernel &scheduler = commandQueue.getDevice().getExecutionEnvironment()->getBuiltIns()->getSchedulerKernel(parentKernel->getContext());
         expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredCS(eventType, reserveProfilingCmdsSpace, reservePerfCounterCmdsSpace, commandQueue, &scheduler);
     }
+    if (DebugManager.flags.EnableTimestampPacket.get()) {
+        expectedSizeCS += 2 * sizeof(typename GfxFamily::PIPE_CONTROL);
+    }
     return commandQueue.getCS(expectedSizeCS);
 }
 

runtime/command_queue/gpgpu_walker.inl

@@ -104,7 +104,6 @@ void GpgpuWalkerHelper<GfxFamily>::addAluReadModifyWriteRegister(
     pCmd4->setDestinationRegisterAddress(aluRegister);
 
     // Add PIPE_CONTROL to flush caches
-    typedef typename GfxFamily::PIPE_CONTROL PIPE_CONTROL;
     auto pCmd5 = reinterpret_cast<PIPE_CONTROL *>(pCommandStream->getSpace(sizeof(PIPE_CONTROL)));
     *pCmd5 = PIPE_CONTROL::sInit();
     pCmd5->setCommandStreamerStallEnable(true);
@@ -157,7 +156,6 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchProfilingCommandsStart(
     HwTimeStamps &hwTimeStamps,
     OCLRT::LinearStream *commandStream) {
     using MI_STORE_REGISTER_MEM = typename GfxFamily::MI_STORE_REGISTER_MEM;
-    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
 
     // PIPE_CONTROL for global timestamp
     uint64_t TimeStampAddress = reinterpret_cast<uint64_t>(&(hwTimeStamps.GlobalStartTS));
@@ -185,7 +183,6 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchProfilingCommandsEnd(
     OCLRT::LinearStream *commandStream) {
 
     using MI_STORE_REGISTER_MEM = typename GfxFamily::MI_STORE_REGISTER_MEM;
-    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
 
     // PIPE_CONTROL for global timestamp
     auto pPipeControlCmd = (PIPE_CONTROL *)commandStream->getSpace(sizeof(PIPE_CONTROL));
@@ -340,7 +337,6 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchPerfCountersCommandsStart(
     OCLRT::HwPerfCounter &hwPerfCounter,
     OCLRT::LinearStream *commandStream) {
 
-    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
     using MI_STORE_REGISTER_MEM = typename GfxFamily::MI_STORE_REGISTER_MEM;
     using MI_REPORT_PERF_COUNT = typename GfxFamily::MI_REPORT_PERF_COUNT;
 
@@ -387,7 +383,6 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchPerfCountersCommandsEnd(
     OCLRT::HwPerfCounter &hwPerfCounter,
     OCLRT::LinearStream *commandStream) {
 
-    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
     using MI_STORE_REGISTER_MEM = typename GfxFamily::MI_STORE_REGISTER_MEM;
     using MI_REPORT_PERF_COUNT = typename GfxFamily::MI_REPORT_PERF_COUNT;
 
@@ -440,6 +435,7 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchWalker(
     KernelOperation **blockedCommandsData,
     HwTimeStamps *hwTimeStamps,
     OCLRT::HwPerfCounter *hwPerfCounter,
+    TimestampPacket *timestampPacket,
     PreemptionMode preemptionMode,
     bool blockQueue,
     uint32_t commandType) {
@@ -519,6 +515,7 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchWalker(
 
     DEBUG_BREAK_IF(offsetInterfaceDescriptorTable % 64 != 0);
 
+    size_t currentDispatchIndex = 0;
     for (auto &dispatchInfo : multiDispatchInfo) {
         auto &kernel = *dispatchInfo.getKernel();
 
@@ -613,11 +610,20 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchWalker(
         // Implement enabling special WA DisableLSQCROPERFforOCL if needed
         GpgpuWalkerHelper<GfxFamily>::applyWADisableLSQCROPERFforOCL(commandStream, kernel, true);
 
+        bool setupTimestampPacket = (DebugManager.flags.EnableTimestampPacket.get()) && (currentDispatchIndex == numDispatches - 1);
+        if (setupTimestampPacket) {
+            GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(commandStream, nullptr, timestampPacket, TimestampPacket::WriteOperationType::Start);
+        }
+
         // Program the walker.  Invokes execution so all state should already be programmed
         typedef typename GfxFamily::GPGPU_WALKER GPGPU_WALKER;
         auto pGpGpuWalkerCmd = (GPGPU_WALKER *)commandStream->getSpace(sizeof(GPGPU_WALKER));
         *pGpGpuWalkerCmd = GfxFamily::cmdInitGpgpuWalker;
 
+        if (setupTimestampPacket) {
+            GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(commandStream, pGpGpuWalkerCmd, timestampPacket, TimestampPacket::WriteOperationType::End);
+        }
+
         size_t globalOffsets[3] = {offset.x, offset.y, offset.z};
         size_t startWorkGroups[3] = {swgs.x, swgs.y, swgs.z};
         size_t numWorkGroups[3] = {nwgs.x, nwgs.y, nwgs.z};
@@ -645,6 +651,7 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchWalker(
         GpgpuWalkerHelper<GfxFamily>::applyWADisableLSQCROPERFforOCL(commandStream, kernel, false);
 
         PreemptionHelper::applyPreemptionWaCmdsEnd<GfxFamily>(commandStream, commandQueue.getDevice());
+        currentDispatchIndex++;
     }
 
     // If hwTimeStamps is passed (not nullptr), then we know that profiling is enabled
@@ -656,6 +663,24 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchWalker(
     }
 }
 
+template <typename GfxFamily>
+void GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(
+    LinearStream *cmdStream,
+    WALKER_HANDLE walkerHandle,
+    TimestampPacket *timestampPacket,
+    TimestampPacket::WriteOperationType writeOperationType) {
+
+    uint64_t address = timestampPacket->pickAddressForPipeControlWrite(writeOperationType);
+
+    auto pipeControlCmd = cmdStream->getSpaceForCmd<PIPE_CONTROL>();
+    *pipeControlCmd = PIPE_CONTROL::sInit();
+    pipeControlCmd->setCommandStreamerStallEnable(true);
+    pipeControlCmd->setPostSyncOperation(PIPE_CONTROL::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA);
+    pipeControlCmd->setAddress(static_cast<uint32_t>(address & 0x0000FFFFFFFFULL));
+    pipeControlCmd->setAddressHigh(static_cast<uint32_t>(address >> 32));
+    pipeControlCmd->setImmediateData(0);
+}
+
 template <typename GfxFamily>
 void GpgpuWalkerHelper<GfxFamily>::dispatchScheduler(
     CommandQueue &commandQueue,
@@ -667,7 +692,6 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchScheduler(
 
     using INTERFACE_DESCRIPTOR_DATA = typename GfxFamily::INTERFACE_DESCRIPTOR_DATA;
     using GPGPU_WALKER = typename GfxFamily::GPGPU_WALKER;
-    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
     using MI_BATCH_BUFFER_START = typename GfxFamily::MI_BATCH_BUFFER_START;
 
     OCLRT::LinearStream *commandStream = nullptr;
@@ -802,14 +826,14 @@ size_t GpgpuWalkerHelper<GfxFamily>::getSizeForWADisableLSQCROPERFforOCL(const K
 template <typename GfxFamily>
 size_t EnqueueOperation<GfxFamily>::getTotalSizeRequiredCS(bool reserveProfilingCmdsSpace, bool reservePerfCounters, CommandQueue &commandQueue, const MultiDispatchInfo &multiDispatchInfo) {
     size_t size = KernelCommandsHelper<GfxFamily>::getSizeRequiredCS() +
-                  sizeof(typename GfxFamily::PIPE_CONTROL) * (KernelCommandsHelper<GfxFamily>::isPipeControlWArequired() ? 2 : 1);
+                  sizeof(PIPE_CONTROL) * (KernelCommandsHelper<GfxFamily>::isPipeControlWArequired() ? 2 : 1);
     if (reserveProfilingCmdsSpace) {
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL) + 4 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
+        size += 2 * sizeof(PIPE_CONTROL) + 4 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
     }
     if (reservePerfCounters) {
         //start cmds
         //P_C: flush CS & TimeStamp BEGIN
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL);
+        size += 2 * sizeof(PIPE_CONTROL);
         //SRM NOOPID & Frequency
         size += 2 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
         //gp registers
@@ -821,7 +845,7 @@ size_t EnqueueOperation<GfxFamily>::getTotalSizeRequiredCS(bool reserveProfiling
 
         //end cmds
         //P_C: flush CS & TimeStamp END;
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL);
+        size += 2 * sizeof(PIPE_CONTROL);
         //OA buffer (status head, tail)
         size += 3 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
         //report perf count
@@ -858,15 +882,15 @@ size_t EnqueueOperation<GfxFamily>::getSizeRequiredCS(uint32_t cmdType, bool res
 template <typename GfxFamily>
 size_t EnqueueOperation<GfxFamily>::getSizeRequiredCSKernel(bool reserveProfilingCmdsSpace, bool reservePerfCounters, CommandQueue &commandQueue, const Kernel *pKernel) {
     size_t size = sizeof(typename GfxFamily::GPGPU_WALKER) + KernelCommandsHelper<GfxFamily>::getSizeRequiredCS() +
-                  sizeof(typename GfxFamily::PIPE_CONTROL) * (KernelCommandsHelper<GfxFamily>::isPipeControlWArequired() ? 2 : 1);
+                  sizeof(PIPE_CONTROL) * (KernelCommandsHelper<GfxFamily>::isPipeControlWArequired() ? 2 : 1);
     size += PreemptionHelper::getPreemptionWaCsSize<GfxFamily>(commandQueue.getDevice());
     if (reserveProfilingCmdsSpace) {
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL) + 2 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
+        size += 2 * sizeof(PIPE_CONTROL) + 2 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
     }
     if (reservePerfCounters) {
         //start cmds
         //P_C: flush CS & TimeStamp BEGIN
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL);
+        size += 2 * sizeof(PIPE_CONTROL);
         //SRM NOOPID & Frequency
         size += 2 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
         //gp registers
@@ -878,7 +902,7 @@ size_t EnqueueOperation<GfxFamily>::getSizeRequiredCSKernel(bool reserveProfilin
 
         //end cmds
         //P_C: flush CS & TimeStamp END;
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL);
+        size += 2 * sizeof(PIPE_CONTROL);
         //OA buffer (status head, tail)
         size += 3 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
         //report perf count
@@ -899,7 +923,7 @@ template <typename GfxFamily>
 size_t EnqueueOperation<GfxFamily>::getSizeRequiredCSNonKernel(bool reserveProfilingCmdsSpace, bool reservePerfCounters, CommandQueue &commandQueue) {
     size_t size = 0;
     if (reserveProfilingCmdsSpace) {
-        size += 2 * sizeof(typename GfxFamily::PIPE_CONTROL) + 4 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
+        size += 2 * sizeof(PIPE_CONTROL) + 4 * sizeof(typename GfxFamily::MI_STORE_REGISTER_MEM);
     }
     return size;
 }

runtime/helpers/CMakeLists.txt

@@ -85,6 +85,7 @@ set(RUNTIME_SRCS_HELPERS_BASE
   ${CMAKE_CURRENT_SOURCE_DIR}/string_helpers.h
   ${CMAKE_CURRENT_SOURCE_DIR}/surface_formats.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/surface_formats.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/timestamp_packet.h
   ${CMAKE_CURRENT_SOURCE_DIR}/task_information.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/task_information.h
   ${CMAKE_CURRENT_SOURCE_DIR}/uint16_avx2.h

runtime/helpers/timestamp_packet.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <array>
+
+namespace OCLRT {
+class TimestampPacket {
+  public:
+    enum class DataIndex : uint32_t {
+        ContextStart,
+        GlobalStart,
+        ContextEnd,
+        GlobalEnd,
+        Max
+    };
+
+    enum class WriteOperationType : uint32_t {
+        Start,
+        End
+    };
+
+    const uint32_t *pickDataPtr() const { return &(data[0]); }
+
+    uint64_t pickAddressForPipeControlWrite(WriteOperationType operationType) const {
+        auto index = WriteOperationType::Start == operationType
+                         ? static_cast<uint32_t>(DataIndex::ContextStart)
+                         : static_cast<uint32_t>(DataIndex::ContextEnd);
+
+        return reinterpret_cast<uint64_t>(&data[index]);
+    }
+
+    uint32_t pickDataValue(DataIndex index) const { return data[static_cast<uint32_t>(index)]; }
+
+  protected:
+    std::array<uint32_t, static_cast<uint32_t>(DataIndex::Max)> data = {{1, 1, 1, 1}};
+};
+} // namespace OCLRT

runtime/memory_manager/memory_manager.cpp

@@ -30,15 +30,15 @@
 #include "runtime/helpers/aligned_memory.h"
 #include "runtime/helpers/basic_math.h"
 #include "runtime/helpers/options.h"
+#include "runtime/helpers/timestamp_packet.h"
 #include "runtime/memory_manager/deferred_deleter.h"
 #include "runtime/utilities/stackvec.h"
 #include "runtime/utilities/tag_allocator.h"
 
 #include <algorithm>
 
 namespace OCLRT {
-constexpr size_t ProfilingTagCount = 512;
-constexpr size_t PerfCounterTagCount = 512;
+constexpr size_t TagCount = 512;
 
 struct ReusableAllocationRequirements {
     size_t requiredMinimalSize;
@@ -226,11 +226,16 @@ void MemoryManager::applyCommonCleanup() {
     if (this->paddingAllocation) {
         this->freeGraphicsMemory(this->paddingAllocation);
     }
-    if (profilingTimeStampAllocator)
+    if (profilingTimeStampAllocator) {
         profilingTimeStampAllocator->cleanUpResources();
-
-    if (perfCounterAllocator)
+    }
+    if (perfCounterAllocator) {
         perfCounterAllocator->cleanUpResources();
+    }
+
+    if (timestampPacketAllocator) {
+        timestampPacketAllocator->cleanUpResources();
+    }
 
     cleanAllocationList(-1, TEMPORARY_ALLOCATION);
     cleanAllocationList(-1, REUSABLE_ALLOCATION);
@@ -263,18 +268,25 @@ void MemoryManager::freeAllocationsList(uint32_t waitTaskCount, AllocationsList
 
 TagAllocator<HwTimeStamps> *MemoryManager::getEventTsAllocator() {
     if (profilingTimeStampAllocator.get() == nullptr) {
-        profilingTimeStampAllocator.reset(new TagAllocator<HwTimeStamps>(this, ProfilingTagCount, MemoryConstants::cacheLineSize));
+        profilingTimeStampAllocator = std::make_unique<TagAllocator<HwTimeStamps>>(this, TagCount, MemoryConstants::cacheLineSize);
     }
     return profilingTimeStampAllocator.get();
 }
 
 TagAllocator<HwPerfCounter> *MemoryManager::getEventPerfCountAllocator() {
     if (perfCounterAllocator.get() == nullptr) {
-        perfCounterAllocator.reset(new TagAllocator<HwPerfCounter>(this, PerfCounterTagCount, MemoryConstants::cacheLineSize));
+        perfCounterAllocator = std::make_unique<TagAllocator<HwPerfCounter>>(this, TagCount, MemoryConstants::cacheLineSize);
     }
     return perfCounterAllocator.get();
 }
 
+TagAllocator<TimestampPacket> *MemoryManager::getTimestampPacketAllocator() {
+    if (timestampPacketAllocator.get() == nullptr) {
+        timestampPacketAllocator = std::make_unique<TagAllocator<TimestampPacket>>(this, TagCount, MemoryConstants::cacheLineSize);
+    }
+    return timestampPacketAllocator.get();
+}
+
 void MemoryManager::pushAllocationForResidency(GraphicsAllocation *gfxAllocation) {
     residencyAllocations.push_back(gfxAllocation);
 }