Improve loop sinking heuristic

Improve applicability of loop sinking optimization in cases when the
loop is located in a non-kernel function and relax the threshold

Author: admitric

IGC/Compiler/CISACodeGen/CodeSinking.cpp

@@ -235,8 +235,11 @@ namespace IGC {
         bool Changed = false;
         for (auto &L : LI->getLoopsInPreorder())
         {
-            if (IGC_IS_FLAG_ENABLED(ForceLoopSink) || needLoopSink(L))
-                Changed |= loopSink(L);
+            LoopSinkMode SinkMode = IGC_IS_FLAG_ENABLED(ForceLoopSink) ? LoopSinkMode::FullSink : LoopSinkMode::NoSink;
+            if (SinkMode == LoopSinkMode::NoSink)
+                SinkMode = needLoopSink(L);
+            if (SinkMode != LoopSinkMode::NoSink)
+                Changed |= loopSink(L, SinkMode);
         }
         return Changed;
     }
@@ -1261,13 +1264,13 @@ namespace IGC {
         return changed;
     }
 
-    bool CodeSinking::needLoopSink(Loop *L)
+    LoopSinkMode CodeSinking::needLoopSink(Loop *L)
     {
         BasicBlock *Preheader = L->getLoopPreheader();
         if (!Preheader)
-            return false;
+            return LoopSinkMode::NoSink;
         if (!RPE)
-            return false;
+            return LoopSinkMode::NoSink;
 
         Function *F = Preheader->getParent();
         uint GRFThresholdDelta = IGC_GET_FLAG_VALUE(LoopSinkThresholdDelta);
@@ -1293,13 +1296,58 @@ namespace IGC {
         // Estimate max pressure in the loop
         uint MaxLoopPressure = RPE->getMaxRegCountForLoop(*L, SIMD);
 
-        // loop sinking is needed if the loop's pressure is higher than number of GRFs by threshold
-        // and preheader's potential to reduce the delta is good enough
-        return ((MaxLoopPressure > NGRF + GRFThresholdDelta) &&
-                (PreheaderDefsSizeInRegs > (MaxLoopPressure - NGRF) * LOOPSINK_PREHEADER_IMPACT_THRESHOLD));
+        auto isSinkCriteriaMet = [&](uint MaxLoopPressure)
+        {
+            // loop sinking is needed if the loop's pressure is higher than number of GRFs by threshold
+            // and preheader's potential to reduce the delta is good enough
+            return ((MaxLoopPressure > NGRF + GRFThresholdDelta) &&
+                    (PreheaderDefsSizeInRegs > (MaxLoopPressure - NGRF) * LOOPSINK_PREHEADER_IMPACT_THRESHOLD));
+        };
+
+        // Check if the real regpressure may be higher, because the function is not a kernel
+        // and is called not as stack call
+        bool MayHaveCallerRegpressure = !isEntryFunc(CTX->getMetaDataUtils(), F) &&
+            !F->hasFnAttribute("visaStackCall");
+
+        // If loop pressure induced in this function only is enough for the criteria, return immediately
+        // Otherwise try to add pressure from callsite as the real regpressure may be higher
+        if (isSinkCriteriaMet(MaxLoopPressure))
+            return MayHaveCallerRegpressure ?
+                LoopSinkMode::FullSink :
+                LoopSinkMode::SinkWhileRegpressureIsHigh;
+
+        // Look just for the direct callers while there is no cross-procedure regpressure analysis
+        if (MayHaveCallerRegpressure)
+        {
+            for (llvm::User *U : F->users()) {
+                CallInst *CI = llvm::dyn_cast<llvm::CallInst>(U);
+                if (CI == nullptr)
+                    continue;
+
+                if (CI->getCalledFunction() != F)
+                    continue;
+
+                BasicBlock *CallerBB = CI->getParent();
+                Function *CallerFunction = CallerBB->getParent();
+
+                // Augment liveness analysis with blocks from the caller function
+                bool RPEHasCallerFunctionInfo = RPE->getOutSet().count(&CallerFunction->getEntryBlock());
+                if (!RPEHasCallerFunctionInfo)
+                    RPE->livenessAnalysis(*CallerFunction);
+
+                // Get call site regpressure
+                auto CallerBBPressureMap = RPE->getPressureMapForBB(*CallerBB, SIMD);
+                uint CallSitePressure = RPE->bytesToRegisters(CallerBBPressureMap[CI]);
+
+                if (isSinkCriteriaMet(MaxLoopPressure + CallSitePressure))
+                    return LoopSinkMode::FullSink;
+            }
+        }
+
+        return LoopSinkMode::NoSink;
     }
 
-    bool CodeSinking::loopSink(Loop *L)
+    bool CodeSinking::loopSink(Loop *L, LoopSinkMode Mode)
     {
         // Sink loop invariants back into the loop body if register
         // pressure can be reduced.
@@ -1380,7 +1428,7 @@ namespace IGC {
                 RPE->rerunLivenessAnalysis(*F);
                 uint MaxLoopPressure = RPE->getMaxRegCountForLoop(*L, SIMD);
                 if (MaxLoopPressure < (NGRF - IGC_GET_FLAG_VALUE(LoopSinkRegpressureMargin))
-                        && IGC_IS_FLAG_DISABLED(ForceLoopSink))
+                        && (Mode == LoopSinkMode::SinkWhileRegpressureIsHigh))
                 {
                     if (IGC_IS_FLAG_ENABLED(EnableLoadChainLoopSink) && !LoadChains.empty())
                     {

IGC/Compiler/CISACodeGen/CodeSinking.hpp

@@ -22,6 +22,7 @@ See LICENSE.TXT for details.
 #include "common/LLVMWarningsPop.hpp"
 
 namespace IGC {
+    typedef enum { NoSink=0, SinkWhileRegpressureIsHigh, FullSink } LoopSinkMode;
 
     class CodeSinking : public llvm::FunctionPass {
         llvm::DominatorTree* DT;
@@ -58,6 +59,7 @@ namespace IGC {
             AU.addPreservedID(WIAnalysis::ID);
         }
     private:
+
         bool treeSink(llvm::Function& F);
         bool ProcessBlock(llvm::BasicBlock& blk);
         bool SinkInstruction(llvm::Instruction* I,
@@ -140,14 +142,14 @@ namespace IGC {
         // try loop sinking in the function if needed
         bool loopSink(llvm::Function& F);
         // move LI back into loop
-        bool loopSink(llvm::Loop* LoopWithPressure);
+        bool loopSink(llvm::Loop* LoopWithPressure, LoopSinkMode Mode);
         // pre-condition to sink an instruction into a loop
         bool isLoopSinkCandidate(llvm::Instruction* I, llvm::Loop* L, bool AllowLoadSinking);
         bool loopSinkInstructions(
             llvm::SmallVector<llvm::Instruction*, 64>& SinkCandidates,
             llvm::SmallPtrSet<llvm::Instruction*, 32>& LoadChains,
             llvm::Loop* L);
-        bool needLoopSink(llvm::Loop* L);
+        LoopSinkMode needLoopSink(llvm::Loop* L);
 
         // Move referencing DbgValueInst intrinsics calls after defining instructions
         void ProcessDbgValueInst(llvm::BasicBlock& blk);

IGC/Compiler/CISACodeGen/IGCLivenessAnalysis.h

@@ -161,6 +161,7 @@ class IGCLivenessAnalysis : public llvm::FunctionPass {
         releaseMemory();
         livenessAnalysis(F);
     }
+    void livenessAnalysis(llvm::Function &F);
 
   private:
     void dumpRegPressure(llvm::Function &F, unsigned int SIMD);
@@ -188,7 +189,6 @@ class IGCLivenessAnalysis : public llvm::FunctionPass {
     void combineOut(llvm::BasicBlock *BB, ValueSet *Set);
     void addToPhiSet(llvm::PHINode *Phi, PhiSet *InPhiSet);
     void processBlock(llvm::BasicBlock *BB, ValueSet &Set, PhiSet *PhiSet);
-    void livenessAnalysis(llvm::Function &F);
 
     virtual bool runOnFunction(llvm::Function &F) override;
     virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {

IGC/common/igc_flags.h

@@ -140,7 +140,7 @@ DECLARE_IGC_REGKEY(DWORD, CodeSinkingMinSize,           32,  "Don't sink if the
 DECLARE_IGC_REGKEY(DWORD, CodeSinkingLoadSchedulingInstr, 20,  "Instructions number to step to schedule loads in advance before the load use to cover latency. 1 to insert it immediately before use", false)
 DECLARE_IGC_REGKEY(DWORD, LoopSinkMinSaveUniform,       6,  "If loop sink can have save more scalar (uniform) values than this Minimum, do it; otherwise, skip", false)
 DECLARE_IGC_REGKEY(DWORD, LoopSinkMinSave,              1,  "If loop sink can have save more 32-bit values than this Minimum, do it; otherwise, skip", false)
-DECLARE_IGC_REGKEY(DWORD, LoopSinkThresholdDelta,       50,  "Do loop sink If the estimated register pressure is higher than this + #avaialble registers", false)
+DECLARE_IGC_REGKEY(DWORD, LoopSinkThresholdDelta,       30,  "Do loop sink If the estimated register pressure is higher than this + #avaialble registers", false)
 DECLARE_IGC_REGKEY(bool, EnableLoopHoistConstant,       false, "Enables pass to check for specific loop patterns where variables are constant across all but the last iteration, and hoist them out of the loop.", false)
 DECLARE_IGC_REGKEY(bool, DisableCodeHoisting,           false, "Setting this to 1/true adds a compiler switch to disable code-hoisting", false)
 DECLARE_IGC_REGKEY(bool, EnableDeSSA,                   true,  "Setting this to 0/false adds a compiler switch to disable De-SSA", false)