Add uniformity check for CloneAddressArithmetic

This commit introduces new functionality and control flags for
CloneAddressArithmetic pass and adds a uniformity check for
rematerialization (disabled by default).

(cherry picked from commit 4707787)

Author: esukhov

IGC/Compiler/CISACodeGen/RematAddressArithmetic.cpp

@@ -51,6 +51,7 @@ class CloneAddressArithmetic : public FunctionPass {
 
 public:
     static char ID;
+    WIAnalysis *WI = nullptr;
 
     ~CloneAddressArithmetic() { Uses.clear(); }
     CloneAddressArithmetic() : FunctionPass(ID) {
@@ -62,22 +63,19 @@ class CloneAddressArithmetic : public FunctionPass {
     virtual void getAnalysisUsage(llvm::AnalysisUsage& AU) const override {
         AU.setPreservesCFG();
         AU.addRequired<IGCLivenessAnalysis>();
+        AU.addRequired<WIAnalysis>();
     }
 
     bool runOnFunction(Function&) override;
     void rematWholeChain(llvm::IntToPtrInst *I);
+    bool isRegPressureLow(Function &F);
     std::unordered_map<llvm::Value*, unsigned int> Uses;
 
 private:
     bool greedyRemat(Function &F);
 };
-
-
-
-
 } // end namespace
 
-
 FunctionPass* IGC::createCloneAddressArithmeticPass() {
     return new CloneAddressArithmetic();
 }
@@ -91,17 +89,50 @@ char CloneAddressArithmetic::ID = 0;
 namespace IGC {
 IGC_INITIALIZE_PASS_BEGIN(CloneAddressArithmetic, PASS_FLAG_2, PASS_DESC_2, PASS_CFG_ONLY_2, PASS_ANALYSIS_2)
 IGC_INITIALIZE_PASS_DEPENDENCY(IGCLivenessAnalysis)
+IGC_INITIALIZE_PASS_DEPENDENCY(WIAnalysis)
 IGC_INITIALIZE_PASS_END(CloneAddressArithmetic, PASS_FLAG_2, PASS_DESC_2, PASS_CFG_ONLY_2, PASS_ANALYSIS_2)
 }
 
+void putAddrSpaceCastClose(Function &F) {
+
+    std::vector<llvm::Instruction *> V;
+
+    for (BasicBlock &BB : F) {
+        for (auto &I : BB) {
+            if (llvm::isa<AddrSpaceCastInst>(I))
+                V.push_back(&I);
+        }
+    }
+
+    for(auto I : V) {
+
+        std::vector<llvm::Use *> VectorOfUses;
+        for(auto &U : I->uses()) { VectorOfUses.push_back(&U); }
+
+        for(auto U : VectorOfUses) {
+            auto User = U->getUser();
+            auto UserInst = llvm::dyn_cast<Instruction>(User);
+
+            if(llvm::isa<PHINode>(UserInst)) continue;
+
+            if(UserInst) {
+                auto Clone = I->clone();
+                Clone->setName(I->getName() + "_clonedAddrSpaceCast");
+                Clone->insertBefore(UserInst);
+                *U = Clone;
+            }
+        }
+    }
+}
 
 static bool isAddressArithmetic(Instruction* I)
 {
     if (isa<GetElementPtrInst>(I) ||
         isa<InsertElementInst>(I) ||
         isa<InsertValueInst>(I) ||
+        isa<BinaryOperator>(I) ||
         (isa<UnaryInstruction>(I) && !isa<LoadInst>(I)) ||
-        isa<BinaryOperator>(I))
+        (IGC_GET_FLAG_VALUE(RematAllowExtractElement) && isa<ExtractElementInst>(I)))
         return true;
 
     return false;
@@ -130,6 +161,7 @@ void CloneAddressArithmetic::rematWholeChain(llvm::IntToPtrInst *I) {
         bool NotPHI = !llvm::isa<llvm::PHINode>(Op);
         bool NotConstant = !llvm::isa<llvm::Constant>(Op);
         bool SameBB = IGC_IS_FLAG_ENABLED(RematSameBBScope) ? Op->getParent() == I->getParent() : true;
+        bool NotUniform = IGC_IS_FLAG_ENABLED(RematRespectUniformity) ? !WI->isUniform(Op) : true;
         bool AddressArithmetic = isAddressArithmetic(Op);
 
         // if operand has more uses than specified, we do not rematerialize it.
@@ -145,7 +177,7 @@ void CloneAddressArithmetic::rematWholeChain(llvm::IntToPtrInst *I) {
         // load r2
         bool NotTooManyUses = Uses[Op] < NumOfUsesLimit;
 
-        if (SameBB && NotConstant && NotPHI && NotTooManyUses && AddressArithmetic) {
+        if (SameBB && NotConstant && NotPHI && NotTooManyUses && AddressArithmetic && NotUniform) {
 
           BFSQ.push(Op);
           RematVector.push_back(Op);
@@ -183,80 +215,92 @@ void CloneAddressArithmetic::rematWholeChain(llvm::IntToPtrInst *I) {
   RematVector.clear();
 }
 
-bool CloneAddressArithmetic::greedyRemat(Function &F) {
+bool isSafelyRematerializable(Use& Use) {
 
-  bool Result = false;
+    auto LI = llvm::isa<LoadInst>(Use.getUser());
+    auto SI = llvm::isa<StoreInst>(Use.getUser());
+    auto BI = llvm::isa<BitCastInst>(Use.getUser());
+    auto SelI = llvm::isa<SelectInst>(Use.getUser());
+    auto CI = IGC_IS_FLAG_ENABLED(RematAddrSpaceCastToUse) ? llvm::isa<AddrSpaceCastInst>(Use.getUser()) : false;
 
-  auto RPE = &getAnalysis<IGCLivenessAnalysis>();
-  unsigned int SIMD = numLanes(RPE->bestGuessSIMDSize());
-  unsigned int PressureLimit = IGC_GET_FLAG_VALUE(RematRPELimit);
-  if(RPE->getMaxRegCountForFunction(F, SIMD) < PressureLimit)
-      return Result;
+    bool Result = LI || SI || BI || CI || SelI;
+    return Result;
+}
 
-  for (BasicBlock &BB : F) {
-    for (auto &I : BB) { Uses[&I] = I.getNumUses(); }
-  }
+bool CloneAddressArithmetic::isRegPressureLow(Function &F) {
 
-  llvm::SmallVector<llvm::IntToPtrInst *, 4> ToProcess;
+    auto RPE = &getAnalysis<IGCLivenessAnalysis>();
+    unsigned int SIMD = numLanes(RPE->bestGuessSIMDSize());
+    unsigned int PressureLimit = IGC_GET_FLAG_VALUE(RematRPELimit);
+    bool Result = RPE->getMaxRegCountForFunction(F, SIMD) < PressureLimit;
+    return Result;
+}
 
-  // go through block, collect all inttoptr instructions to do
-  // remat on them
-  for (BasicBlock &BB : F) {
-    // if block has less than required amount of LLVM IR instructions, skip it
-    const unsigned Limit = IGC_GET_FLAG_VALUE(RematBlockSize);
-    if (BB.getInstList().size() < Limit) continue;
+bool CloneAddressArithmetic::greedyRemat(Function &F) {
 
-    for (auto &I : BB) {
+    bool Result = false;
 
-      auto *CastedIntToPtrInst = llvm::dyn_cast<IntToPtrInst>(&I);
-      if (CastedIntToPtrInst) ToProcess.push_back(CastedIntToPtrInst);
-    }
-  }
+    if (isRegPressureLow(F))
+        return Result;
 
-  for (auto el : ToProcess) {
+    for (BasicBlock &BB : F) {
+        for (auto &I : BB) { Uses[&I] = I.getNumUses(); }
+    }
 
-    Value *V = el;
-    llvm::SmallVector<llvm::Use*, 4> VectorOfUses;
-    // collect all uses of particular intoptr inst
-    bool usedOnlyInLoadOrStore = true;
-    for (auto &use : V->uses()) {
+    // At times, addrspace casts end up far away from their direct users.
+    if(IGC_IS_FLAG_ENABLED(RematMoveAddrSpaceCast)) putAddrSpaceCastClose(F);
 
-      // check that this inttoptr instruction only used in load or stores
-      auto LI = llvm::dyn_cast<LoadInst>(use.getUser());
-      auto SI = llvm::dyn_cast<StoreInst>(use.getUser());
-      usedOnlyInLoadOrStore &= (LI != NULL) || (SI != NULL);
+    llvm::SmallVector<llvm::IntToPtrInst *, 4> ToProcess;
 
-      VectorOfUses.push_back(&use);
+    // go through block, collect all inttoptr instructions to do
+    // remat on them
+    for (BasicBlock &BB : F) {
+        for (auto &I : BB) {
+            auto *CastedIntToPtrInst = llvm::dyn_cast<IntToPtrInst>(&I);
+            if (CastedIntToPtrInst) ToProcess.push_back(CastedIntToPtrInst);
+        }
     }
 
-    if(!usedOnlyInLoadOrStore) continue;
+    for (auto el : ToProcess) {
+
+        Value *V = el;
+        llvm::SmallVector<llvm::Use*, 4> VectorOfUses;
+        // collect all uses of particular intoptr inst
+        bool ShouldBeRemated = true;
+        for (auto &U : V->uses()) {
+            ShouldBeRemated &= isSafelyRematerializable(U);
+            VectorOfUses.push_back(&U);
+        }
 
-    for (auto use : VectorOfUses) {
+        if(!ShouldBeRemated) continue;
 
-      // take use of inttoptr instruction, clone instruction,
-      // insert clone right before the use, swap use to clone, remat
-      auto User = use->getUser();
-      auto UserInst = llvm::dyn_cast<Instruction>(User);
+        for (auto use : VectorOfUses) {
 
-      if(UserInst) {
-        auto Clone = el->clone();
-        Clone->setName("cloned_" + el->getName());
-        Clone->insertBefore(UserInst);
-        *use = Clone;
-        rematWholeChain((llvm::IntToPtrInst *)Clone);
-        Result = true;
-      }
+            // Clone inttoptr instruction, insert clone right before the use,
+            // switch use to clone, remat
+            auto User = use->getUser();
+            auto UserInst = llvm::dyn_cast<Instruction>(User);
+
+            if(UserInst) {
+                auto Clone = el->clone();
+                Clone->setName("cloned_" + el->getName());
+                Clone->insertBefore(UserInst);
+                *use = Clone;
+                rematWholeChain((llvm::IntToPtrInst *)Clone);
+                Result = true;
+            }
+        }
     }
-  }
 
-  return Result;
+    return Result;
 }
 
 bool CloneAddressArithmetic::runOnFunction(Function& F)
 {
     if (skipFunction(F))
         return false;
 
+    WI = &getAnalysis<WIAnalysis>();
     bool Modified = false;
     Modified |= greedyRemat(F);
     return Modified;

IGC/Compiler/CISACodeGen/ShaderCodeGen.cpp

@@ -894,42 +894,40 @@ void AddLegalizationPasses(CodeGenContext& ctx, IGCPassManager& mpm, PSSignature
 
     // Run address remat after GVN as it may hoist address calculations and
     // create PHI nodes with addresses.
+    if (IGC_IS_FLAG_ENABLED(RematEnable) || (ctx.m_retryManager.AllowCloneAddressArithmetic() && ctx.type == ShaderType::OPENCL_SHADER)) {
+
+        if(IGC_GET_FLAG_VALUE(RematInstCombineBefore)) mpm.add(createIGCInstructionCombiningPass());
+        // TODO: This is a workaround that helps to reduce amount of instructions for clone address arithmetic
+        // it helps with chain of instructions like this
+        // %remat12 = add i64 %baseArith, 100780848
+        // %remat13 = add i64 %remat12, %basePtr
+        // %remat14 = add i64 %remat13, %offsetI
+        // %remat15 = add i64 %remat14, %offsetJ
+        // load ...
+        // ....
+        // %remat21 = add i64 %baseArith, 201561696
+        // %remat22 = add i64 %remat21, %basePtr
+        // %remat23 = add i64 %remat22, %offsetI
+        // %remat24 = add i64 %remat23, %offsetJ
+        // load ...
+        // we can compress this chain of instruction into one "add" for each "load"
+        // this is achieved by combining reassoc + cse 3 times (each pair hoists one add)
+        // it should be substituted for general pass when it's implemented
+        //
+        // Now it's accessible through flag, for testing purposes
+        if (IGC_GET_FLAG_VALUE(RematReassocBefore)) {
+            mpm.add(llvm::createReassociatePass());
+            mpm.add(llvm::createEarlyCSEPass());
+            mpm.add(llvm::createReassociatePass());
+            mpm.add(llvm::createEarlyCSEPass());
+            mpm.add(llvm::createReassociatePass());
+            mpm.add(llvm::createEarlyCSEPass());
+        }
 
-    if (IGC_IS_FLAG_ENABLED(EnableRemat) || (ctx.m_retryManager.AllowCloneAddressArithmetic() && ctx.type == ShaderType::OPENCL_SHADER)) {
-
-
-      // TODO: This is a workaround that helps to reduce amount of instructions for clone address arithmetic
-      // it helps with chain of instructions like this
-      // %remat12 = add i64 %baseArith, 100780848
-      // %remat13 = add i64 %remat12, %basePtr
-      // %remat14 = add i64 %remat13, %offsetI
-      // %remat15 = add i64 %remat14, %offsetJ
-      // load ...
-      // ....
-      // %remat21 = add i64 %baseArith, 201561696
-      // %remat22 = add i64 %remat21, %basePtr
-      // %remat23 = add i64 %remat22, %offsetI
-      // %remat24 = add i64 %remat23, %offsetJ
-      // load ...
-      // we can compress this chain of instruction into one "add" for each "load"
-      // this is achieved by combining reassoc + cse 3 times (each pair hoists one add)
-      // it should be substituted for general pass when it's implemented
-      //
-      // Now it's accessible through flag, for testing purposes
-
-      if (IGC_GET_FLAG_VALUE(RematReassocBefore)) {
-        mpm.add(llvm::createReassociatePass());
-        mpm.add(llvm::createEarlyCSEPass());
-        mpm.add(llvm::createReassociatePass());
-        mpm.add(llvm::createEarlyCSEPass());
-        mpm.add(llvm::createReassociatePass());
-        mpm.add(llvm::createEarlyCSEPass());
-      }
-
-      mpm.add(createCloneAddressArithmeticPass());
-      // cloneAddressArithmetic leaves old instructions unnecessary
-      // dce pass helps to clean that up
-      mpm.add(createDeadCodeEliminationPass());
+        mpm.add(createCloneAddressArithmeticPass());
+        // cloneAddressArithmetic leaves old instructions unnecessary
+        // dce pass helps to clean that up
+        mpm.add(createDeadCodeEliminationPass());
     }
 
     mpm.add(createRematAddressArithmeticPass());

IGC/common/igc_flags.h

@@ -737,9 +737,14 @@ DECLARE_IGC_REGKEY(DWORD, RematBlockSize,   10,  "Represents a threshold for a b
 DECLARE_IGC_REGKEY(DWORD, RematUsesThreshold,   5,  "Amount of uses after which operand is not rematerialized", false)
 DECLARE_IGC_REGKEY(DWORD, RematChainLimit,   10,  "If number of instructions we've collected is more than this value, we bail on it", false)
 DECLARE_IGC_REGKEY(DWORD, RematRPELimit,   100,  "Cutoff value for register estimator, lower than that, kernel won't be rematted", false)
+DECLARE_IGC_REGKEY(bool, RematEnable, false, "Enable clone adress arithmetic pass not only on retry", false)
+DECLARE_IGC_REGKEY(bool, RematMoveAddrSpaceCast, false, "Enable clone adress arithmetic pass not only on retry", false)
 DECLARE_IGC_REGKEY(bool, RematSameBBScope, true, "Confine rematerialization only to variables within the same BB, we won't pull down values from predeccors", false)
-DECLARE_IGC_REGKEY(bool, EnableRemat, false, "Enable clone adress arithmetic pass not only on retry", false)
+DECLARE_IGC_REGKEY(bool, RematRespectUniformity, false, "Cutoff computation chain on uniform values", false)
+DECLARE_IGC_REGKEY(bool, RematAllowExtractElement, false, "Allow Extract Element to computation chain", false)
 DECLARE_IGC_REGKEY(bool, RematReassocBefore, false, "Enable short sequence of passes before clone address arithmetic pass to potentially decrese amount of operations that will be rematerialized", false)
+DECLARE_IGC_REGKEY(bool, RematInstCombineBefore, false, "Enable short sequence of passes before clone address arithmetic pass to potentially decrese amount of operations that will be rematerialized", false)
+DECLARE_IGC_REGKEY(bool, RematAddrSpaceCastToUse, false, "Allow rematerialization of inttoptr that are used inside AddrSpaceCastInst", false)
 DECLARE_IGC_REGKEY(bool, DumpRegPressureEstimate, false,  "Dump RegPressureEstimate to a file", false)
 DECLARE_IGC_REGKEY(debugString, DumpRegPressureEstimateFilter, 0, "Only dump RegPressureEstimate for functions matching the given regex", false)
 DECLARE_IGC_REGKEY(bool, EnableReusingXYZWStoreConstPayload, true, "Enable reusing XYZW stores const payload", false)