call ws for partial hw fix

fused call has a partial HW fix, which needs to remove predicate call. Also
any indirect call might have different targets for bigEu and SmallEu,
therefore, needs to have two calls, one for bigEu and the other for smallEU.

The new code is off now and will be turned it on in next submit.

Author: jgu222

visa/FlowGraph.cpp

@@ -606,6 +606,8 @@ bool FlowGraph::matchBranch(int &sn, INST_LIST& instlist, INST_LIST_ITER &it)
 //
 void FlowGraph::preprocess(INST_LIST& instlist)
 {
+    // It is easier to add WA before control-flow is build for partial HW
+    // fix to fused call HW bug.
 
     std::unordered_set<G4_Label*> labels;   // label inst we have seen so far
 
@@ -989,6 +991,12 @@ void FlowGraph::constructFlowGraph(INST_LIST& instlist)
 
     pKernel->dumpToFile("after.CFGConstruction");
 
+    // Do call WA before return/exit processing.
+    if (builder->hasFusedEU() && builder->getuint32Option(vISA_fusedCallWA) == 2)
+    {
+        hasGoto |= convertPredCall(labelMap);
+    }
+
     removeRedundantLabels();
 
     pKernel->dumpToFile("after.RemoveRedundantLabels");
@@ -4880,6 +4888,137 @@ bool FlowGraph::convertJmpiToGoto()
     return Changed;
 }
 
+// Changes a predicated call to a unpredicated call like the following:
+//    BB:
+//      (p) call (execSize) ...
+//    NextBB:
+//
+// It is changed to
+//    BB:
+//      p1 = simdsize > execSize ? (p & ((1<<execSize) - 1)) : p
+//      (~p1) goto (simdsize) target_lbl
+//    newCallBB:
+//         call (execSize) ...
+//    newNextBB:
+//    NextBB:
+//  where target_lbl is newTargetBB.
+//
+bool FlowGraph::convertPredCall(std::unordered_map<G4_Label*, G4_BB*>& aLabelMap)
+{
+    bool changed = false;
+    // Add BB0 and BB1 into the subroutine in which aAnchorBB is.
+    auto updateSubroutineTab = [&](G4_BB* aAnchorBB, G4_BB* BB0, G4_BB* BB1)
+    {
+        for (auto MI : subroutines)
+        {
+            std::vector<G4_BB*>& bblists = MI.second;
+            auto BI = std::find(bblists.begin(), bblists.end(), aAnchorBB);
+            if (BI == bblists.end())
+            {
+                continue;
+            }
+            bblists.push_back(BB0);
+            bblists.push_back(BB1);
+        }
+    };
+
+    const G4_ExecSize SimdSize = pKernel->getSimdSize();
+    G4_Type PredTy = SimdSize > 16 ? Type_UD : Type_UW;
+    auto NextBI = BBs.begin();
+    for (auto BI = NextBI, BE = BBs.end(); BI != BE; BI = NextBI)
+    {
+        ++NextBI;
+        G4_BB* BB = *BI;
+        if (BB->empty())
+        {
+            continue;
+        }
+        G4_BB* NextBB = (NextBI == BE ? nullptr : *NextBI);
+
+        G4_INST* Inst = BB->back();
+        G4_Predicate* Pred = Inst->getPredicate();
+        if (!(Pred && (Inst->isCall() || Inst->isFCall())))
+        {
+            continue;
+        }
+
+        const bool hasFallThru = (NextBB && BB->Succs.size() >= 1 && BB->Succs.front() == NextBB);
+
+        G4_BB* newCallBB = createNewBBWithLabel("predCallWA", Inst->getLineNo(), Inst->getCISAOff());
+        insert(NextBI, newCallBB);
+        G4_Label* callBB_lbl = newCallBB->getLabel();
+        assert(callBB_lbl);
+        aLabelMap.insert(std::make_pair(callBB_lbl, newCallBB));
+
+        G4_BB* targetBB = createNewBBWithLabel("predCallWA", Inst->getLineNo(), Inst->getCISAOff());
+        insert(NextBI, targetBB);
+        G4_Label* target_lbl = targetBB->getLabel();
+        aLabelMap.insert(std::make_pair(target_lbl, targetBB));
+
+        // relink call's succs
+        if (hasFallThru)
+        {
+            removePredSuccEdges(BB, NextBB);
+        }
+        auto SI = BB->Succs.begin(), SE = BB->Succs.end();
+        while (SI != SE)
+        {
+            G4_BB* Succ = *SI;
+            ++SI;
+            removePredSuccEdges(BB, Succ);
+            addPredSuccEdges(newCallBB, Succ, false);
+        }
+        addPredSuccEdges(BB, newCallBB, true);
+        addPredSuccEdges(BB, targetBB, false);
+        addPredSuccEdges(newCallBB, targetBB, true);
+        if (hasFallThru)
+        {
+            addPredSuccEdges(targetBB, NextBB, true);
+        }
+
+        // delink call inst
+        BB->pop_back();
+
+        G4_Predicate* newPred;
+        const G4_ExecSize ExecSize = Inst->getExecSize();
+        if (SimdSize == ExecSize)
+        {
+            // negate predicate
+            newPred = builder->createPredicate(
+                Pred->getState() == PredState_Plus ? PredState_Minus : PredState_Plus,
+                Pred->getBase()->asRegVar(), 0, Pred->getControl());
+        }
+        else
+        {
+            // Common dst and src0 operand for flag.
+            G4_Type oldPredTy = ExecSize > 16 ? Type_UD : Type_UW;
+            G4_Declare* newDcl = builder->createTempFlag(PredTy == Type_UD ? 2 : 1);
+            auto pDst = builder->createDst(newDcl->getRegVar(), 0, 0, 1, PredTy);
+            auto pSrc0 = builder->createSrc(Pred->getBase(), 0, 0, builder->getRegionScalar(), oldPredTy);
+            auto pSrc1 = builder->createImm((1 << ExecSize) - 1, PredTy);
+            auto pInst = builder->createBinOp(
+                G4_and, g4::SIMD1, pDst, pSrc0, pSrc1,
+                InstOpt_M0 | InstOpt_WriteEnable, false);
+            BB->push_back(pInst);
+
+            newPred = builder->createPredicate(
+                Pred->getState() == PredState_Plus ? PredState_Minus : PredState_Plus,
+                newDcl->getRegVar(), 0, Pred->getControl());
+        }
+
+        G4_INST* gotoInst = builder->createGoto(newPred, SimdSize, target_lbl, InstOpt_NoOpt, false);
+        BB->push_back(gotoInst);
+
+        Inst->setPredicate(nullptr);
+        newCallBB->push_back(Inst);
+
+        updateSubroutineTab(BB, newCallBB, targetBB);
+        changed = true;
+    }
+    // if changed is true, it means goto has been added.
+    return changed;
+}
+
 void FlowGraph::print(std::ostream& OS) const
 {
     const char* kname = nullptr;

visa/FlowGraph.h

@@ -384,6 +384,7 @@ class FlowGraph
     void convertGotoToJmpi(G4_INST *gotoInst);
     G4_BB* getSinglePredecessor(G4_BB* BB, G4_BB* ExcludedPred) const;
     bool convertJmpiToGoto();
+    bool convertPredCall(std::unordered_map<G4_Label*, G4_BB*>& aLabelMap); // for WA
 
     unsigned getNumFuncs() const {return unsigned(funcInfoTable.size());}
 

visa/Optimizer.cpp

@@ -7599,8 +7599,7 @@ bool Optimizer::foldPseudoAndOr(G4_BB* bb, INST_LIST_ITER& ii)
         }
 
         // Call WA for fused EU
-        if (builder.hasFusedEU() && builder.getuint32Option(vISA_fusedCallWA) == 1 &&
-            kernel.hasIndirectCall())
+        if (builder.hasFusedEU() && kernel.hasIndirectCall())
         {
             applyFusedCallWA();
         }
@@ -16587,30 +16586,64 @@ void Optimizer::doNoMaskWA_postRA()
     }
 }
 
-// Assumption:
+// Summary:
 //   vISA assumes the call's target would be uniform within a thread. This is consistent with
-//   hardware call instructions. Under EU fusion,  assume that an indirect call invokes A
-//   in thread 0 and invokes B in thread 1, which isn't supported.
+//   hardware call instructions. Under EU fusion, a pair of fused thread 0 and 1 might diverge,
+//   meaning that an indirect call invokes A in thread 0 and invokes B in thread 1, which isn't
+//   supported by fused EU hardware.
 //
-// This function does two things:
-//   1.  For any indirect call like the following:
-//          call
-//       changed it to:
+// This function is used to make sure each fused call will have a single target. As there are HW bugs
+// in fused calls, this function will WA HW bugs as well.  The general idea is:
+//     Given:
+//          (p) call  r5
+//     Changed it to:
 //          if (BigEU)
-//             call
+//             (p) call  r5
 //          else   // SmallEU
-//             call
-//   2. HW has a bug in which call always runs and it always uses BigEU's target as targets for both EUs.
-//      This causes several issues and the WA is used to fix this harware bug.
+//             (p) call  r5
 //
-// Details of 2
+// As HW has a bug in which call always runs (even no active channels) and it always uses BigEU's target
+// as targets for both EUs. This causes several issues and the software WA is used to fix this harware bug.
+// There are several cases:
+//   1. For platforms that has NO HW fix (fusedCallWA 1), applying software WA as described
+//      below in "Details of 1",
+//
+//   2. For platforms that has the PARTIAL HW fix (fusedCallWA 2)
+//      Any predicated call must be changed to unpredicated like the following:
+//           (p) call ...
+//       changed to
+//           if (p)
+//              call ...
+//
+//       This is done in Flowgraph::convertPredCall(), right after control-flow is
+//       constructed.
+//
+//      2.1 for direct call like the following
+//                (p) call r5
+//
+//           if (p)
+//               if (BigEU)  // BigEU
+//                  call  r5
+//               else  // SmallEU
+//                  call  r5
+//   3. For platforms that have a full fix (if any) (fusedCallWA 0),
+//      just do the following for indirect call.
+//         (p) call r5
+//         if (BigEU)  // BigEU
+//             (p) call  r5
+//         else  // SmallEU
+//             (p) call  r5
+//
+//  This function handles 1) and duplicating call for BigEU and SmallEU.
+//
+// Details of 1
 // ============
 //  Under EU fusion,  assume that an indirect call invokes A in thread 0 and invokes B in thread 1.
 //  Assume that these two threads are fused and run on a pair of fused EUs {bigEU, smallEU}. The hardware
-//  will always invoke A: the callee from thread 0 in bigEU even in else (smallEU) barnch, which is
-//  incorrect. To workaround this bug, we have to rely on the fact that cr0.2 is shared among the pair
-//  of fused EUs and copy thread 1's callee into thread 0 via cr0.2. In doing so, thread 1's callee
-//  can be invoked. The details are as follows:
+//  will always invoke A: the callee from thread 0 in bigEU even in else branch (in general case),
+//  which is incorrect. To workaround this bug, we have to rely on the fact that cr0.2 is shared among
+//  the pair of fused EUs and copy thread 1's callee B into thread 0 via cr0.2. In doing so, thread 1's
+//  callee can be invoked. The details are as follows:
 //
 //    before:
 //      BB:
@@ -16644,13 +16677,15 @@ void Optimizer::doNoMaskWA_postRA()
 //            join <nextJoin or null>                                       // finalJoin
 //
 // The BBs and those insts such as I4_patch_add/I5_patch_add, etc are added into m_indirectCallWAInfo
-// so that finishFusedCallWA() can finish post-processing to patch the relative IP and others.
+// so that finishFusedCallWA() can finish post-processing to patch the relative IP and others. If calla
+// can be used,  no IP patching is needed. See code for details.
 //
-// Note that there is another hardware bug. If BigEU is off, the mov instruction
+// In order to make the following to run always even through bigEU is off,
 //    "(W)     mov (1 |M0)  smallEUTarget:ud   cr0.2<0;1,0>:ud"
-// will not run, thus BigEU will not have smallEU's target. Since this WA requires
-// the above move must be run, a special maskOff (M16) must be used to force NoMask to run
-// no matter if the EU is off or on. This will be handled in finishFusedCallWA().
+// a special maskOff (M16) must be used to force NoMask to run no matter if the EU is off or on.
+// This will be handled in finishFusedCallWA().
+// (See details in finishFusedCallWA(). To make it work, any kernel with indirect call is required
+// to be simd16 or simd8, not simd32, so that M16 can be used to force running the inst always.)
 //
 void Optimizer::applyFusedCallWA()
 {
@@ -16707,6 +16742,12 @@ void Optimizer::applyFusedCallWA()
         }
     };
 
+    // Only process call wa (fusedCallWA = 1) or indirect call is non-uniform
+    if (!((builder.getuint32Option(vISA_fusedCallWA) == 1) || !builder.getOption(vISA_fusedCallUniform)))
+    {
+        return;
+    }
+
     for (BB_LIST_ITER BI = fg.begin(), BE = fg.end(); BI != BE;)
     {
         BB_LIST_ITER currBI = BI;
@@ -16837,6 +16878,53 @@ void Optimizer::applyFusedCallWA()
             simdsz > g4::SIMD16 ? g4::SIMD32 : g4::SIMD16,
             builder.createNullDst(Type_UW), I1_Src0, Bit7, InstOpt_WriteEnable);
 
+        if (builder.getuint32Option(vISA_fusedCallWA) != 1)
+        {
+            assert(!builder.getOption(vISA_fusedCallUniform));
+            // Just need to duplicate the call so that one is called under BigEU,
+            // and the other is under SmallEU.
+
+            BB->pop_back();   // unlink the call inst from BB
+            BB->push_back(I0);
+            BB->push_back(I1);
+
+            I0->addDefUse(I1, Opnd_src0);
+
+            G4_Predicate* pred_m1 = builder.createPredicate(PredState_Minus, F->getRegVar(), 0);
+            G4_INST* gotoSmallB0 = builder.createCFInst(pred_m1, G4_goto, simdsz, smallB0Label, smallB0Label, InstOpt_NoOpt, false);
+            BB->push_back(gotoSmallB0);
+            I1->addDefUse(gotoSmallB0, Opnd_pred);
+
+            G4_Predicate* nPred(callI->getPredicate());
+            G4_SrcRegRegion* nSrc = builder.createSrc(Target->getBase(), 0, 0, builder.getRegionScalar(), Type_UD);
+            G4_INST* nCallI = builder.createInternalInst(nPred, callI->opcode(),
+                nullptr, g4::NOSAT, callI->getExecSize(), nullptr, nSrc, nullptr, callI->getOption());
+            (void)bigB0->push_back(callI);
+            (void)smallB0->push_back(nCallI);
+
+            // Need to create fcall info
+            if (G4_FCALL* orig_fcallinfo = builder.getFcallInfo(callI))
+            {
+                builder.addFcallInfo(nCallI, orig_fcallinfo->getArgSize(), orig_fcallinfo->getRetSize());
+            }
+            // Might need to update subroutine table
+            updateSubroutineTableIfNeeded(origNextBB, bigB0, bigB1, smallB0, smallB1, newNextBB);
+
+            if (!fg.globalOpndHT.isOpndGlobal(Target))
+            {
+                callI->removeDefUse(Opnd_src0);
+            }
+            fg.globalOpndHT.addGlobalOpnd(Target);
+            fg.globalOpndHT.addGlobalOpnd(nSrc);
+
+            // done with this indirect call.
+            continue;
+        }
+
+        //
+        // main call WA under fusedCallWA = 1
+        //
+
         // I2:  (!flag) mov cr0.2  callee
         G4_VarBase* V_cr0 = builder.phyregpool.getCr0Reg();
         G4_DstRegRegion* I2_Dst = builder.createDst(V_cr0, 0, 2, 1, Type_UD);

visa/include/VISAOptionsDefs.h

@@ -343,7 +343,8 @@ DEF_VISA_OPTION(vISA_forceNoMaskWA,                 ET_BOOL,  "-forceNoMaskWA",
 DEF_VISA_OPTION(vISA_newNoMaskWA,                   ET_BOOL,  "-newNoMaskWA", "Temp just for VC", true)
 DEF_VISA_OPTION(vISA_noMaskWAOnFuncEntry,           ET_BOOL,  "-noMaskWAOnFuncEntry", UNUSED, true)
 DEF_VISA_OPTION(vISA_newTmpNoMaskWA,                ET_INT32, "-newTmpNoMaskWA", "to control scalar IGC, temporary -newTempNoMaskWA 0|1|2", 0)
-DEF_VISA_OPTION(vISA_fusedCallWA,                   ET_INT32, "-fusedCallWA",   "EU Fusion call ww: 0: no wa, 1: sw wa w/o hw fix; 2: sw wa with partial HW fix", 0)
+DEF_VISA_OPTION(vISA_fusedCallWA,                   ET_INT32, "-fusedCallWA",   "EU Fusion call wa: 0: no hw bug, 1: sw wa w/o hw fix; 2: sw wa with partial HW fix", 0)
+DEF_VISA_OPTION(vISA_fusedCallUniform,              ET_BOOL,  "-fusedCallUniform", "true: fused call is uniform; false otherwise.", false)
 DEF_VISA_OPTION(vISA_DstSrcOverlapWA,               ET_BOOL,  "-dstSrcOverlapWA",  UNUSED, true)
 DEF_VISA_OPTION(vISA_Src1Src2OverlapWA,               ET_BOOL,  "-src1Src2OverlapWA",  UNUSED, false)
 DEF_VISA_OPTION(vISA_noSendSrcDstOverlap,           ET_BOOL,  "-noSendSrcDstOverlap", UNUSED, false)