Enable fusing atomic messages. It is off. Turn it on by setting

igc key EnableAtomicFusion to 1.

Change-Id: I9234b8bab65d05018a6dc2fc78718515c3408327

Author: jgu222

IGC/Compiler/CISACodeGen/CISABuilder.cpp

@@ -3829,14 +3829,25 @@ void CEncoder::InitEncoder( bool canAbortOnSpill )
         vbuilder->SetOption(vISA_FuseTypedWrites, true);
     }
 
-    // Enable SendFusion for SIMD8 
+    // Enable SendFusion for SIMD8
     if (IGC_IS_FLAG_ENABLED(EnableSendFusion) &&
-		m_program->GetContext()->platform.supportSplitSend() &&
+        m_program->GetContext()->platform.supportSplitSend() &&
         m_program->m_dispatchSize == SIMDMode::SIMD8 &&
         (IGC_GET_FLAG_VALUE(EnableSendFusion) == FLAG_LEVEL_2 ||   // 2: force send fusion
          context->m_DriverInfo.AllowSendFusion()))
     {
         vbuilder->SetOption(vISA_EnableSendFusion, true);
+        if (IGC_IS_FLAG_ENABLED(EnableAtomicFusion) &&
+            context->type == ShaderType::OPENCL_SHADER)
+        {
+            vbuilder->SetOption(vISA_EnableAtomicFusion, true);
+        }
+    }
+
+    if (context->getModuleMetaData()->compOpt.FastRelaxedMath ||
+        context->getModuleMetaData()->compOpt.UnsafeMathOptimizations)
+    {
+        vbuilder->SetOption(vISA_unsafeMath, true);
     }
 
     // With statelessToStatefull on, it is possible that two different BTI messages

IGC/common/igc_flags.def

@@ -56,6 +56,7 @@ DECLARE_IGC_REGKEY(bool, ExpandPlane,                   0,     "Enable pln to ma
 DECLARE_IGC_REGKEY(bool, EnableBCR,                     false,  "Enable bank conflict reduction.")
 DECLARE_IGC_REGKEY(bool, GlobalSendVarSplit, false, "Enable global send variable splitting when we are about to spill")
 DECLARE_IGC_REGKEY(DWORD,EnableSendFusion,              1,     "Enable(!=0)/disable(0)/force(2) send fusion. Valid for simd8 shader/kernel only.")
+DECLARE_IGC_REGKEY(bool, EnableAtomicFusion,            false, "To enable/disable atomic send fusion (simd8 shaders). Valid if EnableSendFusion is on.")
 
 DECLARE_IGC_GROUP("IGC Optimization")
 DECLARE_IGC_REGKEY(bool, DisableIGCOptimizations,       false, "Setting this to 1/true adds a compiler switch to disables all the above IGC optimizations")

visa/Gen4_IR.hpp

@@ -409,28 +409,61 @@ class G4_SendMsgDescriptor
             funcID == SFID_SPAWNER || funcID == SFID_URB || funcID == SFID_NUM;
     }
 
-    bool isAtomicMessage() const
+    bool isIntAtomicMessage() const
     {
         auto funcID = extDesc.layout.funcID;
         if (funcID != SFID_DP_DC1)
             return false;
 
-        uint32_t funcCtrl = getFuncCtrl();
-        uint16_t msgType = (funcCtrl >> 14) & 0xF;
+        uint16_t msgType = getMessageType();
         if (msgType == DC1_UNTYPED_ATOMIC || msgType == DC1_A64_ATOMIC)
         {
             return true;
         }
         if (getGenxPlatform() >= GENX_SKL)
+        {
+            if (msgType == DC1_TYPED_ATOMIC)
+                return true;
+        }
+        return false;
+    }
+
+    bool isFloatAtomicMessage() const
+    {
+        auto funcID = extDesc.layout.funcID;
+        if (funcID != SFID_DP_DC1)
+            return false;
+
+        uint16_t msgType = getMessageType();
+        if (getGenxPlatform() >= GENX_SKL)
         {
             if (msgType == DC1_UNTYPED_FLOAT_ATOMIC ||
-                msgType == DC1_A64_UNTYPED_FLOAT_ATOMIC ||
-                msgType == DC1_TYPED_ATOMIC)
+                msgType == DC1_A64_UNTYPED_FLOAT_ATOMIC)
                 return true;
         }
         return false;
     }
 
+    bool isAtomicMessage() const
+    {
+        return isIntAtomicMessage() || isFloatAtomicMessage();
+    }
+
+    uint16_t getAtomicOp() const
+    {
+        assert(isAtomicMessage() && "ICE: getting atomicOp from non-atomic message!");
+        uint32_t funcCtrl = getFuncCtrl();
+        if (isIntAtomicMessage())
+        {
+            // bits: 11:8
+            return (uint16_t)((funcCtrl >> 8) & 0xF);
+        }
+
+        // must be float Atomic
+        // bits: 10:8
+        return (int16_t)((funcCtrl >> 8) & 0x7);
+    }
+
     bool isBarrierMsg() const
     {
         auto funcID = extDesc.layout.funcID;

visa/SendFusion.cpp

@@ -130,6 +130,7 @@ namespace vISA
         G4_VarBase* getVarBase(G4_VarBase* RegVar, G4_Type Ty);
         uint32_t getFuncCtrlWithSimd16(G4_SendMsgDescriptor* Desc);
         void simplifyMsg(INST_LIST_ITER SendIter);
+        bool isAtomicCandidate(G4_SendMsgDescriptor* msgDesc);
 
 		bool WAce0Read;
 
@@ -190,6 +191,11 @@ uint32_t SendFusion::getFuncCtrlWithSimd16(G4_SendMsgDescriptor* Desc)
             // bit13-12: SM3
             FC = ((FC & ~0x3000) | (MDC_SM3_SIMD16 << 12));
             break;
+        case DC1_UNTYPED_ATOMIC:
+        case DC1_UNTYPED_FLOAT_ATOMIC:
+            // bit12: SM2R
+            FC = ((FC & ~0x1000) | (MDC_SM2R_SIMD16 << 12));
+            break;
         }
     }
     else if (funcID == SFID_DP_DC2)
@@ -220,9 +226,74 @@ uint32_t SendFusion::getFuncCtrlWithSimd16(G4_SendMsgDescriptor* Desc)
     return FC;
 }
 
+bool SendFusion::isAtomicCandidate(G4_SendMsgDescriptor* msgDesc)
+{
+    uint32_t funcID = msgDesc->getFuncId();
+    if (funcID != SFID_DP_DC1) {
+        return false;
+    }
+
+    // Right now, the following atomic messages are DW per simd-lane.
+    uint16_t msgType = msgDesc->getMessageType();
+    bool intAtomic = true; // true: int; false : float
+    switch (msgType) {
+    default:
+        return false;
+    case DC1_UNTYPED_ATOMIC:
+        break;
+    case DC1_UNTYPED_FLOAT_ATOMIC:
+        intAtomic = false;
+        break;
+    }
+
+    // Had right atomic type, now check AtomicOp
+    uint16_t atomicOp = msgDesc->getAtomicOp();
+    if (intAtomic)
+    {
+        switch (atomicOp)
+        {
+        default:
+            return false;
+        case GEN_ATOMIC_AND:
+        case GEN_ATOMIC_OR:
+        case GEN_ATOMIC_XOR:
+        case GEN_ATOMIC_INC:
+        case GEN_ATOMIC_DEC:
+        case GEN_ATOMIC_ADD:
+        case GEN_ATOMIC_SUB:
+        case GEN_ATOMIC_REVSUB:
+        case GEN_ATOMIC_IMAX:
+        case GEN_ATOMIC_IMIN:
+        case GEN_ATOMIC_UMAX:
+        case GEN_ATOMIC_UMIN:
+        case GEN_ATOMIC_PREDEC:
+            break;
+        }
+    }
+    else
+    {
+        if (!Builder->getOption(vISA_unsafeMath))
+        {
+            return false;
+        }
+
+        switch (atomicOp)
+        {
+        default:
+            return false;
+        case GEN_ATOMIC_FMAX:
+        case GEN_ATOMIC_FMIN:
+            break;
+        }
+    }
+    return true;
+
+    // Need to check if it is packed half integer/float ?
+}
+
 // We will do send fusion for a few messages. Those messages all
-// have the address payload for each channel, thus address payload
-// is 1 GRF for exec_size=8 (no A64 messages for now).
+// have DW-sized address for each lane, thus address payload is
+// 1 GRF for exec_size=8 (no A64 messages for now).
 //
 // The optimization is performed for the following cases:
 //    1) [(w)] send(8) + send(8) --> (W&flag) send(16), and
@@ -286,10 +357,52 @@ bool SendFusion::simplifyAndCheckCandidate(INST_LIST_ITER Iter)
 		return false;
 	}
 
-    // Unless we can prove there are no aliases of two sends, we will not be
-    // able to do fusion (or we know for sure that the first addr's value or
-    // last addr's value is taken). For now, disable it.
-    if  (msgDesc->isDataPortWrite())
+    // For write messages:
+    //   unless we can prove there are no aliases of two sends's address payload,
+    //   we will not be able to do fusion as hardware does not have deterministic
+    //   behavior if the same address appear more than once. For example,
+    //
+    //     send (8)  (a1_0, C, a1_2, ..., a1_7) (d1_0, d1_1, d1_2, ..., d1_7)
+    //     send (8)  (a2_0, C, a2_2, ..., a2_7) (d2_0, d2_1, d2_2, ..., d2_7)
+    //
+    //   Both sends have the same addr 'C' in lane 1, and C's value is d2_1 after
+    //   two sends. However, if they are fused, they become:
+    //
+    //     send (16) (a1_0, C,    ..., a2_0, C,    ..., a2_7)
+    //               (d1_0, d1_1, ..., d2_0, d2_1, ..., d2_7)
+    //
+    //   The hardware cannot guarantee that d2_1 will be the value written to addr 'C'.
+    //   Thus, we have to disallow fusion for writes
+    //
+    // But we can do it for atomic messages if certain conditions meet:
+    // (Checked in isAtomicCandidate().)
+    //    1. Can fuse if no return value.
+    //       Atomic messages has both read/write. Let us take a look at atomic_add
+    //       for example: let's assume atomic_add returns the old value and does updating.
+    //
+    //          location at p :  10 (original value)
+    //          x = atomic_add p, 1
+    //          y = atomic_add p, 2
+    //       then, x = 10  & y = 11
+    //
+    //       If fused them, it becomes:
+    //          {x, y} = atomic_add {p, p}, {1, 2}
+    //            it's possible that we have x = 12 & y = 10 as the 2nd atomic operations
+    //            could be performed first. This does change the behavior of the program.
+    //
+    //       Since we don't know if two sends share the same address, we will conservatively
+    //       avoid fusing two sends with return values for now.
+    //    2. Assume unsafe math is present for float atomic.
+    //       As two sends might share the same location, and fused send might change the order
+    //       of float atomic operations.  We need to have unsafe-math to perform fusing legally.
+    //
+    bool isAtomicCand = false;
+    if (Builder->getOption(vISA_EnableAtomicFusion))
+    {
+        isAtomicCand = isAtomicCandidate(msgDesc);
+    }
+    if  ((!isAtomicCand && msgDesc->isDataPortWrite()) ||
+         (isAtomicCand && rspLen > 0))
     {
         return false;
     }
@@ -317,6 +430,14 @@ bool SendFusion::simplifyAndCheckCandidate(INST_LIST_ITER Iter)
         return false;
     }
 
+    // Only handling the following messages that have DW as data element.
+    // Untyped with up to 4 DW per lane is handled.
+
+    // special handling of atomic
+    if (isAtomicCand) {
+        return true;
+    }
+
     uint32_t funcID = msgDesc->getFuncId();
     uint32_t msgType = msgDesc->getMessageType();
     if (funcID == SFID_DP_DC)
@@ -542,9 +663,12 @@ bool SendFusion::canFusion(INST_LIST_ITER IT0, INST_LIST_ITER IT1)
     //        and hardware does not have deterministic behavior about which data,
     //        data0 or data1, will be stored into x! For this reason, no write
     //        will be fused if they have common address. Since we don't know (for now)
-    //        if addresses of two sends can point to the same address, we just
+    //        if addresses of two sends can point to the same location, we just
     //        conservatively do not fuse any write messages.
-    //       
+    //
+    // Atomic messages:
+    //    As only no-return-value atomic can be fused, RAW will be false always.
+    //
     G4_SendMsgDescriptor* desc0 = I0->getMsgDesc();
     G4_SendMsgDescriptor* desc1 = I1->getMsgDesc();
     bool fusion = I0->getOption() == I1->getOption() &&
@@ -1382,33 +1506,39 @@ bool SendFusion::run(G4_BB* BB)
             ++II0;
             continue;
         }
- 
+
         G4_INST* inst1 = nullptr;
         INST_LIST_ITER II1 = II0;
         ++II1;
         while(II1 != IE)
         {
             G4_INST* tmp = *II1;
-            if (tmp->opcode() == inst0->opcode() &&
-				tmp->getExecSize() == inst0->getExecSize() &&
-                simplifyAndCheckCandidate(II1))
+            if (simplifyAndCheckCandidate(II1))
             {
-                if (canFusion(II0, II1))
+                // possible 2nd send to be fused
+                if (tmp->opcode() == inst0->opcode() &&
+                    tmp->getExecSize() == inst0->getExecSize())
                 {
-                    // Found
-                    inst1 = tmp;
+                    if (canFusion(II0, II1))
+                    {
+                        // Found
+                        inst1 = tmp;
+                    }
                 }
 
-                // Don't advance II1 as II1 might be the first send
-                // for the next pair of candidates.
+                // If found (inst1 != null), exit the inner loop to start fusing;
+                // if not found, exit the inner loop and use this one (II1) as
+                // the 1st send of possible next pair to start the outer loop again.
+                //
+                // In both case, don't advance II1.
                 break;
             }
 
             ++II1;
             if (tmp->isSend() || tmp->isFence() || tmp->isOptBarrier())
             {
                 // Don't try to fusion two sends that are separated
-                // by other memory/barrier instructions.             
+                // by other memory/barrier instructions.
                 break;
             }
         }
@@ -1457,9 +1587,8 @@ bool SendFusion::run(G4_BB* BB)
 //
 //    [(w)] send(8) + send(8) --> (W&flag) send(16)
 //
-//      Either noMask or not. When no NoMask
-//
-// Note that (w) send(1|2|4) is also supported.
+// Either noMask or not. When no NoMask, send insts with
+// execsize=1|2|4 are also supported.
 //
 bool vISA::doSendFusion(FlowGraph* aCFG, Mem_Manager* aMMgr)
 {

visa/include/VISAOptions.def

@@ -33,6 +33,7 @@ DEF_VISA_OPTION(vISA_dumpRPE,               ET_BOOL, "-dumpRPE",         UNUSED,
 //=== Optimization options ===
 DEF_VISA_OPTION(vISA_EnableAlways,          ET_BOOL, NULLSTR,            UNUSED, true)
 DEF_VISA_OPTION(vISA_EnableSendFusion,      ET_BOOL, "-enableSendFusion",   UNUSED, false)
+DEF_VISA_OPTION(vISA_EnableAtomicFusion,    ET_BOOL, "-enableAtomicFusion", UNUSED, false)
 DEF_VISA_OPTION(vISA_LocalCopyProp,         ET_BOOL, "-nocopyprop",      UNUSED, true)
 DEF_VISA_OPTION(vISA_LocalFlagOpt,          ET_BOOL, "-noflagopt",       UNUSED, true)
 DEF_VISA_OPTION(vISA_LocalMACopt,           ET_BOOL, "-nomacopt",        UNUSED, true)
@@ -62,6 +63,7 @@ DEF_VISA_OPTION(vISA_LVN,                   ET_BOOL, "-nolvn",       UNUSED, tru
 // only affects acc substitution for now
 DEF_VISA_OPTION(vISA_numGeneralAcc,         ET_INT32, "-numGeneralAcc", "USAGE: -numGeneralAcc <accNum>\n", 0)
 DEF_VISA_OPTION(vISA_reassociate,           ET_BOOL, "-noreassoc",   UNUSED, true)
+DEF_VISA_OPTION(vISA_unsafeMath,            ET_BOOL, "-unsafeMath",  UNUSED, false)
 DEF_VISA_OPTION(vISA_split4GRFVar,          ET_BOOL, "-no4GRFSplit", UNUSED, true)
 
 //=== code gen options ===