Refactor alignment code. Even align variables with size between 1-2GRF. Variables > 2GRF use weak edges.

pratikashar · Zuul · commit 8b0b9d18e0bc · 2019-08-13T22:45:08.000-07:00
Change-Id: I301bb859afd07d67772f3213e9cc3f1dd9c11c85
diff --git a/visa/FlowGraph.cpp b/visa/FlowGraph.cpp
@@ -4413,6 +4413,102 @@ void GlobalOpndHashTable::dump()
     }
 }
 
+void G4_Kernel::computeChannelSlicing()
+{
+    std::unordered_set<G4_Declare*> skipSendDcls;
+    unsigned int simdSize = getSimdSize();
+    channelSliced = true;
+
+    if (simdSize == 8 || simdSize == 16)
+    {
+        // SIMD8/16 kernels are not sliced
+        channelSliced = false;
+        return;
+    }
+
+    for (auto bb : fg)
+    {
+        for (auto inst : bb->getInstList())
+        {
+            if (inst->isPseudoKill() || inst->isWriteEnableInst())
+                continue;
+
+            if (inst->isSend())
+            {
+                auto dst = inst->getDst();
+                if (dst && dst->isDstRegRegion())
+                    skipSendDcls.insert(dst->getTopDcl());
+
+                auto src = inst->getSrc(0);
+                if (src && src->isSrcRegRegion())
+                    skipSendDcls.insert(src->getTopDcl());
+
+                src = inst->getSrc(1);
+                if (src && src->isSrcRegRegion())
+                    skipSendDcls.insert(src->getTopDcl());
+            }
+        }
+    }
+
+    // .dcl V1 size = 128 bytes
+    // op (16|M0) V1(0,0)     ..
+    // op (16|M16) V1(2,0)    ..
+    // For above sequence, return 32. Instruction
+    // is broken in to 2 only due to hw restriction.
+    // Allocation of dcl is still as if it were a
+    // SIMD32 kernel.
+
+    // dcl -> lb, rb, emask offset
+    std::unordered_map<G4_Declare*, std::vector<std::tuple<unsigned int, unsigned int, unsigned int>>> defaultDefs;
+    for (auto bb : fg)
+    {
+        for (auto inst : bb->getInstList())
+        {
+            auto dst = inst->getDst();
+            if (!dst || !dst->isDstRegRegion() || !dst->getTopDcl() || 
+                skipSendDcls.find(dst->getTopDcl()) != skipSendDcls.end() ||
+                dst->asDstRegRegion()->getHorzStride() != 1)
+                continue;
+
+            auto regFileKind = dst->getTopDcl()->getRegFile();
+            if (regFileKind != G4_RegFileKind::G4_GRF && regFileKind != G4_RegFileKind::G4_INPUT)
+                continue;
+
+            auto dstElemSize = G4_Type_Table[dst->getType()].byteSize;
+
+            if (dst->getTopDcl()->getByteSize() <= dstElemSize * simdSize)
+                continue;
+
+            std::vector<std::tuple<unsigned int, unsigned int, unsigned int>> v =
+                { std::make_tuple(dst->getLeftBound(), dst->getRightBound(), inst->getMaskOffset()) };
+            defaultDefs.insert(std::make_pair(dst->getTopDcl(), v));
+        }
+    }
+
+    for (auto dd : defaultDefs)
+    {
+        auto elemSize = dd.first->getElemSize();
+        for (auto defs : dd.second)
+        {
+            auto lb = std::get<0>(defs);
+            auto rb = std::get<1>(defs);
+            auto emaskOffset = std::get<2>(defs);
+
+            // Look for single instruction
+            if (emaskOffset == 0 && lb == 0 && rb == elemSize * 32)
+                channelSliced = false;
+            // Or broken instruction
+            if (emaskOffset == 16 && lb == elemSize * 16 && rb == elemSize * 32)
+                channelSliced = false;
+        }
+
+        if (!channelSliced)
+            break;
+    }
+
+    return;
+}
+
 void G4_Kernel::calculateSimdSize()
 {
     // Iterate over all instructions in kernel to check
@@ -4440,15 +4536,19 @@ void G4_Kernel::calculateSimdSize()
                 if (size > 16)
                 {
                     simdSize = 32;
-                    return;
+                    break;
                 }
                 else if (size > 8)
                 {
                     simdSize = 16;
                 }
             }
         }
+        if (simdSize == 32)
+            break;
     }
+
+    computeChannelSlicing();
 }
 
 void G4_Kernel::dump() const
diff --git a/visa/FlowGraph.h b/visa/FlowGraph.h
@@ -1297,6 +1297,7 @@ class G4_Kernel
     const char* name;
     unsigned numRegTotal;
     unsigned int simdSize;
+    bool channelSliced = true;
     bool hasAddrTaken;
     Options *m_options;
 
@@ -1440,8 +1441,11 @@ class G4_Kernel
 
     Options *getOptions(){ return m_options; }
     bool getOption(vISAOptions opt) const { return m_options->getOption(opt); }
+    void computeChannelSlicing();
     void calculateSimdSize();
     unsigned int getSimdSize() { return simdSize; }
+    bool getChannelSlicing() { return channelSliced; }
+    unsigned int getSimdSizeWithSlicing() { return channelSliced ? simdSize/2 : simdSize; }
 
     void setHasAddrTaken(bool val) { hasAddrTaken = val; }
     bool getHasAddrTaken() { return hasAddrTaken;  }
diff --git a/visa/GraphColor.cpp b/visa/GraphColor.cpp
@@ -2228,34 +2228,48 @@ void GlobalRA::updateSubRegAlignment(G4_SubReg_Align subAlign)
     }
 }
 
+bool GlobalRA::evenAlignNeeded(G4_Declare* dcl)
+{
+    // Return true if even alignment is needed
+    // Even align needed if for given SIMD size and elem type,
+    // a complete def uses between 1-2 GRFs.
+    auto kernelSimdSizeToUse = kernel.getSimdSizeWithSlicing();
+    G4_Declare* topdcl = dcl->getRootDeclare();
+    auto topdclAugMask = getAugmentationMask(topdcl);
+
+    if (!areAllDefsNoMask(topdcl) && !topdcl->getIsPartialDcl() &&
+        topdclAugMask != AugmentationMasks::NonDefault)
+    {
+        auto elemSizeToUse = topdcl->getElemSize();
+        if (elemSizeToUse < 4 && topdclAugMask == AugmentationMasks::Default32Bit)
+            // :uw with hstride 2 can also be Default32Bit and hence needs even alignment
+            elemSizeToUse = 4;
+        else if (elemSizeToUse < 8 && topdclAugMask == AugmentationMasks::Default64Bit)
+            elemSizeToUse = 8;
+
+        if (// Even align if size is between 1-2 GRFs, for >2GRF sizes use weak edges
+            (elemSizeToUse * kernelSimdSizeToUse) > (unsigned int)GENX_GRF_REG_SIZ &&
+            (elemSizeToUse * kernelSimdSizeToUse) <= (unsigned int)(2 * GENX_GRF_REG_SIZ) &&
+            !(kernel.fg.builder->getOption(vISA_enablePreemption) &&
+                dcl == kernel.fg.builder->getBuiltinR0()))
+        {
+            return true;
+        }
+    }
+        return false;
+}
+
 // This function can be invoked before local RA or after augmentation.
-// When invoked before local RA, it sets all vars to be Even aligned,
-// including NoMask ones. This is safe, but conservative. Post
-// augmentation, dcl masks are available so only non-NoMask vars will
-// be Even aligned. Others will be Either aligned. There is no need
-// to store old value of align because HW has no restriction on
-// even/odd alignment that HW conformity computes.
 void GlobalRA::evenAlign()
 {
     // Update alignment of all GRF declares to align
     for (auto dcl : kernel.Declares)
     {
         if (dcl->getRegFile() & G4_GRF)
         {
-            G4_Declare* topdcl = dcl->getRootDeclare();
-            auto topdclAugMask = getAugmentationMask(topdcl);
-
-            if (!areAllDefsNoMask(topdcl) && !topdcl->getIsPartialDcl() &&
-                topdclAugMask != AugmentationMasks::NonDefault &&
-                topdclAugMask != AugmentationMasks::Default64Bit)
+            if (evenAlignNeeded(dcl))
             {
-                if ((topdcl->getElemSize() >= 4 || topdclAugMask == AugmentationMasks::Default32Bit) &&
-                    topdcl->getByteSize() >= GENX_GRF_REG_SIZ &&
-                    !(kernel.fg.builder->getOption(vISA_enablePreemption) &&
-                        dcl == kernel.fg.builder->getBuiltinR0()))
-                {
-                    setEvenAligned(dcl, true);
-                }
+                setEvenAligned(dcl, true);
             }
         }
     }
@@ -3113,9 +3127,7 @@ bool Augmentation::markNonDefaultMaskDef()
             prevAugMask = gra.getAugmentationMask(dcl);
         }
 
-        if (liveAnalysis.livenessClass(G4_GRF) &&
-            gra.getAugmentationMask(dcl) == AugmentationMasks::Default32Bit &&
-            kernel.getSimdSize() > NUM_DWORDS_PER_GRF)
+        if (gra.evenAlignNeeded(dcl))
         {
             auto dclLR = gra.getLocalLR(dcl);
             if (dclLR)
@@ -3124,7 +3136,7 @@ bool Augmentation::markNonDefaultMaskDef()
                 auto phyReg = dclLR->getPhyReg(s);
                 if (phyReg && phyReg->asGreg()->getRegNum() % 2 != 0)
                 {
-                    // If LRA assignment is not 2GRF aligned for SIMD16 then
+                    // If LRA assignment is not 2GRF aligned for then
                     // mark it as non-default. GRA candidates cannot fully
                     // overlap with such ranges. Partial overlap is illegal.
                     gra.setAugmentationMask(dcl, AugmentationMasks::NonDefault);
@@ -4166,6 +4178,22 @@ bool Interference::isStrongEdgeBetween(G4_Declare* dcl1, G4_Declare* dcl2)
     return false;
 }
 
+bool Augmentation::weakEdgeNeeded(AugmentationMasks m)
+{
+    // Weak edge needed in case #GRF exceeds 2
+
+    if (m == AugmentationMasks::Default64Bit)
+        return (G4_Type_Table[Type_Q].byteSize*kernel.getSimdSizeWithSlicing()) > (unsigned int)(2 * GENX_GRF_REG_SIZ);
+
+    if (m == AugmentationMasks::Default32Bit)
+    {
+        // Even align up to 2 GRFs size variable, use weak edges beyond
+        return (G4_Type_Table[Type_D].byteSize*kernel.getSimdSizeWithSlicing()) > (unsigned int)(2 * GENX_GRF_REG_SIZ);
+    }
+
+    return false;
+}
+
 //
 // Mark interference between newDcl and other incompatible dcls in current active lists.
 //
@@ -4183,10 +4211,8 @@ void Augmentation::buildSIMDIntfDcl(G4_Declare* newDcl, bool isCall)
         {
             if (liveAnalysis.livenessClass(G4_GRF) &&
                 // Populate compatible sparse intf data structure
-                // only for 64-bit bit types since others can be
-                // handled using Even align.
-                gra.getAugmentationMask(defaultDcl) == AugmentationMasks::Default64Bit &&
-                newDclAugMask == AugmentationMasks::Default64Bit)
+                // only for weak edges.
+                weakEdgeNeeded(newDclAugMask))
             {
                 if (defaultDcl->getRegVar()->isPhyRegAssigned() &&
                     newDcl->getRegVar()->isPhyRegAssigned())
@@ -4402,7 +4428,7 @@ void Augmentation::augmentIntfGraph()
 
         if (liveAnalysis.livenessClass(G4_GRF))
         {
-            if (kernel.getSimdSize() > NUM_DWORDS_PER_GRF)
+            if (kernel.getSimdSize() >= NUM_DWORDS_PER_GRF)
             {
                 // Set alignment of all GRF candidates
                 // to 2GRF except for NoMask variables
@@ -10544,9 +10570,9 @@ void VerifyAugmentation::verifyAlign(G4_Declare* dcl)
     if (it == masks.end())
         return;
 
-    auto dclMask = std::get<1>((*it).second);
-
-    if (dclMask == AugmentationMasks::Default32Bit)
+    if (dcl->getByteSize() >= NUM_DWORDS_PER_GRF * G4_Type_Table[Type_UD].byteSize &&
+        dcl->getByteSize() <= 2 * NUM_DWORDS_PER_GRF * G4_Type_Table[Type_UD].byteSize &&
+        kernel->getSimdSize() > NUM_DWORDS_PER_GRF)
     {
         auto assignment = dcl->getRegVar()->getPhyReg();
         if (assignment && assignment->isGreg())
@@ -10642,6 +10668,14 @@ void VerifyAugmentation::labelBBs()
 #endif
 }
 
+unsigned int getGRFBaseOffset(G4_Declare* dcl)
+{
+    unsigned int regNum = dcl->getRegVar()->getPhyReg()->asGreg()->getRegNum();
+    unsigned int regOff = dcl->getRegVar()->getPhyRegOff();
+    auto type = dcl->getElemType();
+    return (regNum * G4_GRF_REG_NBYTES) + (regOff * getTypeSize(type));
+}
+
 bool VerifyAugmentation::interfereBetween(G4_Declare* dcl1, G4_Declare* dcl2)
 {
     bool interferes = true;
@@ -10710,8 +10744,8 @@ bool VerifyAugmentation::interfereBetween(G4_Declare* dcl1, G4_Declare* dcl2)
 
             if (lr1->getAssigned() && lr2->getAssigned())
             {
-                auto preg1Start = dcl1->getGRFBaseOffset();
-                auto preg2Start = dcl2->getGRFBaseOffset();
+                auto preg1Start = getGRFBaseOffset(dcl1);
+                auto preg2Start = getGRFBaseOffset(dcl2);
                 auto preg1End = preg1Start + dcl1->getByteSize();
                 auto preg2End = preg2Start + dcl2->getByteSize();
 
@@ -10756,8 +10790,8 @@ void VerifyAugmentation::verify()
     {
         if (dcl1->getRegFile() == G4_RegFileKind::G4_GRF && dcl2->getRegFile() == G4_RegFileKind::G4_GRF)
         {
-            auto preg1Start = dcl1->getGRFBaseOffset();
-            auto preg2Start = dcl2->getGRFBaseOffset();
+            auto preg1Start = getGRFBaseOffset(dcl1);
+            auto preg2Start = getGRFBaseOffset(dcl2);
             auto preg1End = preg1Start + dcl1->getByteSize();
             auto preg2End = preg2Start + dcl2->getByteSize();
 
@@ -10823,6 +10857,9 @@ void VerifyAugmentation::verify()
             {
                 bool interfere = interfereBetween(activeDcl, dcl);
 
+                if (activeDcl->getIsPartialDcl() || dcl->getIsPartialDcl())
+                    continue;
+
                 if (!interfere)
                 {
                     std::cerr << dcl->getRegVar()->getName() << "(" << getStr(dclMask) << ") and " << activeDcl->getRegVar()->getName() << "(" <<
diff --git a/visa/GraphColor.h b/visa/GraphColor.h
@@ -310,6 +310,7 @@ namespace vISA
         void buildSIMDIntfDcl(G4_Declare* newDcl, bool isCall);
         void buildSIMDIntfAll(G4_Declare* newDcl);
         void handleSIMDIntf(G4_Declare* firstDcl, G4_Declare* secondDcl, bool isCall);
+        bool weakEdgeNeeded(AugmentationMasks);
 
     public:
         Augmentation(G4_Kernel& k, Interference& i, LivenessAnalysis& l, LiveRange* ranges[], GlobalRA& g);
@@ -1198,7 +1199,9 @@ namespace vISA
         static uint32_t getRefCount(int loopNestLevel);
         bool isReRAPass();
         void updateSubRegAlignment(G4_SubReg_Align subAlign);
+        bool isChannelSliced();
         void evenAlign();
+        bool evenAlignNeeded(G4_Declare*);
         void getBankAlignment(LiveRange* lr, BankAlign &align);
         void printLiveIntervals();
         void reportUndefinedUses(LivenessAnalysis& liveAnalysis, G4_BB* bb, G4_INST* inst, G4_Declare* referencedDcl, std::set<G4_Declare*>& defs, std::ofstream& optreport, Gen4_Operand_Number opndNum);
diff --git a/visa/LocalRA.cpp b/visa/LocalRA.cpp
@@ -130,7 +130,7 @@ void LocalRA::evenAlign()
 {
     if (kernel.getOptions()->getTarget() == VISA_3D && kernel.fg.size() > 2)
     {
-        if (kernel.getSimdSize() >= 16)
+        if (kernel.getSimdSize() >= NUM_DWORDS_PER_GRF)
         {
             // Set alignment of all GRF candidates
             // to 2GRF except for NoMask variables

Original file line number	Diff line number	Diff line change
`@@ -130,7 +130,7 @@ void LocalRA::evenAlign()`
`130`	`130`	`{`
`131`	`131`	`if (kernel.getOptions()->getTarget() == VISA_3D && kernel.fg.size() > 2)`
`132`	`132`	`{`
`133`		`- if (kernel.getSimdSize() >= 16)`
	`133`	`+ if (kernel.getSimdSize() >= NUM_DWORDS_PER_GRF)`
`134`	`134`	`{`
`135`	`135`	`// Set alignment of all GRF candidates`
`136`	`136`	`// to 2GRF except for NoMask variables`