Add tgm cache control with registry keys

fftzeng · igcbot · commit 2102e7e739e8 · 2022-07-01T09:35:25.000+02:00
diff --git a/IGC/Compiler/CISACodeGen/CISABuilder.cpp b/IGC/Compiler/CISACodeGen/CISABuilder.cpp
@@ -8368,6 +8368,7 @@ namespace IGC
     }
 
     void CEncoder::LSC_TypedReadWrite(
+        LSC_CACHE_OPTS cacheOpts,
         LSC_OP subOp,
         ResourceDescriptor* resource,
         CVariable* pU,
@@ -8404,7 +8405,7 @@ namespace IGC
 
         VISA_EMask_Ctrl mask = ConvertMaskToVisaType(m_encoderState.m_mask, m_encoderState.m_noMask);
         VISA_VectorOpnd* globalOffsetOpnd = GetVISALSCSurfaceOpnd(resource->m_surfaceType, resource->m_resource);
-        LSC_CACHE_OPTS cache{ LSC_CACHING_DEFAULT, LSC_CACHING_DEFAULT };
+
         LSC_DATA_SHAPE dataShape{};
         dataShape.size = LSC_GetElementSize(elemSize);
         dataShape.order = LSC_DATA_ORDER_NONTRANSPOSE;
@@ -8416,7 +8417,7 @@ namespace IGC
             predOpnd,
             execSize,
             mask,
-            cache,
+            cacheOpts,
             getLSCAddrType(resource),
             addr_size,
             dataShape,
diff --git a/IGC/Compiler/CISACodeGen/CISABuilder.hpp b/IGC/Compiler/CISACodeGen/CISABuilder.hpp
@@ -297,6 +297,7 @@ namespace IGC
             CVariable* flatImageHeight, CVariable* flatImagePitch);
         void NamedBarrier(e_barrierKind BarrierKind, CVariable* src0, CVariable* src1);
         void LSC_TypedReadWrite(
+            LSC_CACHE_OPTS cacheOpts,
             LSC_OP subOp, ResourceDescriptor* resource,
             CVariable* pU, CVariable* pV, CVariable* pR, CVariable* pLOD,
             CVariable* pSrcDst,
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.cpp
@@ -8767,7 +8767,7 @@ void EmitPass::emitLoadRawIndexed(
             setRovCacheCtrl(inst);
         }
         LSC_CACHE_OPTS cacheOpts =
-            translateLSCCacheControlsFromMetadata(inst, true);
+            translateLSCCacheControlsFromMetadata(inst, true, false);
         emitLSCVectorLoad(
             bufPtrv,
             varOffset,
@@ -9075,7 +9075,7 @@ void EmitPass::emitLoad(LoadInst* inst, Value* offset, ConstantInt* immOffset)
     {
         offset = immOffset ? offset : inst->getPointerOperand();
         LSC_CACHE_OPTS cacheOpts =
-            translateLSCCacheControlsFromMetadata(inst, true);
+            translateLSCCacheControlsFromMetadata(inst, true, false);
         emitLSCVectorLoad(
             inst->getPointerOperand(),
             offset,
@@ -10031,7 +10031,7 @@ void EmitPass::emitStoreRawIndexed(
     if (shouldGenerateLSC(inst))
     {
         LSC_CACHE_OPTS cacheOpts =
-            translateLSCCacheControlsFromMetadata(inst, false);
+            translateLSCCacheControlsFromMetadata(inst, false, false);
         emitLSCVectorStore(
             pBufPtr,
             varOffset,
@@ -10194,7 +10194,7 @@ void EmitPass::emitStore(StoreInst* inst, Value* varOffset, ConstantInt* immOffs
     if (shouldGenerateLSC(inst))
     {
         LSC_CACHE_OPTS cacheOpts =
-            translateLSCCacheControlsFromMetadata(inst, false);
+            translateLSCCacheControlsFromMetadata(inst, false, false);
         emitLSCVectorStore(
             inst->getPointerOperand(),
             varOffset,
@@ -13066,7 +13066,7 @@ void EmitPass::emitTypedRead(llvm::Instruction* pInsn)
     pLOD = pLOD ? BroadcastIfUniform(pLOD, m_currShader->GetIsUniform(pInsn)) : nullptr;
 
     ResourceDescriptor resource = GetResourceVariable(pllSrcBuffer);
-
+    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(pInsn, true, true);
     uint numChannels = iSTD::BitCount(writeMask.getEM());
     auto doLSC = shouldGenerateLSC(pInsn);
 
@@ -13085,7 +13085,7 @@ void EmitPass::emitTypedRead(llvm::Instruction* pInsn)
 
         if (doLSC)
         {
-            m_encoder->LSC_TypedReadWrite(LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, tempdst, 4 * 8,
+            m_encoder->LSC_TypedReadWrite(cacheOpts, LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, tempdst, 4 * 8,
                 numLanes(nativeDispatchMode), LSC_ADDR_SIZE_32b, writeMask.getEM());
         }
         else
@@ -13129,7 +13129,7 @@ void EmitPass::emitTypedRead(llvm::Instruction* pInsn)
 
             if (doLSC)
             {
-                m_encoder->LSC_TypedReadWrite(LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, m_destination, 4 * 8,
+                m_encoder->LSC_TypedReadWrite(cacheOpts, LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, m_destination, 4 * 8,
                     numLanes(SIMDMode::SIMD16), LSC_ADDR_SIZE_32b, writeMask.getEM());
             }
             else
@@ -13161,7 +13161,7 @@ void EmitPass::emitTypedRead(llvm::Instruction* pInsn)
 
                 if (doLSC)
                 {
-                    m_encoder->LSC_TypedReadWrite(LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, tempdst[i], 4 * 8,
+                    m_encoder->LSC_TypedReadWrite(cacheOpts, LSC_LOAD_QUAD, &resource, pU, pV, pR, pLOD, tempdst[i], 4 * 8,
                         numLanes(SIMDMode::SIMD16), LSC_ADDR_SIZE_32b, writeMask.getEM());
                 }
                 else
@@ -13209,6 +13209,7 @@ void EmitPass::emitTypedWrite(llvm::Instruction* pInsn)
     pV = pV ? BroadcastIfUniform(pV) : nullptr;
     pR = pR ? BroadcastIfUniform(pR) : nullptr;
     pLOD = pLOD ? BroadcastIfUniform(pLOD) : nullptr;
+    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(pInsn, false, true);
 
     uint writeMask =
         (!llvm::isa<UndefValue>(pllSrc_X) ? 1 : 0) |
@@ -18720,21 +18721,37 @@ EmitPass::setCacheOptionsForConstantBufferLoads(Instruction& inst) const
     return cacheOpts;
 }
 
-static bool tryOverrideCacheOpts(LSC_CACHE_OPTS& cacheOpts, bool isLoad)
+static bool tryOverrideCacheOpts(LSC_CACHE_OPTS& cacheOpts, bool isLoad, bool isTGM)
 {
-    uint32_t l1l3CacheVal = isLoad ?
-        IGC_GET_FLAG_VALUE(LscLoadCacheControlOverride) :
-        IGC_GET_FLAG_VALUE(LscStoreCacheControlOverride);
-    if (l1l3CacheVal != 0)
+    if (isTGM)
     {
-        cacheOpts = translateLSCCacheControlsEnum(
-            static_cast<LSC_L1_L3_CC>(l1l3CacheVal), isLoad);
+        uint32_t tgmCacheVal = isLoad ?
+            IGC_GET_FLAG_VALUE(TgmLoadCacheControlOverride) :
+            IGC_GET_FLAG_VALUE(TgmStoreCacheControlOverride);
+        if (tgmCacheVal != 0)
+        {
+            cacheOpts = translateLSCCacheControlsEnum(
+                static_cast<LSC_L1_L3_CC>(tgmCacheVal), isLoad);
+        }
+        return tgmCacheVal != 0;
+
+    }
+    else
+    {
+        uint32_t l1l3CacheVal = isLoad ?
+            IGC_GET_FLAG_VALUE(LscLoadCacheControlOverride) :
+            IGC_GET_FLAG_VALUE(LscStoreCacheControlOverride);
+        if (l1l3CacheVal != 0)
+        {
+            cacheOpts = translateLSCCacheControlsEnum(
+                static_cast<LSC_L1_L3_CC>(l1l3CacheVal), isLoad);
+        }
+        return l1l3CacheVal != 0;
     }
-    return l1l3CacheVal != 0;
 }
 
 LSC_CACHE_OPTS EmitPass::translateLSCCacheControlsFromMetadata(
-    Instruction* inst, bool isLoad) const
+    Instruction* inst, bool isLoad, bool isTGM) const
 {
     LSC_CACHE_OPTS cacheOpts{ LSC_CACHING_DEFAULT, LSC_CACHING_DEFAULT };
 
@@ -18773,7 +18790,7 @@ LSC_CACHE_OPTS EmitPass::translateLSCCacheControlsFromMetadata(
         }
     }
 
-    if (tryOverrideCacheOpts(cacheOpts, isLoad))
+    if (tryOverrideCacheOpts(cacheOpts, isLoad, isTGM))
     {
         // global override cache settings have highest priority
         return cacheOpts;
@@ -19215,7 +19232,7 @@ void EmitPass::emitLSCLoad(
     LSC_DATA_ORDER data_order,
     int immOffset)
 {
-    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, true);
+    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, true, false);
     emitLSCLoad(cacheOpts, dst, offset, elemSize, numElems, blockOffset,
                 resource, addr_size, data_order, immOffset);
 }
@@ -19251,7 +19268,7 @@ void EmitPass::emitLSCStore(
     LSC_DATA_ORDER data_order,
     int immOffset)
 {
-    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
+    LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false, false);
     emitLSCStore(cacheOpts, src, offset, elemSize, numElems, blockOffset,
                  resource, addr_size, data_order, immOffset);
 }
diff --git a/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp b/IGC/Compiler/CISACodeGen/EmitVISAPass.hpp
@@ -460,7 +460,7 @@ class EmitPass : public llvm::FunctionPass
     LSC_CACHE_OPTS translateLSCCacheControlsFromValue(
         llvm::Value *value, bool isLoad) const;
     LSC_CACHE_OPTS translateLSCCacheControlsFromMetadata(
-        llvm::Instruction* inst, bool isLoad) const;
+        llvm::Instruction* inst, bool isLoad, bool isTGM) const;
     struct LscMessageFragmentInfo {
         LSC_DATA_ELEMS fragElem;
         int            fragElemCount;
diff --git a/IGC/common/igc_flags.h b/IGC/common/igc_flags.h
@@ -503,6 +503,8 @@ DECLARE_IGC_REGKEY(bool, Enable_Wa22010493955, false, "Enable Wa_22010493955 reg
 DECLARE_IGC_REGKEY(bool, EnablePartialEmuI64,  true,  "Enable the partial I64 emulation for PVC-B", true)
 DECLARE_IGC_REGKEY_ENUM(LscLoadCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC loads.", LSC_CACHE_CTRL_OPTIONS, true)
 DECLARE_IGC_REGKEY_ENUM(LscStoreCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC stores.", LSC_CACHE_CTRL_OPTIONS, true)
+DECLARE_IGC_REGKEY_ENUM(TgmLoadCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC tgm loads.", LSC_CACHE_CTRL_OPTIONS, true)
+DECLARE_IGC_REGKEY_ENUM(TgmStoreCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC tgm stores.", LSC_CACHE_CTRL_OPTIONS, true)
 DECLARE_IGC_REGKEY(bool, LscForceSpillNonStackcall, false, "Non-stack call kernels that spill will use LSC on DG2+", true)
 DECLARE_IGC_REGKEY(bool, EnableQWAddSupport, true, "Enable QW Add support", true)
 DECLARE_IGC_REGKEY(bool, ForceQWAddSupport, false, "Force enabling the QW Add support along with ForcePartialInt64", true)
