In synchronous raytracing, emulate LSC fence with LSC load

In synchronous raytracing, emulate LSC fence with LSC load

Author: jaladreips

IGC/AdaptorCommon/RayTracing/AutoGenRTStackAccessPrivateOS.h

@@ -1832,3 +1832,12 @@ auto* _getSyncStackID_Xe2(const Twine& _ReturnName = "")
   return V_6;
 }
 
+auto* _getHitAddress_Xe(Value* arg_0, Value* arg_1, const Twine& _ReturnName = "")
+{
+  auto* V_2 = CreateInBoundsGEP(_struct_RTStackFormat__RTStack(*Ctx.getModule()), arg_0, { getInt64(0), getInt32(0) });
+  auto* V_3 = CreateInBoundsGEP(_struct_RTStackFormat__RTStack(*Ctx.getModule()), arg_0, { getInt64(0), getInt32(1) });
+  auto* V_4 = CreateSelect(arg_1, V_2, V_3);
+  auto* V_5 = CreateBitCast(V_4, PointerType::get(getInt8Ty(), arg_0->getType()->getPointerAddressSpace()), _ReturnName);
+  return V_5;
+}
+

IGC/AdaptorCommon/RayTracing/RTBuilder.cpp

@@ -1994,3 +1994,17 @@ void RTBuilder::commitProceduralPrimitiveHit(
 #undef STYLE
     }
 }
+
+Value* RTBuilder::getHitAddress(StackPointerVal* StackPtr, bool Committed)
+{
+    switch (getMemoryStyle())
+    {
+#define STYLE(X)           \
+    case RTMemoryStyle::X: \
+        return _getHitAddress_##X(StackPtr, VAdapt{ *this, Committed });
+#include "RayTracingMemoryStyle.h"
+#undef STYLE
+    }
+    IGC_ASSERT(0);
+    return {};
+}

IGC/AdaptorCommon/RayTracing/RTBuilder.h

@@ -496,6 +496,8 @@ class RTBuilder : public IGCIRBuilder<>
     Value* getCandidateType(SyncStackPointerVal* SMStackPtr);
 
     void commitProceduralPrimitiveHit(SyncStackPointerVal* SMStackPtr, Value* THit);
+
+    Value* getHitAddress(StackPointerVal* StackPtr, bool Committed);
 };
 
 } // namespace llvm

IGC/AdaptorCommon/RayTracing/TraceRayInlineLoweringPass.cpp

@@ -361,7 +361,36 @@ Value* TraceRayInlineLoweringPass::emitProceedMainBody(
     Value* ShdowMemRTCtrlPtr = getShMemRTCtrl(builder, queryObjIndex);
     Value* traceRayCtrl = builder.getSyncTraceRayControl(ShdowMemRTCtrlPtr);
 
-    builder.CreateLSCFence(LSC_UGM, LSC_SCOPE_LOCAL, LSC_FENCE_OP_NONE);
+    if (IGC_IS_FLAG_ENABLED(DisableLoadAsFenceOpInRaytracing))
+    {
+        builder.CreateLSCFence(LSC_UGM, LSC_SCOPE_LOCAL, LSC_FENCE_OP_NONE);
+    }
+    else
+    {
+        // this is an optimization
+        // it's based on the idea that stores and loads are queued, so if a load completes, all stores before it are also completed
+        // the requirement is that the load and the store should use the same address, so we use the potential hit (last write in copyMemHitInProceed)
+        auto* potentialHit = builder.getHitAddress(HWStackPointer, false);
+
+        auto* M = builder.GetInsertPoint()->getModule();
+        auto* fn = GenISAIntrinsic::getDeclaration(
+            M,
+            GenISAIntrinsic::GenISA_LSCLoadWithSideEffects,
+            { builder.getInt32Ty(), potentialHit->getType() }
+        );
+
+        builder.CreateCall(
+            fn,
+            {
+                potentialHit,
+                builder.getInt32(0),
+                builder.getInt32(LSC_DATA_SIZE_32b),
+                builder.getInt32(LSC_DATA_ELEMS_1),
+                builder.getInt32(LSC_L1C_WT_L3C_WB)
+            },
+            VALUE_NAME("LSCLoadAsFence")
+        );
+    }
 
     //TraceRay
     Value* retSyncRT = builder.createSyncTraceRay(
@@ -470,9 +499,9 @@ void TraceRayInlineLoweringPass::LowerSyncStackToShadowMemory(Function& F)
 
         if (IGC_IS_FLAG_DISABLED(DisableInvalidateRTStackAfterLastRead))
         {
-            auto* LSCInvalidate = GenISAIntrinsic::getDeclaration(
+            auto* fn = GenISAIntrinsic::getDeclaration(
                 F.getParent(),
-                GenISAIntrinsic::GenISA_LSCInvalidate,
+                GenISAIntrinsic::GenISA_LSCLoadWithSideEffects,
                 { builder.getInt32Ty(), HWStackPointer->getType() }
             );
 
@@ -487,7 +516,7 @@ void TraceRayInlineLoweringPass::LowerSyncStackToShadowMemory(Function& F)
             for (uint i = 0; i < getSyncStackSize() / m_CGCtx->platform.LSCCachelineSize(); i++)
             {
                 builder.CreateCall(
-                    LSCInvalidate,
+                    fn,
                     {
                         HWStackPointer,
                         builder.getInt32(i * m_CGCtx->platform.LSCCachelineSize()),

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8980,7 +8980,7 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst* inst)
     case GenISAIntrinsic::GenISA_LSCStore:
     case GenISAIntrinsic::GenISA_LSCStoreBlock:
     case GenISAIntrinsic::GenISA_LSCLoad:
-    case GenISAIntrinsic::GenISA_LSCInvalidate:
+    case GenISAIntrinsic::GenISA_LSCLoadWithSideEffects:
     case GenISAIntrinsic::GenISA_LSCLoadBlock:
     case GenISAIntrinsic::GenISA_LSCStoreCmask:
     case GenISAIntrinsic::GenISA_LSCLoadCmask:
@@ -23151,7 +23151,7 @@ void EmitPass::emitLSCIntrinsic(llvm::GenIntrinsicInst* GII)
     GenISAIntrinsic::ID iid = GII->getIntrinsicID();
     switch (iid) {
     case GenISAIntrinsic::GenISA_LSCLoad:
-    case GenISAIntrinsic::GenISA_LSCInvalidate:
+    case GenISAIntrinsic::GenISA_LSCLoadWithSideEffects:
     case GenISAIntrinsic::GenISA_LSCLoadBlock:
         emitLscIntrinsicLoad(GII);
         break;

IGC/Compiler/CISACodeGen/helper.cpp

@@ -1246,7 +1246,7 @@ namespace IGC
             case GenISAIntrinsic::GenISA_simdBlockRead:
             case GenISAIntrinsic::GenISA_simdBlockWrite:
             case GenISAIntrinsic::GenISA_LSCLoad:
-            case GenISAIntrinsic::GenISA_LSCInvalidate:
+            case GenISAIntrinsic::GenISA_LSCLoadWithSideEffects:
             case GenISAIntrinsic::GenISA_LSCLoadBlock:
             case GenISAIntrinsic::GenISA_LSCLoadCmask:
                 pBuffer = intr->getOperand(0);

IGC/GenISAIntrinsics/Intrinsic_definitions.py

@@ -2871,7 +2871,7 @@
      ("int",                           "cache controls options (LSC_CACHE_OPTS)")],
     "ReadMem"]],
 ####################################################################################################
-"GenISA_LSCInvalidate": ["LSC load meant to be used for cache invalidation",
+"GenISA_LSCLoadWithSideEffects": ["LSC load meant to be used just for issuing load instruction (i.e. making sure writes complete or to invalidate caches)",
     [("anyint",                        ""),
     [("anyptr",                        "address of value to invalidate"),
      ("int",                           "immediate offset (in bytes)"),

IGC/common/igc_flags.h

@@ -955,6 +955,7 @@ DECLARE_IGC_GROUP("Raytracing Options")
     DECLARE_IGC_REGKEY(bool, EnableFillScheduling, false, "Schedule fills for reduced register pressure", false)
     DECLARE_IGC_REGKEY(bool, DisableSWStackOffsetElision, false, "Avoid loading offseting when known at compile-time", false)
     DECLARE_IGC_REGKEY(DWORD, OverrideTMax, 0, "Force TMax to the given value. When 0, do nothing.", false)
+    DECLARE_IGC_REGKEY(bool, DisableLoadAsFenceOpInRaytracing, true, "Disable load as fence op in raytracing (rayquery only)", false)
 
 DECLARE_IGC_GROUP("VectorCompiler Options")
     DECLARE_IGC_REGKEY(bool, DisableEuFusion, false, "Require disable of EU fusion", true)

IGC/common/igc_regkeys.cpp

@@ -878,6 +878,7 @@ static void setImpliedIGCKeys()
     IGC_SET_IMPLIED_REGKEY(DisableRayTracingOptimizations, 1, DisableDPSE, true);
     IGC_SET_IMPLIED_REGKEY(DisableRayTracingOptimizations, 1, DisableSWStackOffsetElision, true);
     IGC_SET_IMPLIED_REGKEY(DisableRayTracingOptimizations, 1, DisableInvalidateRTStackAfterLastRead, true);
+    IGC_SET_IMPLIED_REGKEY(DisableRayTracingOptimizations, 1, DisableLoadAsFenceOpInRaytracing, true);
 
     IGC_SET_IMPLIED_REGKEY(ForceRTRetry, 1, RetryManagerFirstStateId, 1);