Avoid generating private branch for generic accesses

This change fixes an incorrect condition which was causing generation
of branch for private memory even if allocatePrivateAsGlobalBuffer was
enabled.

This change also implements back `-cl-intel-no-local-to-generic` option
which was removed in ee325d492f12192a8fc58f440469d2204da1eeca.

Author: aratajew

IGC/Compiler/CISACodeGen/CastToGASAnalysis.cpp

@@ -7,6 +7,7 @@ SPDX-License-Identifier: MIT
 ============================= end_copyright_notice ===========================*/
 
 #include "Compiler/CISACodeGen/CastToGASAnalysis.h"
+#include "Compiler/CodeGenPublic.h"
 #include "Compiler/CodeGenPublicEnums.h"
 #include "Compiler/IGCPassSupport.h"
 #include "Probe/Assertion.h"
@@ -127,6 +128,13 @@ void CastToGASWrapperPass::setInfoForGroup(
 
 bool CastToGASWrapperPass::runOnModule(Module& M)
 {
+    m_ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
+    GI.noLocalToGenericOptionEnabled = m_ctx->noLocalToGenericOptionEnabled();
+    GI.allocatePrivateAsGlobalBuffer = m_ctx->allocatePrivateAsGlobalBuffer();
+
+    if (GI.noLocalToGenericOptionEnabled && GI.allocatePrivateAsGlobalBuffer)
+        return false;
+
     castInfoCache.clear();
     CallGraph& CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
     for (auto& F : M.functions()) {

IGC/Compiler/CISACodeGen/CastToGASAnalysis.h

@@ -14,6 +14,8 @@ SPDX-License-Identifier: MIT
 #include <llvm/Analysis/CallGraph.h>
 #include "common/LLVMWarningsPop.hpp"
 
+#include "Compiler/CodeGenContextWrapper.hpp"
+
 namespace IGC
 {
     enum {
@@ -24,6 +26,9 @@ namespace IGC
     class GASInfo {
     public:
         bool canGenericPointToPrivate(llvm::Function& F) const {
+            if (allocatePrivateAsGlobalBuffer)
+                return false;
+
             auto E = FunctionMap.find(&F);
             if (E == FunctionMap.end())
                 return true;
@@ -32,6 +37,9 @@ namespace IGC
         }
 
         bool canGenericPointToLocal(llvm::Function& F) const {
+            if (noLocalToGenericOptionEnabled)
+                return false;
+
             auto E = FunctionMap.find(&F);
             if (E == FunctionMap.end())
                 return true;
@@ -42,6 +50,10 @@ namespace IGC
         using FunctionMapTy = llvm::DenseMap<const llvm::Function*, unsigned>;
         FunctionMapTy FunctionMap;
 
+        // True when -cl-intel-no-local-to-generic is enabled
+        bool noLocalToGenericOptionEnabled = false;
+        bool allocatePrivateAsGlobalBuffer = false;
+
         friend class CastToGASWrapperPass;
     };
 
@@ -63,11 +75,13 @@ namespace IGC
         void getAnalysisUsage(llvm::AnalysisUsage & AU) const override {
             AU.setPreservesAll();
             AU.addRequired<llvm::CallGraphWrapperPass>();
+            AU.addRequired<CodeGenContextWrapper>();
         }
 
         GASInfo& getGASInfo() { return GI; }
 
     private:
+        CodeGenContext* m_ctx = nullptr;
         GASInfo GI;
         llvm::DenseMap<const llvm::Function*, unsigned> castInfoCache;
 

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8383,10 +8383,9 @@ void EmitPass::emitAddrSpaceCast(llvm::AddrSpaceCastInst* addrSpaceCast)
 
     CVariable* srcV = GetSymbol(addrSpaceCast->getOperand(0));
 
-    if ((m_pCtx->allocatePrivateAsGlobalBuffer() || !m_canGenericPointToPrivate) &&
-        !m_canGenericPointToLocal)
+    if (!m_canGenericPointToPrivate && !m_canGenericPointToLocal)
     {
-        // If forcing global memory allocacion and there are no generic pointers to local AS,
+        // If forcing global memory allocation and there are no generic pointers to local AS,
         // there is no need to tag generic pointers.
         m_encoder->Cast(m_destination, srcV);
         m_encoder->Push();

IGC/Compiler/CISACodeGen/ResolveGAS.cpp

@@ -1430,6 +1430,7 @@ void LowerGPCallArg::processGASInst(Module& M)
         return true;
     };
 
+    bool changed = false;
     IRBuilder<> IRB(M.getContext());
     // Change GAS inst, such as ld/st, etc to global ld/st, etc.
     for (Function& F : M)
@@ -1462,27 +1463,31 @@ void LowerGPCallArg::processGASInst(Module& M)
                         tI->setDebugLoc(I->getDebugLoc());
                     }
 
-                    m_changed = true;
-                }
-            }
-            else if (CallInst* CallI = dyn_cast<CallInst>(I))
-            {
-                Function* Callee = CallI->getCalledFunction();
-                if (Callee &&
-                    (Callee->getName().equals("__builtin_IB_to_local") ||
-                     Callee->getName().equals("__builtin_IB_to_private")) &&
-                    !toSkip(CallI->getOperand(0)))
-                {
-                    Type* DstTy = I->getType();
-                    Value* NewPtr = Constant::getNullValue(DstTy);
-                    I->replaceAllUsesWith(NewPtr);
-                    I->eraseFromParent();
-
-                    m_changed = true;
+                    changed = true;
                 }
             }
         }
     }
+
+    if (changed)
+    {
+        // Above optimization changes an addrspace of load and store instructions
+        // operating on a generic pointer. One of the cases when optimization may
+        // be applied is when private memory is allocated in a global buffer. Together
+        // with an information that local pointers are not casted to generic addrspace,
+        // compiler is assured that all generic pointers point to a global memory.
+        // If the optimization happens, usage of generic pointer in any load and store
+        // instructions disappear.
+        //
+        // m_mustAllocatePrivateAsGlobalBuffer variable is necessary to point out that
+        // above optimization has been applied. Since PrivateMemoryResolution pass
+        // allocates private memory in a global buffer only if there is any load or
+        // store operating on a generic addrspace, the above optimization could mislead
+        // the logic in PrivateMemoryResolution causing private memory not being
+        // allocated in a global buffer.
+        m_ctx->m_mustAllocatePrivateAsGlobalBuffer = true;
+        m_changed = true;
+    }
 }
 
 std::vector<Function*> LowerGPCallArg::findCandidates(CallGraph& CG)

IGC/Compiler/CodeGenContext.cpp

@@ -567,7 +567,15 @@ namespace IGC
 
     bool OpenCLProgramContext::allocatePrivateAsGlobalBuffer() const
     {
-        return forceGlobalMemoryAllocation() || (m_instrTypes.hasDynamicGenericLoadStore && platform.canForcePrivateToGlobal());
+        return forceGlobalMemoryAllocation() ||
+            (m_instrTypes.hasDynamicGenericLoadStore &&
+                platform.canForcePrivateToGlobal()) ||
+            m_mustAllocatePrivateAsGlobalBuffer;
+    }
+
+    bool OpenCLProgramContext::noLocalToGenericOptionEnabled() const
+    {
+        return m_InternalOptions.NoLocalToGeneric;
     }
 
     bool OpenCLProgramContext::enableTakeGlobalAddress() const
@@ -778,6 +786,11 @@ namespace IGC
             {
                 IntelEnablePreRAScheduling = false;
             }
+            // -cl-intel-no-local-to-generic
+            else if (suffix.equals("-no-local-to-generic"))
+            {
+                NoLocalToGeneric = true;
+            }
             // -cl-intel-force-global-mem-allocation
             else if (suffix.equals("-force-global-mem-allocation"))
             {
@@ -1421,6 +1434,11 @@ namespace IGC
         return false;
     }
 
+    bool CodeGenContext::noLocalToGenericOptionEnabled() const
+    {
+        return false;
+    }
+
     bool CodeGenContext::enableTakeGlobalAddress() const
     {
         return false;

IGC/Compiler/CodeGenPublic.h

@@ -1090,6 +1090,8 @@ namespace IGC
         // When this is true, the flag "ForceGlobalMemoryAllocation" is enabled as a WA
         bool m_hasGlobalInPrivateAddressSpace = false;
 
+        bool m_mustAllocatePrivateAsGlobalBuffer = false;
+
         /////  used for instruction statistic before/after pass
         int instrStat[TOTAL_TYPES][TOTAL_STAGE];
 
@@ -1264,6 +1266,7 @@ namespace IGC
         virtual uint32_t getNumGRFPerThread() const;
         virtual bool forceGlobalMemoryAllocation() const;
         virtual bool allocatePrivateAsGlobalBuffer() const;
+        virtual bool noLocalToGenericOptionEnabled() const;
         virtual bool enableTakeGlobalAddress() const;
         virtual int16_t getVectorCoalescingControl() const;
         virtual uint32_t getPrivateMemoryMinimalSizePerThread() const;
@@ -1765,6 +1768,7 @@ namespace IGC
             bool CompileOneKernelAtTime = false;
 
             // Generic address related
+            bool NoLocalToGeneric = false;
             bool ForceGlobalMemoryAllocation = false;
 
             // -1 : initial value that means it is not set from cmdline
@@ -1915,6 +1919,7 @@ namespace IGC
         uint32_t getNumThreadsPerEU() const override;
         bool forceGlobalMemoryAllocation() const override;
         bool allocatePrivateAsGlobalBuffer() const override;
+        bool noLocalToGenericOptionEnabled() const override;
         bool enableTakeGlobalAddress() const override;
         int16_t getVectorCoalescingControl() const override;
         uint32_t getPrivateMemoryMinimalSizePerThread() const override;

IGC/Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/GenericAddressDynamicResolution.cpp

@@ -220,8 +220,7 @@ bool GenericAddressDynamicResolution::visitLoadStoreInst(Instruction& I)
     }
 
     if (pointerAddressSpace == ADDRESS_SPACE_GENERIC) {
-        if((m_ctx->allocatePrivateAsGlobalBuffer() || !m_needPrivateBranches) &&
-            !m_needLocalBranches)
+        if(!m_needPrivateBranches && !m_needLocalBranches)
         {
             resolveGASWithoutBranches(I, pointerOperand);
         }
@@ -294,7 +293,7 @@ void GenericAddressDynamicResolution::resolveGAS(Instruction& I, Value* pointerO
     // GAS needs to resolve to private only if
     //     1) private is NOT allocated in global space; and
     //     2) there is a cast from private to GAS.
-    bool needPrivateBranch = !(m_ctx->allocatePrivateAsGlobalBuffer()) || m_needPrivateBranches;
+    bool needPrivateBranch = m_needPrivateBranches;
     bool needLocalBranch = m_needLocalBranches;
 
     auto createBlock = [&](const Twine& BlockName, const Twine& LoadName, IGC::ADDRESS_SPACE addressSpace, Value*& load)

IGC/Compiler/tests/GenericAddressDynamicResolution/CastToGASAnalysis-cyclic-call-graph.ll

@@ -31,14 +31,9 @@ define spir_func void @f2(i32 addrspace(4)* %ptr) {
     ; CHECK: %[[PTI:.*]] = ptrtoint i32 addrspace(4)* %ptr to i64
     ; CHECK: %[[TAG:.*]] = lshr i64 %1, 61
     ; CHECK: switch i64 %2, label %GlobalBlock [
-    ; CHECK:   i64 1, label %PrivateBlock
     ; CHECK:   i64 2, label %LocalBlock
     ; CHECK: ]
 
-    ; CHECK: PrivateBlock:
-    ; CHECK:   %[[PRIVATE_PTR:.*]] = addrspacecast i32 addrspace(4)* %ptr to i32*
-    ; CHECK:   store i32 123, i32* %[[PRIVATE_PTR]], align 4
-
     ; CHECK: LocalBlock:
     ; CHECK:   %[[LOCAL_PTR:.*]] = addrspacecast i32 addrspace(4)* %ptr to i32 addrspace(3)*
     ; CHECK:   store i32 123, i32 addrspace(3)* %[[LOCAL_PTR]], align 4

IGC/Compiler/tests/GenericAddressDynamicResolution/CastToGASAnalysis-referenced-indirectly.ll

@@ -27,14 +27,9 @@ define spir_kernel void @kernel(i32 addrspace(1)* %global_buffer) {
   ; CHECK: %[[PTI:.*]] = ptrtoint i32 addrspace(4)* %generic_ptr to i64
   ; CHECK: %[[TAG:.*]] = lshr i64 %[[PTI]], 61
   ; CHECK: switch i64 %[[TAG]], label %GlobalBlock [
-  ; CHECK:   i64 1, label %PrivateBlock
   ; CHECK:   i64 2, label %LocalBlock
   ; CHECK: ]
 
-  ; CHECK: PrivateBlock:
-  ; CHECK:   %[[PRIVATE_PTR:.*]] = addrspacecast i32 addrspace(4)* %generic_ptr to i32*
-  ; CHECK:   store i32 5, i32* %[[PRIVATE_PTR]], align 4
-
   ; CHECK: LocalBlock:
   ; CHECK:   %[[LOCAL_PTR:.*]] = addrspacecast i32 addrspace(4)* %generic_ptr to i32 addrspace(3)*
   ; CHECK:   store i32 5, i32 addrspace(3)* %[[LOCAL_PTR]], align 4

IGC/Compiler/tests/GenericAddressDynamicResolution/CastToGASAnalysis-simple.ll

@@ -41,14 +41,9 @@ return:
   ; CHECK: %[[PTI:.*]] = ptrtoint i32 addrspace(4)* %generic_ptr to i64
   ; CHECK: %[[TAG:.*]] = lshr i64 %[[PTI]], 61
   ; CHECK: switch i64 %[[TAG]], label %GlobalBlock [
-  ; CHECK:   i64 1, label %PrivateBlock
   ; CHECK:   i64 2, label %LocalBlock
   ; CHECK: ]
 
-  ; CHECK: PrivateBlock:
-  ; CHECK:   %[[PRIVATE_PTR:.*]] = addrspacecast i32 addrspace(4)* %generic_ptr to i32*
-  ; CHECK:   store i32 5, i32* %[[PRIVATE_PTR]], align 4
-
   ; CHECK: LocalBlock:
   ; CHECK:   %[[LOCAL_PTR:.*]] = addrspacecast i32 addrspace(4)* %generic_ptr to i32 addrspace(3)*
   ; CHECK:   store i32 5, i32 addrspace(3)* %[[LOCAL_PTR]], align 4