Address patching for zeroed global variables and i64 offsets

fda0 · igcbot · commit 4f7756c73295 · 2023-09-11T10:45:45.000+02:00
Fix size overflows for .data and .bss zebin sections by using
64 bit integers to calculate them.
Use 64 bits to represent offsets into inlineProgramScopeOffsets,
globalBase and constBase to prevent overflows.

Add address patching support for zeroinitialized global variables.
Previously we cannot do the zero-initializer optimization if the variable is used by another global.
This is because the offsets for the zero-init variables have not been calculated at the time of patching.
With this change, we save the patching info, and patch later after those offsets have been calculated.
This enables the optimization to defer copying zero values to the global/const buffers.
diff --git a/IGC/AdaptorOCL/OCL/sp/zebin_builder.cpp b/IGC/AdaptorOCL/OCL/sp/zebin_builder.cpp
@@ -193,9 +193,9 @@ void ZEBinaryBuilder::addGlobalConstants(const IGC::SOpenCLProgramInfo& annotati
     if (annotations.m_initConstantAnnotation && annotations.m_initConstantAnnotation->AllocSize) {
         auto& ca = annotations.m_initConstantAnnotation;
         // the normal .data.const size
-        uint32_t dataSize = ca->InlineData.size();
+        uint64_t dataSize = ca->InlineData.size();
         // the zero-initialize variables size, the .bss.const size
-        uint32_t bssSize = ca->AllocSize - dataSize;
+        uint64_t bssSize = ca->AllocSize - dataSize;
         uint32_t alignment = ca->Alignment;
 
         if (IGC_IS_FLAG_ENABLED(AllocateZeroInitializedVarsInBss)) {
@@ -220,8 +220,9 @@ void ZEBinaryBuilder::addGlobalConstants(const IGC::SOpenCLProgramInfo& annotati
         } else {
             // before runtime can support bss section, we create all 0s in .const.data section by adding
             // bssSize of padding
+            IGC_ASSERT_MESSAGE(bssSize == static_cast<uint32_t>(bssSize), ".const.data padding size overflows 32 bits");
             mGlobalConstSectID = mBuilder.addSectionData("const", (const uint8_t*)ca->InlineData.data(),
-                dataSize, bssSize, alignment, /*rodata*/true);
+                dataSize, static_cast<uint32_t>(bssSize), alignment, /*rodata*/true);
         }
     }
 
@@ -248,8 +249,8 @@ void ZEBinaryBuilder::addGlobals(const IGC::SOpenCLProgramInfo& annotations)
     if (!ca->AllocSize)
         return;
 
-    uint32_t dataSize = ca->InlineData.size();
-    uint32_t bssSize = ca->AllocSize - dataSize;
+    uint64_t dataSize = ca->InlineData.size();
+    uint64_t bssSize = ca->AllocSize - dataSize;
     uint32_t alignment = ca->Alignment;
 
     if (IGC_IS_FLAG_ENABLED(AllocateZeroInitializedVarsInBss)) {
@@ -271,8 +272,9 @@ void ZEBinaryBuilder::addGlobals(const IGC::SOpenCLProgramInfo& annotations)
     } else {
         // before runtime can support bss section, we create all 0s in .global.data section by adding
         // bssSize of padding
+        IGC_ASSERT_MESSAGE(bssSize == static_cast<uint32_t>(bssSize), ".global.data padding size overflows 32 bits");
         mGlobalSectID = mBuilder.addSectionData("global", (const uint8_t*)ca->InlineData.data(),
-            dataSize, bssSize, alignment, /*rodata*/false);
+            dataSize, static_cast<uint32_t>(bssSize), alignment, /*rodata*/false);
     }
 }
 
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantAnalysis.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantAnalysis.cpp
@@ -187,22 +187,8 @@ bool ProgramScopeConstantAnalysis::runOnModule(Module& M)
 
         if (initializer->isZeroValue())
         {
-            // For zero initialized values, we dont need to copy the data, just tell driver how much to allocate
-            // However, if it's used as a pointer value, we need to do patching and therefore cannot defer the offset calculation
-            bool hasPointerUser = false;
-            for (auto UI : globalVar->users())
-            {
-                if (isa<Constant>(UI) && UI->getType()->isPointerTy())
-                {
-                    hasPointerUser = true;
-                    break;
-                }
-            }
-            if (!hasPointerUser)
-            {
-                zeroInitializedGlobals.push_back(globalVar);
-                continue;
-            }
+            zeroInitializedGlobals.push_back(globalVar);
+            continue;
         }
 
         // Align the buffer.
@@ -245,6 +231,22 @@ bool ProgramScopeConstantAnalysis::runOnModule(Module& M)
         offset += (unsigned)(m_DL->getTypeAllocSize(IGCLLVM::getNonOpaquePtrEltTy(globalVar->getType())));
     }
 
+    // Patch the offsets for usages of zero initialized globals after those offsets have been calculated in the previous step.
+    // TODO: Remove this logic after enabling ZeBinary, since we will switch the patching to use relocation table instead.
+    for (ZeroInitPatchInfo &patchData : m_PatchLaterDataVector)
+    {
+        char* whereToPatch = std::get<0>(patchData);
+        unsigned patchSize = std::get<1>(patchData);
+        GlobalVariable* globalVar = std::get<2>(patchData);
+        uint64_t offset = std::get<3>(patchData);
+
+        auto iter = inlineProgramScopeOffsets.find(globalVar);
+        IGC_ASSERT(iter != inlineProgramScopeOffsets.end());
+
+        const uint64_t patchOffset = iter->second + offset;
+        memcpy_s(whereToPatch, patchSize, (char*)&patchOffset, patchSize);
+    }
+
     if (inlineProgramScopeOffsets.size())
     {
         // Add globals tracked in metadata to the "llvm.used" list so they won't be deleted by optimizations
@@ -346,7 +348,7 @@ bool ProgramScopeConstantAnalysis::runOnModule(Module& M)
     const bool changed = !inlineProgramScopeOffsets.empty();
     for (auto offset : inlineProgramScopeOffsets)
     {
-        m_pModuleMd->inlineProgramScopeOffsets[offset.first] = offset.second;
+        m_pModuleMd->inlineProgramScopeOffsets[offset.first] = static_cast<uint64_t>(offset.second);
     }
 
     // Update LLVM metadata based on IGC MetadataUtils
@@ -476,20 +478,26 @@ void ProgramScopeConstantAnalysis::addData(Constant* initializer,
                 }
                 else
                 {
-                    auto iter = inlineProgramScopeOffsets.find(ptrBase);
-                    IGC_ASSERT(iter != inlineProgramScopeOffsets.end());
-
-                    const uint64_t pointeeOffset = iter->second + offset;
+                    // Add the patching info for runtime
+                    pointerOffsetInfoList.push_back(PointerOffsetInfo(addressSpace, inlineProgramScopeBuffer.size(), pointedToAddrSpace));
 
-                    pointerOffsetInfoList.push_back(
-                        PointerOffsetInfo(
-                            addressSpace,
-                            inlineProgramScopeBuffer.size(),
-                            pointedToAddrSpace));
-
-                    // For old patching logic, write the offset relative to the entire global/constant buffer where the base global resides.
-                    // The base address of the buffer will be added to it at runtime.
-                    inlineProgramScopeBuffer.insert(inlineProgramScopeBuffer.end(), (char*)&pointeeOffset, ((char*)&pointeeOffset) + pointerSize);
+                    auto iter = inlineProgramScopeOffsets.find(ptrBase);
+                    if (iter != inlineProgramScopeOffsets.end())
+                    {
+                        const uint64_t pointeeOffset = iter->second + offset;
+                        // For old patching logic, write the offset relative to the entire global/constant buffer where the base global resides.
+                        // The base address of the buffer will be added to it at runtime.
+                        inlineProgramScopeBuffer.insert(inlineProgramScopeBuffer.end(), (char*)&pointeeOffset, ((char*)&pointeeOffset) + pointerSize);
+                    }
+                    else
+                    {
+                        // If we can't find the base global variable, it must be a zero initialized value whose data is not directly copied.
+                        // Save the info for now, and patch it later once we have the offsets for this zeroinit global vars.
+                        unsigned patchIdx = inlineProgramScopeBuffer.size();
+                        inlineProgramScopeBuffer.insert(inlineProgramScopeBuffer.end(), pointerSize, 0);
+
+                        m_PatchLaterDataVector.push_back(std::make_tuple((char*)(&inlineProgramScopeBuffer[patchIdx]), pointerSize, ptrBase, offset));
+                    }
                 }
             }
             else
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantAnalysis.hpp b/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantAnalysis.hpp
@@ -54,7 +54,7 @@ namespace IGC
 
     protected:
         typedef std::vector<unsigned char> DataVector;
-        typedef llvm::MapVector<llvm::GlobalVariable*, int> BufferOffsetMap;
+        typedef llvm::MapVector<llvm::GlobalVariable*, int64_t> BufferOffsetMap;
 
         struct PointerOffsetInfo
         {
@@ -91,6 +91,10 @@ namespace IGC
 
         const llvm::DataLayout* m_DL;
         ModuleMetaData* m_pModuleMd;
+
+        // Used to patch offsets for zero initialized globals
+        typedef std::tuple<char*, unsigned, llvm::GlobalVariable*, uint64_t> ZeroInitPatchInfo;
+        std::vector<ZeroInitPatchInfo> m_PatchLaterDataVector;
     };
 
 } // namespace IGC
diff --git a/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantResolution.cpp b/IGC/Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantResolution.cpp
@@ -126,22 +126,14 @@ bool ProgramScopeConstantResolution::runOnModule(Module& M)
             continue;
         }
 
-        // Get the offset of this constant from the base.
-        int offset = -1;
-
         auto bufferOffset = modMD->inlineProgramScopeOffsets.find(pGlobalVar);
-        if (bufferOffset != modMD->inlineProgramScopeOffsets.end())
-        {
-            offset = bufferOffset->second;
-        }
+        if (bufferOffset == modMD->inlineProgramScopeOffsets.end())
+            continue; // This constant is not used, so it didn't get an offset.
 
-        // This constant is not used, so it didn't get an offset.
-        if (offset == -1)
-        {
-            continue;
-        }
+        // Get the offset of this constant from the base.
+        int64_t offset = bufferOffset->second;
+        ConstantInt* pOffset = ConstantInt::get(Type::getInt64Ty(C), offset);
 
-        ConstantInt* pOffset = ConstantInt::get(Type::getInt32Ty(C), offset);
         const ImplicitArg::ArgType argType =
             AS == ADDRESS_SPACE_GLOBAL ? ImplicitArg::GLOBAL_BASE : ImplicitArg::CONSTANT_BASE;
 
diff --git a/IGC/Compiler/tests/ProgramScopeConstantAnalysis/basic.ll b/IGC/Compiler/tests/ProgramScopeConstantAnalysis/basic.ll
@@ -66,8 +66,8 @@ entry:
 ; CHECK-DAG: [[SYMBOL]] = !{!"Symbol", !"a"}
 ; CHECK-DAG: !{!"inlineProgramScopeOffsets", [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP0:![0-9]*]], [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE0:![0-9]*]], [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP1:![0-9]*]], [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE1:![0-9]*]], [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP2:![0-9]*]], [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE2:![0-9]*]]}
 ; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP0]] = !{!"inlineProgramScopeOffsetsMap[0]", [2 x i32] addrspace(2)* @a}
-; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE0]] = !{!"inlineProgramScopeOffsetsValue[0]", i32 0}
+; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE0]] = !{!"inlineProgramScopeOffsetsValue[0]", i64 0}
 ; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP1]] = !{!"inlineProgramScopeOffsetsMap[1]", i32 addrspace(2)* addrspace(1)* @d}
-; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE1]] = !{!"inlineProgramScopeOffsetsValue[1]", i32 0}
+; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE1]] = !{!"inlineProgramScopeOffsetsValue[1]", i64 0}
 ; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_MAP2]] = !{!"inlineProgramScopeOffsetsMap[2]", i32 addrspace(1)* @c}
-; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE2]] = !{!"inlineProgramScopeOffsetsValue[2]", i32 8}
+; CHECK-DAG: [[INLINE_PROGRAM_SCOPE_OFFSETS_VALUE2]] = !{!"inlineProgramScopeOffsetsValue[2]", i64 8}
diff --git a/IGC/Compiler/tests/ProgramScopeConstantResolution/basic.ll b/IGC/Compiler/tests/ProgramScopeConstantResolution/basic.ll
@@ -29,11 +29,11 @@
 define spir_kernel void @test_program(i32 addrspace(1)* %dst, <8 x i32> %r0, <8 x i32> %payloadHeader, i8 addrspace(2)* %constBase, i8 addrspace(1)* %globalBase, i8* %privateBase, i32 %bufferOffset) #0 {
 ; CHECK-LABEL: @test_program(
 ; CHECK:  entry:
-; CHECK:    [[OFFC:%.*]] = getelementptr i8, i8 addrspace(1)* %globalBase, i32 8
+; CHECK:    [[OFFC:%.*]] = getelementptr i8, i8 addrspace(1)* %globalBase, i64 8
 ; CHECK:    [[CASTC:%.*]] = bitcast i8 addrspace(1)* [[OFFC]] to i32 addrspace(1)*
-; CHECK:    [[OFFD:%.*]] = getelementptr i8, i8 addrspace(1)* %globalBase, i32 0
+; CHECK:    [[OFFD:%.*]] = getelementptr i8, i8 addrspace(1)* %globalBase, i64 0
 ; CHECK:    [[CASTD:%.*]] = bitcast i8 addrspace(1)* [[OFFD]] to i32 addrspace(2)* addrspace(1)*
-; CHECK:    [[OFFA:%.*]] = getelementptr i8, i8 addrspace(2)* %constBase, i32 0
+; CHECK:    [[OFFA:%.*]] = getelementptr i8, i8 addrspace(2)* %constBase, i64 0
 ; CHECK:    [[CASTA:%.*]] = bitcast i8 addrspace(2)* [[OFFA]] to [2 x i32] addrspace(2)*
 ; CHECK:    [[DST_ADDR:%.*]] = alloca i32 addrspace(1)*, align 8
 ; CHECK:    [[AA:%.*]] = alloca i32, align 4
diff --git a/IGC/common/MDFrameWork.h b/IGC/common/MDFrameWork.h
@@ -692,7 +692,7 @@ namespace IGC
         std::vector<uint32_t> RasterizerOrderedByteAddressBuffer;
         std::set<uint32_t> RasterizerOrderedViews;
         unsigned int MinNOSPushConstantSize = 0;
-        llvm::MapVector<llvm::GlobalVariable*, int> inlineProgramScopeOffsets;
+        llvm::MapVector<llvm::GlobalVariable*, uint64_t> inlineProgramScopeOffsets;
         ShaderData shaderData;
         URBLayoutInfo URBInfo;
         bool UseBindlessImage = false;
