@@ -812,135 +812,6 @@ SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name,
812
812
return llvm::MDNode::get (*Context, Metadata);
813
813
}
814
814
815
- class IVDepMetadataEmitter {
816
- public:
817
- using PointerSafeLenMapTy = std::map<Value *, unsigned >;
818
- static void emit (LLVMContext *Context, const Loop *LoopObj,
819
- const PointerSafeLenMapTy &PointerSafeLenMap,
820
- std::vector<llvm::Metadata *> &Metadata) {
821
- if (LoopsEmitted.contains (LoopObj))
822
- return ;
823
- const auto ArrayGEPMap = mapArrayToGEPs (LoopObj, PointerSafeLenMap);
824
- emitMetadata (Context, ArrayGEPMap, PointerSafeLenMap, Metadata);
825
- LoopsEmitted.insert (LoopObj);
826
- }
827
-
828
- private:
829
- static llvm::DenseSet<const Loop *> LoopsEmitted;
830
-
831
- using ArrayGEPMapTy = std::map<Value *, std::vector<GetElementPtrInst *>>;
832
- // A single run over the loop to retrieve all GetElementPtr instructions
833
- // that access relevant array variables
834
- static ArrayGEPMapTy
835
- mapArrayToGEPs (const Loop *LoopObj,
836
- const PointerSafeLenMapTy &PointerSafeLenMap) {
837
- ArrayGEPMapTy ArrayGEPMap;
838
- for (const auto &BB : LoopObj->blocks ()) {
839
- for (Instruction &I : *BB) {
840
- auto *GEP = dyn_cast<GetElementPtrInst>(&I);
841
- if (!GEP)
842
- continue ;
843
-
844
- Value *AccessedPointer = GEP->getPointerOperand ();
845
- if (auto *LI = dyn_cast<LoadInst>(AccessedPointer))
846
- AccessedPointer = LI->getPointerOperand ();
847
- auto PointerSafeLenIt = PointerSafeLenMap.find (AccessedPointer);
848
- if (PointerSafeLenIt != PointerSafeLenMap.end ()) {
849
- ArrayGEPMap[AccessedPointer].push_back (GEP);
850
- }
851
- }
852
- }
853
- return ArrayGEPMap;
854
- }
855
-
856
- // Create index group metadata nodes - one per each of the array
857
- // variables. Mark each GEP accessing a particular array variable
858
- // into a corresponding index group
859
- static void emitMetadata (LLVMContext *Context,
860
- const ArrayGEPMapTy &ArrayGEPMap,
861
- const PointerSafeLenMapTy &PointerSafeLenMap,
862
- std::vector<llvm::Metadata *> &Metadata) {
863
- using SafeLenIdxGroupMapTy = std::map<unsigned , SmallSet<MDNode *, 4 >>;
864
- SafeLenIdxGroupMapTy SafeLenIdxGroupMap;
865
- // Whenever a kernel closure field access is pointed to instead of
866
- // an array/pointer variable, ensure that all GEPs to that memory
867
- // share the same index group by hashing the newly added index groups.
868
- // "Memory offset info" represents a handle to the whole closure block
869
- // + an integer offset to a particular captured parameter.
870
- using MemoryOffsetInfo = std::pair<Value *, unsigned >;
871
- std::map<MemoryOffsetInfo, MDNode *> OffsetIdxGroupMap;
872
-
873
- for (auto &ArrayGEPIt : ArrayGEPMap) {
874
- MDNode *CurrentDepthIdxGroup = nullptr ;
875
- if (auto *PrecedingGEP = dyn_cast<GetElementPtrInst>(ArrayGEPIt.first )) {
876
- Value *ClosureFieldPointer = PrecedingGEP->getPointerOperand ();
877
- unsigned Offset =
878
- cast<ConstantInt>(PrecedingGEP->getOperand (2 ))->getZExtValue ();
879
- MemoryOffsetInfo Info{ClosureFieldPointer, Offset};
880
- auto OffsetIdxGroupIt = OffsetIdxGroupMap.find (Info);
881
- if (OffsetIdxGroupIt == OffsetIdxGroupMap.end ()) {
882
- // This is the first GEP encountered for this closure field.
883
- // Emit a distinct index group that will be referenced from
884
- // llvm.loop.parallel_access_indices metadata; hash the new
885
- // MDNode for future accesses to the same memory.
886
- CurrentDepthIdxGroup = llvm::MDNode::getDistinct (*Context, None);
887
- OffsetIdxGroupMap.emplace (Info, CurrentDepthIdxGroup);
888
- } else {
889
- // Previous accesses to that field have already been indexed,
890
- // just use the already-existing metadata.
891
- CurrentDepthIdxGroup = OffsetIdxGroupIt->second ;
892
- }
893
- } else /* Regular kernel-scope array/pointer variable */ {
894
- // Emit a distinct index group that will be referenced from
895
- // llvm.loop.parallel_access_indices metadata
896
- CurrentDepthIdxGroup = llvm::MDNode::getDistinct (*Context, None);
897
- }
898
-
899
- unsigned SafeLen = PointerSafeLenMap.find (ArrayGEPIt.first )->second ;
900
- SafeLenIdxGroupMap[SafeLen].insert (CurrentDepthIdxGroup);
901
- for (auto *GEP : ArrayGEPIt.second ) {
902
- StringRef IdxGroupMDName (" llvm.index.group" );
903
- llvm::MDNode *PreviousIdxGroup = GEP->getMetadata (IdxGroupMDName);
904
- if (!PreviousIdxGroup) {
905
- GEP->setMetadata (IdxGroupMDName, CurrentDepthIdxGroup);
906
- continue ;
907
- }
908
-
909
- // If we're dealing with an embedded loop, it may be the case
910
- // that GEP instructions for some of the arrays were already
911
- // marked by the algorithm when it went over the outer level loops.
912
- // In order to retain the IVDep information for each "loop
913
- // dimension", we will mark such GEP's into a separate joined node
914
- // that will refer to the previous levels' index groups AND to the
915
- // index group specific to the current loop.
916
- std::vector<llvm::Metadata *> CurrentDepthOperands (
917
- PreviousIdxGroup->op_begin (), PreviousIdxGroup->op_end ());
918
- if (CurrentDepthOperands.empty ())
919
- CurrentDepthOperands.push_back (PreviousIdxGroup);
920
- CurrentDepthOperands.push_back (CurrentDepthIdxGroup);
921
- auto *JointIdxGroup = llvm::MDNode::get (*Context, CurrentDepthOperands);
922
- GEP->setMetadata (IdxGroupMDName, JointIdxGroup);
923
- }
924
- }
925
-
926
- for (auto &SafeLenIdxGroupIt : SafeLenIdxGroupMap) {
927
- auto *Name = MDString::get (*Context, " llvm.loop.parallel_access_indices" );
928
- unsigned SafeLenValue = SafeLenIdxGroupIt.first ;
929
- llvm::Metadata *SafeLenMDOp =
930
- SafeLenValue ? ConstantAsMetadata::get (ConstantInt::get (
931
- Type::getInt32Ty (*Context), SafeLenValue))
932
- : nullptr ;
933
- std::vector<llvm::Metadata *> Parameters{Name};
934
- for (auto *Node : SafeLenIdxGroupIt.second )
935
- Parameters.push_back (Node);
936
- if (SafeLenMDOp)
937
- Parameters.push_back (SafeLenMDOp);
938
- Metadata.push_back (llvm::MDNode::get (*Context, Parameters));
939
- }
940
- }
941
- };
942
- llvm::DenseSet<const Loop *> IVDepMetadataEmitter::LoopsEmitted;
943
-
944
815
template <typename LoopInstType>
945
816
void SPIRVToLLVM::setLLVMLoopMetadata (const LoopInstType *LM,
946
817
const Loop *LoopObj) {
@@ -1035,7 +906,7 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
1035
906
}
1036
907
if (LC & LoopControlDependencyArrayINTELMask) {
1037
908
// Collect pointer variable <-> safelen information
1038
- IVDepMetadataEmitter::PointerSafeLenMapTy PointerSafeLenMap ;
909
+ std::map<Value *, unsigned > PointerSflnMap ;
1039
910
unsigned NumOperandPairs = LoopControlParameters[NumParam];
1040
911
unsigned OperandsEndIndex = NumParam + NumOperandPairs * 2 ;
1041
912
assert (OperandsEndIndex <= LoopControlParameters.size () &&
@@ -1044,13 +915,109 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
1044
915
while (NumParam < OperandsEndIndex) {
1045
916
SPIRVId ArraySPIRVId = LoopControlParameters[++NumParam];
1046
917
Value *PointerVar = ValueMap[M->getValue (ArraySPIRVId)];
1047
- unsigned SafeLen = LoopControlParameters[++NumParam];
1048
- PointerSafeLenMap.emplace (PointerVar, SafeLen);
918
+ unsigned Safelen = LoopControlParameters[++NumParam];
919
+ PointerSflnMap.emplace (PointerVar, Safelen);
920
+ }
921
+
922
+ // A single run over the loop to retrieve all GetElementPtr instructions
923
+ // that access relevant array variables
924
+ std::map<Value *, std::vector<GetElementPtrInst *>> ArrayGEPMap;
925
+ for (const auto &BB : LoopObj->blocks ()) {
926
+ for (Instruction &I : *BB) {
927
+ auto *GEP = dyn_cast<GetElementPtrInst>(&I);
928
+ if (!GEP)
929
+ continue ;
930
+
931
+ Value *AccessedPointer = GEP->getPointerOperand ();
932
+ if (auto *LI = dyn_cast<LoadInst>(AccessedPointer))
933
+ AccessedPointer = LI->getPointerOperand ();
934
+ auto PointerSflnIt = PointerSflnMap.find (AccessedPointer);
935
+ if (PointerSflnIt != PointerSflnMap.end ()) {
936
+ ArrayGEPMap[AccessedPointer].push_back (GEP);
937
+ }
938
+ }
939
+ }
940
+
941
+ // Create index group metadata nodes - one per each of the array
942
+ // variables. Mark each GEP accessing a particular array variable
943
+ // into a corresponding index group
944
+ std::map<unsigned , SmallSet<MDNode *, 4 >> SafelenIdxGroupMap;
945
+ // Whenever a kernel closure field access is pointed to instead of
946
+ // an array/pointer variable, ensure that all GEPs to that memory
947
+ // share the same index group by hashing the newly added index groups.
948
+ // "Memory offset info" represents a handle to the whole closure block
949
+ // + an integer offset to a particular captured parameter.
950
+ using MemoryOffsetInfo = std::pair<Value *, unsigned >;
951
+ std::map<MemoryOffsetInfo, MDNode *> OffsetIdxGroupMap;
952
+
953
+ for (auto &ArrayGEPIt : ArrayGEPMap) {
954
+ MDNode *CurrentDepthIdxGroup = nullptr ;
955
+ if (auto *PrecedingGEP = dyn_cast<GetElementPtrInst>(ArrayGEPIt.first )) {
956
+ Value *ClosureFieldPointer = PrecedingGEP->getPointerOperand ();
957
+ unsigned Offset =
958
+ cast<ConstantInt>(PrecedingGEP->getOperand (2 ))->getZExtValue ();
959
+ MemoryOffsetInfo Info{ClosureFieldPointer, Offset};
960
+ auto OffsetIdxGroupIt = OffsetIdxGroupMap.find (Info);
961
+ if (OffsetIdxGroupIt == OffsetIdxGroupMap.end ()) {
962
+ // This is the first GEP encountered for this closure field.
963
+ // Emit a distinct index group that will be referenced from
964
+ // llvm.loop.parallel_access_indices metadata; hash the new
965
+ // MDNode for future accesses to the same memory.
966
+ CurrentDepthIdxGroup = llvm::MDNode::getDistinct (*Context, None);
967
+ OffsetIdxGroupMap.emplace (Info, CurrentDepthIdxGroup);
968
+ } else {
969
+ // Previous accesses to that field have already been indexed,
970
+ // just use the already-existing metadata.
971
+ CurrentDepthIdxGroup = OffsetIdxGroupIt->second ;
972
+ }
973
+ } else /* Regular kernel-scope array/pointer variable */ {
974
+ // Emit a distinct index group that will be referenced from
975
+ // llvm.loop.parallel_access_indices metadata
976
+ CurrentDepthIdxGroup = llvm::MDNode::getDistinct (*Context, None);
977
+ }
978
+
979
+ unsigned Safelen = PointerSflnMap.find (ArrayGEPIt.first )->second ;
980
+ SafelenIdxGroupMap[Safelen].insert (CurrentDepthIdxGroup);
981
+ for (auto *GEP : ArrayGEPIt.second ) {
982
+ StringRef IdxGroupMDName (" llvm.index.group" );
983
+ llvm::MDNode *PreviousIdxGroup = GEP->getMetadata (IdxGroupMDName);
984
+ if (!PreviousIdxGroup) {
985
+ GEP->setMetadata (IdxGroupMDName, CurrentDepthIdxGroup);
986
+ continue ;
987
+ }
988
+
989
+ // If we're dealing with an embedded loop, it may be the case
990
+ // that GEP instructions for some of the arrays were already
991
+ // marked by the algorithm when it went over the outer level loops.
992
+ // In order to retain the IVDep information for each "loop
993
+ // dimension", we will mark such GEP's into a separate joined node
994
+ // that will refer to the previous levels' index groups AND to the
995
+ // index group specific to the current loop.
996
+ std::vector<llvm::Metadata *> CurrentDepthOperands (
997
+ PreviousIdxGroup->op_begin (), PreviousIdxGroup->op_end ());
998
+ if (CurrentDepthOperands.empty ())
999
+ CurrentDepthOperands.push_back (PreviousIdxGroup);
1000
+ CurrentDepthOperands.push_back (CurrentDepthIdxGroup);
1001
+ auto *JointIdxGroup = llvm::MDNode::get (*Context, CurrentDepthOperands);
1002
+ GEP->setMetadata (IdxGroupMDName, JointIdxGroup);
1003
+ }
1004
+ }
1005
+
1006
+ for (auto &SflnIdxGroupIt : SafelenIdxGroupMap) {
1007
+ auto *Name = MDString::get (*Context, " llvm.loop.parallel_access_indices" );
1008
+ unsigned SflnValue = SflnIdxGroupIt.first ;
1009
+ llvm::Metadata *SafelenMDOp =
1010
+ SflnValue ? ConstantAsMetadata::get (ConstantInt::get (
1011
+ Type::getInt32Ty (*Context), SflnValue))
1012
+ : nullptr ;
1013
+ std::vector<llvm::Metadata *> Parameters{Name};
1014
+ for (auto *Node : SflnIdxGroupIt.second )
1015
+ Parameters.push_back (Node);
1016
+ if (SafelenMDOp)
1017
+ Parameters.push_back (SafelenMDOp);
1018
+ Metadata.push_back (llvm::MDNode::get (*Context, Parameters));
1049
1019
}
1050
- IVDepMetadataEmitter::emit (Context, LoopObj, PointerSafeLenMap, Metadata);
1051
1020
++NumParam;
1052
- assert (NumParam <= LoopControlParameters.size () &&
1053
- " Missing loop control parameter!" );
1054
1021
}
1055
1022
if (LC & LoopControlPipelineEnableINTELMask) {
1056
1023
Metadata.push_back (llvm::MDNode::get (
0 commit comments