Skip to content

Commit 40862b1

Browse files
wsmoseswsmoses
committed
[SROA] Propagate correct TBAA/TBAA Struct offsets
SROA does not correctly account for offsets in TBAA/TBAA struct metadata. This patch creates functionality for generating new MD with the corresponding offset and updates SROA to use this functionality. Differential Revision: https://reviews.llvm.org/D95826
1 parent 908ac47 commit 40862b1

File tree

7 files changed

+269
-72
lines changed

7 files changed

+269
-72
lines changed

‎llvm/include/llvm/IR/Metadata.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,12 @@ struct AAMDNodes {
667667
/// The tag specifying the noalias scope.
668668
MDNode *NoAlias = nullptr;
669669

670+
// Shift tbaa Metadata node to start off bytes later
671+
static MDNode *ShiftTBAA(MDNode *M, size_t off);
672+
673+
// Shift tbaa.struct Metadata node to start off bytes later
674+
static MDNode *ShiftTBAAStruct(MDNode *M, size_t off);
675+
670676
/// Given two sets of AAMDNodes that apply to the same pointer,
671677
/// give the best AAMDNodes that are compatible with both (i.e. a set of
672678
/// nodes whose allowable aliasing conclusions are a subset of those
@@ -680,6 +686,18 @@ struct AAMDNodes {
680686
Result.NoAlias = Other.NoAlias == NoAlias ? NoAlias : nullptr;
681687
return Result;
682688
}
689+
690+
/// Create a new AAMDNode that describes this AAMDNode after applying a
691+
/// constant offset to the start of the pointer
692+
AAMDNodes shift(size_t Offset) {
693+
AAMDNodes Result;
694+
Result.TBAA = TBAA ? ShiftTBAA(TBAA, Offset) : nullptr;
695+
Result.TBAAStruct =
696+
TBAAStruct ? ShiftTBAAStruct(TBAAStruct, Offset) : nullptr;
697+
Result.Scope = Scope;
698+
Result.NoAlias = NoAlias;
699+
return Result;
700+
}
683701
};
684702

685703
// Specialize DenseMapInfo for AAMDNodes.

‎llvm/include/llvm/IR/Operator.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,11 @@ class GEPOperator
571571
bool accumulateConstantOffset(
572572
const DataLayout &DL, APInt &Offset,
573573
function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr) const;
574+
575+
static bool accumulateConstantOffset(
576+
Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL,
577+
APInt &Offset,
578+
function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr);
574579
};
575580

576581
class PtrToIntOperator

‎llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,3 +737,84 @@ bool TypeBasedAAWrapperPass::doFinalization(Module &M) {
737737
void TypeBasedAAWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
738738
AU.setPreservesAll();
739739
}
740+
741+
MDNode *AAMDNodes::ShiftTBAA(MDNode *MD, size_t Offset) {
742+
// Fast path if there's no offset
743+
if (Offset == 0)
744+
return MD;
745+
// Fast path if there's no path tbaa node (and thus scalar)
746+
if (!isStructPathTBAA(MD))
747+
return MD;
748+
749+
TBAAStructTagNode Tag(MD);
750+
SmallVector<Metadata *, 5> Sub;
751+
Sub.push_back(MD->getOperand(0));
752+
Sub.push_back(MD->getOperand(1));
753+
ConstantInt *InnerOffset = mdconst::extract<ConstantInt>(MD->getOperand(2));
754+
755+
if (Tag.isNewFormat()) {
756+
ConstantInt *InnerSize = mdconst::extract<ConstantInt>(MD->getOperand(3));
757+
758+
if (InnerOffset->getZExtValue() + InnerSize->getZExtValue() <= Offset) {
759+
return nullptr;
760+
}
761+
762+
uint64_t NewSize = InnerSize->getZExtValue();
763+
uint64_t NewOffset = InnerOffset->getZExtValue() - Offset;
764+
if (InnerOffset->getZExtValue() < Offset) {
765+
NewOffset = 0;
766+
NewSize -= Offset - InnerOffset->getZExtValue();
767+
}
768+
769+
Sub.push_back(ConstantAsMetadata::get(
770+
ConstantInt::get(InnerOffset->getType(), NewOffset)));
771+
772+
Sub.push_back(ConstantAsMetadata::get(
773+
ConstantInt::get(InnerSize->getType(), NewSize)));
774+
775+
// immutable type
776+
if (MD->getNumOperands() >= 5)
777+
Sub.push_back(MD->getOperand(4));
778+
} else {
779+
if (InnerOffset->getZExtValue() < Offset)
780+
return nullptr;
781+
782+
Sub.push_back(ConstantAsMetadata::get(ConstantInt::get(
783+
InnerOffset->getType(), InnerOffset->getZExtValue() - Offset)));
784+
785+
// immutable type
786+
if (MD->getNumOperands() >= 4)
787+
Sub.push_back(MD->getOperand(3));
788+
}
789+
return MDNode::get(MD->getContext(), Sub);
790+
}
791+
792+
MDNode *AAMDNodes::ShiftTBAAStruct(MDNode *MD, size_t Offset) {
793+
// Fast path if there's no offset
794+
if (Offset == 0)
795+
return MD;
796+
SmallVector<Metadata *, 3> Sub;
797+
for (size_t i = 0, size = MD->getNumOperands(); i < size; i += 3) {
798+
ConstantInt *InnerOffset = mdconst::extract<ConstantInt>(MD->getOperand(i));
799+
ConstantInt *InnerSize =
800+
mdconst::extract<ConstantInt>(MD->getOperand(i + 1));
801+
// Don't include any triples that aren't in bounds
802+
if (InnerOffset->getZExtValue() + InnerSize->getZExtValue() <= Offset)
803+
continue;
804+
805+
uint64_t NewSize = InnerSize->getZExtValue();
806+
uint64_t NewOffset = InnerOffset->getZExtValue() - Offset;
807+
if (InnerOffset->getZExtValue() < Offset) {
808+
NewOffset = 0;
809+
NewSize -= Offset - InnerOffset->getZExtValue();
810+
}
811+
812+
// Shift the offset of the triple
813+
Sub.push_back(ConstantAsMetadata::get(
814+
ConstantInt::get(InnerOffset->getType(), NewOffset)));
815+
Sub.push_back(ConstantAsMetadata::get(
816+
ConstantInt::get(InnerSize->getType(), NewSize)));
817+
Sub.push_back(MD->getOperand(i + 2));
818+
}
819+
return MDNode::get(MD->getContext(), Sub);
820+
}

‎llvm/lib/IR/Operator.cpp

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,17 @@ Align GEPOperator::getMaxPreservedAlignment(const DataLayout &DL) const {
6161
bool GEPOperator::accumulateConstantOffset(
6262
const DataLayout &DL, APInt &Offset,
6363
function_ref<bool(Value &, APInt &)> ExternalAnalysis) const {
64-
assert(Offset.getBitWidth() ==
65-
DL.getIndexSizeInBits(getPointerAddressSpace()) &&
66-
"The offset bit width does not match DL specification.");
64+
assert(Offset.getBitWidth() ==
65+
DL.getIndexSizeInBits(getPointerAddressSpace()) &&
66+
"The offset bit width does not match DL specification.");
67+
SmallVector<const Value *> Index(value_op_begin() + 1, value_op_end());
68+
return GEPOperator::accumulateConstantOffset(getSourceElementType(), Index,
69+
DL, Offset, ExternalAnalysis);
70+
}
6771

72+
bool GEPOperator::accumulateConstantOffset(
73+
Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL,
74+
APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis) {
6875
bool UsedExternalAnalysis = false;
6976
auto AccumulateOffset = [&](APInt Index, uint64_t Size) -> bool {
7077
Index = Index.sextOrTrunc(Offset.getBitWidth());
@@ -85,9 +92,10 @@ bool GEPOperator::accumulateConstantOffset(
8592
}
8693
return true;
8794
};
88-
89-
for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this);
90-
GTI != GTE; ++GTI) {
95+
auto begin = generic_gep_type_iterator<decltype(Index.begin())>::begin(
96+
SourceType, Index.begin());
97+
auto end = generic_gep_type_iterator<decltype(Index.end())>::end(Index.end());
98+
for (auto GTI = begin, GTE = end; GTI != GTE; ++GTI) {
9199
// Scalable vectors are multiplied by a runtime constant.
92100
bool ScalableType = false;
93101
if (isa<ScalableVectorType>(GTI.getIndexedType()))

‎llvm/lib/Transforms/Scalar/SROA.cpp

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2524,7 +2524,7 @@ class llvm::sroa::AllocaSliceRewriter
25242524
NewAI.getAlign(), LI.isVolatile(),
25252525
LI.getName());
25262526
if (AATags)
2527-
NewLI->setAAMetadata(AATags);
2527+
NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
25282528
if (LI.isVolatile())
25292529
NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
25302530
if (NewLI->isAtomic())
@@ -2563,7 +2563,7 @@ class llvm::sroa::AllocaSliceRewriter
25632563
IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
25642564
getSliceAlign(), LI.isVolatile(), LI.getName());
25652565
if (AATags)
2566-
NewLI->setAAMetadata(AATags);
2566+
NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
25672567
if (LI.isVolatile())
25682568
NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
25692569

@@ -2626,7 +2626,7 @@ class llvm::sroa::AllocaSliceRewriter
26262626
}
26272627
StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
26282628
if (AATags)
2629-
Store->setAAMetadata(AATags);
2629+
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
26302630
Pass.DeadInsts.push_back(&SI);
26312631

26322632
LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
@@ -2650,7 +2650,7 @@ class llvm::sroa::AllocaSliceRewriter
26502650
Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
26512651
LLVMContext::MD_access_group});
26522652
if (AATags)
2653-
Store->setAAMetadata(AATags);
2653+
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
26542654
Pass.DeadInsts.push_back(&SI);
26552655
LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
26562656
return true;
@@ -2720,7 +2720,7 @@ class llvm::sroa::AllocaSliceRewriter
27202720
NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
27212721
LLVMContext::MD_access_group});
27222722
if (AATags)
2723-
NewSI->setAAMetadata(AATags);
2723+
NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
27242724
if (SI.isVolatile())
27252725
NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
27262726
if (NewSI->isAtomic())
@@ -2816,7 +2816,7 @@ class llvm::sroa::AllocaSliceRewriter
28162816
getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
28172817
MaybeAlign(getSliceAlign()), II.isVolatile());
28182818
if (AATags)
2819-
New->setAAMetadata(AATags);
2819+
New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
28202820
LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
28212821
return false;
28222822
}
@@ -2885,7 +2885,7 @@ class llvm::sroa::AllocaSliceRewriter
28852885
StoreInst *New =
28862886
IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign(), II.isVolatile());
28872887
if (AATags)
2888-
New->setAAMetadata(AATags);
2888+
New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
28892889
LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
28902890
return !II.isVolatile();
28912891
}
@@ -3006,7 +3006,7 @@ class llvm::sroa::AllocaSliceRewriter
30063006
CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
30073007
Size, II.isVolatile());
30083008
if (AATags)
3009-
New->setAAMetadata(AATags);
3009+
New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
30103010
LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
30113011
return false;
30123012
}
@@ -3060,7 +3060,7 @@ class llvm::sroa::AllocaSliceRewriter
30603060
LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
30613061
II.isVolatile(), "copyload");
30623062
if (AATags)
3063-
Load->setAAMetadata(AATags);
3063+
Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
30643064
Src = Load;
30653065
}
30663066

@@ -3080,7 +3080,7 @@ class llvm::sroa::AllocaSliceRewriter
30803080
StoreInst *Store = cast<StoreInst>(
30813081
IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
30823082
if (AATags)
3083-
Store->setAAMetadata(AATags);
3083+
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
30843084
LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
30853085
return !II.isVolatile();
30863086
}
@@ -3381,8 +3381,13 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
33813381
IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
33823382
LoadInst *Load =
33833383
IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load");
3384-
if (AATags)
3385-
Load->setAAMetadata(AATags);
3384+
3385+
APInt Offset(
3386+
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
3387+
if (AATags &&
3388+
GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
3389+
Load->setAAMetadata(AATags.shift(Offset.getZExtValue()));
3390+
33863391
Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
33873392
LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
33883393
}
@@ -3428,8 +3433,13 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
34283433
IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
34293434
StoreInst *Store =
34303435
IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment);
3431-
if (AATags)
3432-
Store->setAAMetadata(AATags);
3436+
3437+
APInt Offset(
3438+
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
3439+
if (AATags &&
3440+
GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
3441+
Store->setAAMetadata(AATags.shift(Offset.getZExtValue()));
3442+
34333443
LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
34343444
}
34353445
};

0 commit comments

Comments
 (0)