Skip to content

Commit a8e0e4a

Browse files
fhahnfhahn
committed
[ValueTracking] Suport GEPs in matchSimpleRecurrence.
Update matchSimpleRecurrence to also support GEPs. This allows inferring larger alignments in a number of cases. I noticed that we fail to infer alignments from calls when dropping assumptions; inferring alignment from assumptions uses SCEV, if we drop an assume for a aligned function return value, we fail to infer the better alignment in InferAlignment without this patch. It comes with a bit of a compile-time impact: stage1-O3: +0.05% stage1-ReleaseThinLTO: +0.04% stage1-ReleaseLTO-g: +0.03% stage1-O0-g: -0.04% stage2-O3: +0.04% stage2-O0-g: +0.02% stage2-clang: +0.03% https://llvm-compile-time-tracker.com/compare.php?from=a8c60790fd4f70a461113f0721bdb4a114ddf420&to=9a207c52e9c644691573a40ceb5b89a3c09ab609&stat=instructions:u
1 parent 112c9fd commit a8e0e4a

File tree

3 files changed

+63
-22
lines changed

3 files changed

+63
-22
lines changed

‎llvm/include/llvm/Analysis/ValueTracking.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1245,7 +1245,11 @@ bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start,
12451245
Value *&Step);
12461246

12471247
/// Analogous to the above, but starting from the binary operator
1248-
bool matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P, Value *&Start,
1248+
bool matchSimpleRecurrence(const Instruction *I, PHINode *&P, Value *&Start,
1249+
Value *&Step);
1250+
1251+
/// Analogous to the above, but also supporting non-binary operators.
1252+
bool matchSimpleRecurrence(const PHINode *P, Instruction *&BO, Value *&Start,
12491253
Value *&Step);
12501254

12511255
/// Return true if RHS is known to be implied true by LHS. Return false if

‎llvm/lib/Analysis/ValueTracking.cpp

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1489,7 +1489,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
14891489
}
14901490
case Instruction::PHI: {
14911491
const PHINode *P = cast<PHINode>(I);
1492-
BinaryOperator *BO = nullptr;
1492+
Instruction *BO = nullptr;
14931493
Value *R = nullptr, *L = nullptr;
14941494
if (matchSimpleRecurrence(P, BO, R, L)) {
14951495
// Handle the case of a simple two-predecessor recurrence PHI.
@@ -1553,6 +1553,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
15531553
case Instruction::Sub:
15541554
case Instruction::And:
15551555
case Instruction::Or:
1556+
case Instruction::GetElementPtr:
15561557
case Instruction::Mul: {
15571558
// Change the context instruction to the "edge" that flows into the
15581559
// phi. This is important because that is where the value is actually
@@ -1571,12 +1572,21 @@ static void computeKnownBitsFromOperator(const Operator *I,
15711572

15721573
// We need to take the minimum number of known bits
15731574
KnownBits Known3(BitWidth);
1575+
if (BitWidth != getBitWidth(L->getType(), Q.DL)) {
1576+
assert(isa<GetElementPtrInst>(BO) &&
1577+
"Bitwidth should only be different for GEPs.");
1578+
break;
1579+
}
15741580
RecQ.CxtI = LInst;
15751581
computeKnownBits(L, DemandedElts, Known3, Depth + 1, RecQ);
15761582

15771583
Known.Zero.setLowBits(std::min(Known2.countMinTrailingZeros(),
15781584
Known3.countMinTrailingZeros()));
15791585

1586+
// Don't apply logic below for GEPs.
1587+
if (isa<GetElementPtrInst>(BO))
1588+
break;
1589+
15801590
auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(BO);
15811591
if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(OverflowOp))
15821592
break;
@@ -1737,6 +1747,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
17371747
Known.resetAll();
17381748
}
17391749
}
1750+
17401751
if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
17411752
switch (II->getIntrinsicID()) {
17421753
default:
@@ -2270,7 +2281,7 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
22702281
/// always a power of two (or zero).
22712282
static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
22722283
unsigned Depth, SimplifyQuery &Q) {
2273-
BinaryOperator *BO = nullptr;
2284+
Instruction *BO = nullptr;
22742285
Value *Start = nullptr, *Step = nullptr;
22752286
if (!matchSimpleRecurrence(PN, BO, Start, Step))
22762287
return false;
@@ -2308,7 +2319,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
23082319
// Divisor must be a power of two.
23092320
// If OrZero is false, cannot guarantee induction variable is non-zero after
23102321
// division, same for Shr, unless it is exact division.
2311-
return (OrZero || Q.IIQ.isExact(BO)) &&
2322+
return (OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO))) &&
23122323
isKnownToBeAPowerOfTwo(Step, false, Depth, Q);
23132324
case Instruction::Shl:
23142325
return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO);
@@ -2317,7 +2328,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
23172328
return false;
23182329
[[fallthrough]];
23192330
case Instruction::LShr:
2320-
return OrZero || Q.IIQ.isExact(BO);
2331+
return OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO));
23212332
default:
23222333
return false;
23232334
}
@@ -2727,7 +2738,7 @@ static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value)
27272738
/// Try to detect a recurrence that monotonically increases/decreases from a
27282739
/// non-zero starting value. These are common as induction variables.
27292740
static bool isNonZeroRecurrence(const PHINode *PN) {
2730-
BinaryOperator *BO = nullptr;
2741+
Instruction *BO = nullptr;
27312742
Value *Start = nullptr, *Step = nullptr;
27322743
const APInt *StartC, *StepC;
27332744
if (!matchSimpleRecurrence(PN, BO, Start, Step) ||
@@ -3560,9 +3571,9 @@ getInvertibleOperands(const Operator *Op1,
35603571
// If PN1 and PN2 are both recurrences, can we prove the entire recurrences
35613572
// are a single invertible function of the start values? Note that repeated
35623573
// application of an invertible function is also invertible
3563-
BinaryOperator *BO1 = nullptr;
3574+
Instruction *BO1 = nullptr;
35643575
Value *Start1 = nullptr, *Step1 = nullptr;
3565-
BinaryOperator *BO2 = nullptr;
3576+
Instruction *BO2 = nullptr;
35663577
Value *Start2 = nullptr, *Step2 = nullptr;
35673578
if (PN1->getParent() != PN2->getParent() ||
35683579
!matchSimpleRecurrence(PN1, BO1, Start1, Step1) ||
@@ -9199,6 +9210,17 @@ llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
91999210

92009211
bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92019212
Value *&Start, Value *&Step) {
9213+
Instruction *I;
9214+
if (matchSimpleRecurrence(P, I, Start, Step)) {
9215+
BO = dyn_cast<BinaryOperator>(I);
9216+
if (BO)
9217+
return true;
9218+
}
9219+
return false;
9220+
}
9221+
9222+
bool llvm::matchSimpleRecurrence(const PHINode *P, Instruction *&BO,
9223+
Value *&Start, Value *&Step) {
92029224
// Handle the case of a simple two-predecessor recurrence PHI.
92039225
// There's a lot more that could theoretically be done here, but
92049226
// this is sufficient to catch some interesting cases.
@@ -9208,7 +9230,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92089230
for (unsigned i = 0; i != 2; ++i) {
92099231
Value *L = P->getIncomingValue(i);
92109232
Value *R = P->getIncomingValue(!i);
9211-
auto *LU = dyn_cast<BinaryOperator>(L);
9233+
auto *LU = dyn_cast<Instruction>(L);
92129234
if (!LU)
92139235
continue;
92149236
unsigned Opcode = LU->getOpcode();
@@ -9240,6 +9262,21 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92409262

92419263
break; // Match!
92429264
}
9265+
case Instruction::GetElementPtr: {
9266+
if (LU->getNumOperands() != 2 ||
9267+
!cast<GetElementPtrInst>(L)->getSourceElementType()->isIntegerTy(8))
9268+
continue;
9269+
9270+
Value *LL = LU->getOperand(0);
9271+
Value *LR = LU->getOperand(1);
9272+
// Find a recurrence.
9273+
if (LL == P) {
9274+
// Found a match
9275+
L = LR;
9276+
break;
9277+
}
9278+
continue;
9279+
}
92439280
};
92449281

92459282
// We have matched a recurrence of the form:
@@ -9256,9 +9293,9 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92569293
return false;
92579294
}
92589295

9259-
bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9296+
bool llvm::matchSimpleRecurrence(const Instruction *I, PHINode *&P,
92609297
Value *&Start, Value *&Step) {
9261-
BinaryOperator *BO = nullptr;
9298+
Instruction *BO = nullptr;
92629299
P = dyn_cast<PHINode>(I->getOperand(0));
92639300
if (!P)
92649301
P = dyn_cast<PHINode>(I->getOperand(1));

‎llvm/test/Transforms/InferAlignment/gep-recurrence.ll

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ define void @test_recur_i8_128(ptr align 128 %dst) {
1212
; CHECK-NEXT: br label [[LOOP:%.*]]
1313
; CHECK: loop:
1414
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
15-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
15+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
1616
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 128
1717
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
1818
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -40,7 +40,7 @@ define void @test_recur_i8_128_no_inbounds(ptr align 128 %dst) {
4040
; CHECK-NEXT: br label [[LOOP:%.*]]
4141
; CHECK: loop:
4242
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
43-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
43+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
4444
; CHECK-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 128
4545
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
4646
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -68,7 +68,7 @@ define void @test_recur_i8_64(ptr align 128 %dst) {
6868
; CHECK-NEXT: br label [[LOOP:%.*]]
6969
; CHECK: loop:
7070
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
71-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
71+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 64
7272
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 64
7373
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
7474
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -124,7 +124,7 @@ define void @test_recur_i8_32(ptr align 128 %dst) {
124124
; CHECK-NEXT: br label [[LOOP:%.*]]
125125
; CHECK: loop:
126126
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
127-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
127+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 32
128128
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 32
129129
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
130130
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -152,7 +152,7 @@ define void @test_recur_i8_16(ptr align 128 %dst) {
152152
; CHECK-NEXT: br label [[LOOP:%.*]]
153153
; CHECK: loop:
154154
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
155-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
155+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 16
156156
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 16
157157
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
158158
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -180,7 +180,7 @@ define void @test_recur_i8_8(ptr align 128 %dst) {
180180
; CHECK-NEXT: br label [[LOOP:%.*]]
181181
; CHECK: loop:
182182
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
183-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
183+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 8
184184
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 8
185185
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
186186
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -208,7 +208,7 @@ define void @test_recur_i8_4(ptr align 128 %dst) {
208208
; CHECK-NEXT: br label [[LOOP:%.*]]
209209
; CHECK: loop:
210210
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
211-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
211+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 4
212212
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 4
213213
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
214214
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -236,7 +236,7 @@ define void @test_recur_i8_2(ptr align 128 %dst) {
236236
; CHECK-NEXT: br label [[LOOP:%.*]]
237237
; CHECK: loop:
238238
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
239-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
239+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 2
240240
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 2
241241
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
242242
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -468,7 +468,7 @@ define void @test_recur_i8_neg_128(ptr align 128 %dst) {
468468
; CHECK-NEXT: br label [[LOOP:%.*]]
469469
; CHECK: loop:
470470
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
471-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
471+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
472472
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -128
473473
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
474474
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -496,7 +496,7 @@ define void @test_recur_i8_neg64(ptr align 128 %dst) {
496496
; CHECK-NEXT: br label [[LOOP:%.*]]
497497
; CHECK: loop:
498498
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
499-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
499+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 64
500500
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -64
501501
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
502502
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -552,7 +552,7 @@ define void @test_recur_i8_neg_32(ptr align 128 %dst) {
552552
; CHECK-NEXT: br label [[LOOP:%.*]]
553553
; CHECK: loop:
554554
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
555-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
555+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 32
556556
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -32
557557
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
558558
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]

0 commit comments

Comments
 (0)