Skip to content

Commit 31c4524

Browse files
fhahnfhahn
committed
[ValueTracking] Suport GEPs in matchSimpleRecurrence.
Update matchSimpleRecurrence to also support GEPs. This allows inferring larger alignments in a number of cases. I noticed that we fail to infer alignments from calls when dropping assumptions; inferring alignment from assumptions uses SCEV, if we drop an assume for a aligned function return value, we fail to infer the better alignment in InferAlignment without this patch. It comes with a bit of a compile-time impact: stage1-O3: +0.05% stage1-ReleaseThinLTO: +0.04% stage1-ReleaseLTO-g: +0.03% stage1-O0-g: -0.04% stage2-O3: +0.04% stage2-O0-g: +0.02% stage2-clang: +0.03% https://llvm-compile-time-tracker.com/compare.php?from=a8c60790fd4f70a461113f0721bdb4a114ddf420&to=9a207c52e9c644691573a40ceb5b89a3c09ab609&stat=instructions:u
1 parent 874c7c3 commit 31c4524

File tree

3 files changed

+63
-22
lines changed

3 files changed

+63
-22
lines changed

‎llvm/include/llvm/Analysis/ValueTracking.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1245,7 +1245,11 @@ bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start,
12451245
Value *&Step);
12461246

12471247
/// Analogous to the above, but starting from the binary operator
1248-
bool matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P, Value *&Start,
1248+
bool matchSimpleRecurrence(const Instruction *I, PHINode *&P, Value *&Start,
1249+
Value *&Step);
1250+
1251+
/// Analogous to the above, but also supporting non-binary operators.
1252+
bool matchSimpleRecurrence(const PHINode *P, Instruction *&BO, Value *&Start,
12491253
Value *&Step);
12501254

12511255
/// Return true if RHS is known to be implied true by LHS. Return false if

‎llvm/lib/Analysis/ValueTracking.cpp

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1524,7 +1524,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
15241524
}
15251525
case Instruction::PHI: {
15261526
const PHINode *P = cast<PHINode>(I);
1527-
BinaryOperator *BO = nullptr;
1527+
Instruction *BO = nullptr;
15281528
Value *R = nullptr, *L = nullptr;
15291529
if (matchSimpleRecurrence(P, BO, R, L)) {
15301530
// Handle the case of a simple two-predecessor recurrence PHI.
@@ -1588,6 +1588,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
15881588
case Instruction::Sub:
15891589
case Instruction::And:
15901590
case Instruction::Or:
1591+
case Instruction::GetElementPtr:
15911592
case Instruction::Mul: {
15921593
// Change the context instruction to the "edge" that flows into the
15931594
// phi. This is important because that is where the value is actually
@@ -1606,12 +1607,21 @@ static void computeKnownBitsFromOperator(const Operator *I,
16061607

16071608
// We need to take the minimum number of known bits
16081609
KnownBits Known3(BitWidth);
1610+
if (BitWidth != getBitWidth(L->getType(), Q.DL)) {
1611+
assert(isa<GetElementPtrInst>(BO) &&
1612+
"Bitwidth should only be different for GEPs.");
1613+
break;
1614+
}
16091615
RecQ.CxtI = LInst;
16101616
computeKnownBits(L, DemandedElts, Known3, Depth + 1, RecQ);
16111617

16121618
Known.Zero.setLowBits(std::min(Known2.countMinTrailingZeros(),
16131619
Known3.countMinTrailingZeros()));
16141620

1621+
// Don't apply logic below for GEPs.
1622+
if (isa<GetElementPtrInst>(BO))
1623+
break;
1624+
16151625
auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(BO);
16161626
if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(OverflowOp))
16171627
break;
@@ -1768,6 +1778,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
17681778
Known.resetAll();
17691779
}
17701780
}
1781+
17711782
if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
17721783
switch (II->getIntrinsicID()) {
17731784
default:
@@ -2301,7 +2312,7 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
23012312
/// always a power of two (or zero).
23022313
static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
23032314
unsigned Depth, SimplifyQuery &Q) {
2304-
BinaryOperator *BO = nullptr;
2315+
Instruction *BO = nullptr;
23052316
Value *Start = nullptr, *Step = nullptr;
23062317
if (!matchSimpleRecurrence(PN, BO, Start, Step))
23072318
return false;
@@ -2339,7 +2350,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
23392350
// Divisor must be a power of two.
23402351
// If OrZero is false, cannot guarantee induction variable is non-zero after
23412352
// division, same for Shr, unless it is exact division.
2342-
return (OrZero || Q.IIQ.isExact(BO)) &&
2353+
return (OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO))) &&
23432354
isKnownToBeAPowerOfTwo(Step, false, Depth, Q);
23442355
case Instruction::Shl:
23452356
return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO);
@@ -2348,7 +2359,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
23482359
return false;
23492360
[[fallthrough]];
23502361
case Instruction::LShr:
2351-
return OrZero || Q.IIQ.isExact(BO);
2362+
return OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO));
23522363
default:
23532364
return false;
23542365
}
@@ -2760,7 +2771,7 @@ static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value)
27602771
/// Try to detect a recurrence that monotonically increases/decreases from a
27612772
/// non-zero starting value. These are common as induction variables.
27622773
static bool isNonZeroRecurrence(const PHINode *PN) {
2763-
BinaryOperator *BO = nullptr;
2774+
Instruction *BO = nullptr;
27642775
Value *Start = nullptr, *Step = nullptr;
27652776
const APInt *StartC, *StepC;
27662777
if (!matchSimpleRecurrence(PN, BO, Start, Step) ||
@@ -3593,9 +3604,9 @@ getInvertibleOperands(const Operator *Op1,
35933604
// If PN1 and PN2 are both recurrences, can we prove the entire recurrences
35943605
// are a single invertible function of the start values? Note that repeated
35953606
// application of an invertible function is also invertible
3596-
BinaryOperator *BO1 = nullptr;
3607+
Instruction *BO1 = nullptr;
35973608
Value *Start1 = nullptr, *Step1 = nullptr;
3598-
BinaryOperator *BO2 = nullptr;
3609+
Instruction *BO2 = nullptr;
35993610
Value *Start2 = nullptr, *Step2 = nullptr;
36003611
if (PN1->getParent() != PN2->getParent() ||
36013612
!matchSimpleRecurrence(PN1, BO1, Start1, Step1) ||
@@ -9215,6 +9226,17 @@ llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
92159226

92169227
bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92179228
Value *&Start, Value *&Step) {
9229+
Instruction *I;
9230+
if (matchSimpleRecurrence(P, I, Start, Step)) {
9231+
BO = dyn_cast<BinaryOperator>(I);
9232+
if (BO)
9233+
return true;
9234+
}
9235+
return false;
9236+
}
9237+
9238+
bool llvm::matchSimpleRecurrence(const PHINode *P, Instruction *&BO,
9239+
Value *&Start, Value *&Step) {
92189240
// Handle the case of a simple two-predecessor recurrence PHI.
92199241
// There's a lot more that could theoretically be done here, but
92209242
// this is sufficient to catch some interesting cases.
@@ -9224,7 +9246,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92249246
for (unsigned i = 0; i != 2; ++i) {
92259247
Value *L = P->getIncomingValue(i);
92269248
Value *R = P->getIncomingValue(!i);
9227-
auto *LU = dyn_cast<BinaryOperator>(L);
9249+
auto *LU = dyn_cast<Instruction>(L);
92289250
if (!LU)
92299251
continue;
92309252
unsigned Opcode = LU->getOpcode();
@@ -9256,6 +9278,21 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92569278

92579279
break; // Match!
92589280
}
9281+
case Instruction::GetElementPtr: {
9282+
if (LU->getNumOperands() != 2 ||
9283+
!cast<GetElementPtrInst>(L)->getSourceElementType()->isIntegerTy(8))
9284+
continue;
9285+
9286+
Value *LL = LU->getOperand(0);
9287+
Value *LR = LU->getOperand(1);
9288+
// Find a recurrence.
9289+
if (LL == P) {
9290+
// Found a match
9291+
L = LR;
9292+
break;
9293+
}
9294+
continue;
9295+
}
92599296
};
92609297

92619298
// We have matched a recurrence of the form:
@@ -9272,9 +9309,9 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
92729309
return false;
92739310
}
92749311

9275-
bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9312+
bool llvm::matchSimpleRecurrence(const Instruction *I, PHINode *&P,
92769313
Value *&Start, Value *&Step) {
9277-
BinaryOperator *BO = nullptr;
9314+
Instruction *BO = nullptr;
92789315
P = dyn_cast<PHINode>(I->getOperand(0));
92799316
if (!P)
92809317
P = dyn_cast<PHINode>(I->getOperand(1));

‎llvm/test/Transforms/InferAlignment/gep-recurrence.ll

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ define void @test_recur_i8_128(ptr align 128 %dst) {
1212
; CHECK-NEXT: br label [[LOOP:%.*]]
1313
; CHECK: loop:
1414
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
15-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
15+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
1616
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 128
1717
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
1818
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -40,7 +40,7 @@ define void @test_recur_i8_128_no_inbounds(ptr align 128 %dst) {
4040
; CHECK-NEXT: br label [[LOOP:%.*]]
4141
; CHECK: loop:
4242
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
43-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
43+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
4444
; CHECK-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 128
4545
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
4646
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -68,7 +68,7 @@ define void @test_recur_i8_64(ptr align 128 %dst) {
6868
; CHECK-NEXT: br label [[LOOP:%.*]]
6969
; CHECK: loop:
7070
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
71-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
71+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 64
7272
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 64
7373
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
7474
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -124,7 +124,7 @@ define void @test_recur_i8_32(ptr align 128 %dst) {
124124
; CHECK-NEXT: br label [[LOOP:%.*]]
125125
; CHECK: loop:
126126
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
127-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
127+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 32
128128
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 32
129129
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
130130
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -152,7 +152,7 @@ define void @test_recur_i8_16(ptr align 128 %dst) {
152152
; CHECK-NEXT: br label [[LOOP:%.*]]
153153
; CHECK: loop:
154154
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
155-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
155+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 16
156156
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 16
157157
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
158158
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -180,7 +180,7 @@ define void @test_recur_i8_8(ptr align 128 %dst) {
180180
; CHECK-NEXT: br label [[LOOP:%.*]]
181181
; CHECK: loop:
182182
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
183-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
183+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 8
184184
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 8
185185
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
186186
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -208,7 +208,7 @@ define void @test_recur_i8_4(ptr align 128 %dst) {
208208
; CHECK-NEXT: br label [[LOOP:%.*]]
209209
; CHECK: loop:
210210
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
211-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
211+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 4
212212
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 4
213213
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
214214
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -236,7 +236,7 @@ define void @test_recur_i8_2(ptr align 128 %dst) {
236236
; CHECK-NEXT: br label [[LOOP:%.*]]
237237
; CHECK: loop:
238238
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
239-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
239+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 2
240240
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 2
241241
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
242242
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -468,7 +468,7 @@ define void @test_recur_i8_neg_128(ptr align 128 %dst) {
468468
; CHECK-NEXT: br label [[LOOP:%.*]]
469469
; CHECK: loop:
470470
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
471-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
471+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 128
472472
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -128
473473
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
474474
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -496,7 +496,7 @@ define void @test_recur_i8_neg64(ptr align 128 %dst) {
496496
; CHECK-NEXT: br label [[LOOP:%.*]]
497497
; CHECK: loop:
498498
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
499-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
499+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 64
500500
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -64
501501
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
502502
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
@@ -552,7 +552,7 @@ define void @test_recur_i8_neg_32(ptr align 128 %dst) {
552552
; CHECK-NEXT: br label [[LOOP:%.*]]
553553
; CHECK: loop:
554554
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
555-
; CHECK-NEXT: store i64 0, ptr [[IV]], align 1
555+
; CHECK-NEXT: store i64 0, ptr [[IV]], align 32
556556
; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds i8, ptr [[IV]], i64 -32
557557
; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
558558
; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]

0 commit comments

Comments
 (0)