Skip to content

Commit eaa5b4c

Browse files
aeubanksaeubanks
committed
[SROA] Unfold gep of index phi
If a gep has only one phi as one of its operands and the remaining indexes are constant, we can unfold `gep ptr, (phi idx1, idx2)` to `phi ((gep ptr, idx1), (gep ptr, idx2))`. Take care not to unfold recursive phis. Followup to llvm#80983.
1 parent 9de78c4 commit eaa5b4c

File tree

3 files changed

+161
-73
lines changed

3 files changed

+161
-73
lines changed

‎llvm/lib/Transforms/Scalar/SROA.cpp

Lines changed: 68 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3985,8 +3985,8 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
39853985
return false;
39863986

39873987
LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):"
3988-
<< "\n original: " << *Sel
3989-
<< "\n " << GEPI);
3988+
<< "\n original: " << *Sel << "\n "
3989+
<< GEPI);
39903990

39913991
auto GetNewOps = [&](Value *SelOp) {
39923992
SmallVector<Value *> NewOps;
@@ -4023,64 +4023,88 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
40234023
Visited.insert(NSelI);
40244024
enqueueUsers(*NSelI);
40254025

4026-
LLVM_DEBUG(dbgs() << "\n to: " << *NTrue
4027-
<< "\n " << *NFalse
4028-
<< "\n " << *NSel << '\n');
4026+
LLVM_DEBUG(dbgs() << "\n to: " << *NTrue << "\n "
4027+
<< *NFalse << "\n " << *NSel << '\n');
40294028

40304029
return true;
40314030
}
40324031

4033-
// Fold gep (phi ptr1, ptr2) => phi gep(ptr1), gep(ptr2)
4032+
// Fold gep (phi ptr1, ptr2), idx
4033+
// => phi ((gep ptr1, idx), (gep ptr2, idx))
4034+
// and gep ptr, (phi idx1, idx2)
4035+
// => phi ((gep ptr, idx1), (gep ptr, idx2))
40344036
bool foldGEPPhi(GetElementPtrInst &GEPI) {
4035-
if (!GEPI.hasAllConstantIndices())
4037+
// Check whether the GEP has exactly one phi operand and all indices
4038+
// will become constant after the transform.
4039+
PHINode *Phi = dyn_cast<PHINode>(GEPI.getPointerOperand());
4040+
// To prevent infinitely expanding recursive phis, bail if the GEP pointer
4041+
// operand is the phi and any of its incoming values is not an alloca or a
4042+
// constant.
4043+
if (Phi && any_of(Phi->operands(), [](Value *V) {
4044+
return isa<Instruction>(V) && !isa<AllocaInst>(V);
4045+
})) {
40364046
return false;
4047+
}
4048+
for (Value *Op : GEPI.indices()) {
4049+
if (auto *SI = dyn_cast<PHINode>(Op)) {
4050+
if (Phi)
4051+
return false;
4052+
4053+
Phi = SI;
4054+
if (!all_of(Phi->incoming_values(),
4055+
[](Value *V) { return isa<ConstantInt>(V); }))
4056+
return false;
4057+
continue;
4058+
}
4059+
4060+
if (!isa<ConstantInt>(Op))
4061+
return false;
4062+
}
40374063

4038-
PHINode *PHI = cast<PHINode>(GEPI.getPointerOperand());
4039-
if (GEPI.getParent() != PHI->getParent() ||
4040-
llvm::any_of(PHI->incoming_values(), [](Value *In)
4041-
{ Instruction *I = dyn_cast<Instruction>(In);
4042-
return !I || isa<GetElementPtrInst>(I) || isa<PHINode>(I) ||
4043-
succ_empty(I->getParent()) ||
4044-
!I->getParent()->isLegalToHoistInto();
4045-
}))
4064+
if (!Phi)
40464065
return false;
40474066

40484067
LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):"
4049-
<< "\n original: " << *PHI
4050-
<< "\n " << GEPI
4051-
<< "\n to: ");
4068+
<< "\n original: " << *Phi << "\n "
4069+
<< GEPI);
40524070

4053-
SmallVector<Value *, 4> Index(GEPI.indices());
4054-
bool IsInBounds = GEPI.isInBounds();
4055-
IRB.SetInsertPoint(GEPI.getParent(), GEPI.getParent()->getFirstNonPHIIt());
4056-
PHINode *NewPN = IRB.CreatePHI(GEPI.getType(), PHI->getNumIncomingValues(),
4057-
PHI->getName() + ".sroa.phi");
4058-
for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
4059-
BasicBlock *B = PHI->getIncomingBlock(I);
4060-
Value *NewVal = nullptr;
4061-
int Idx = NewPN->getBasicBlockIndex(B);
4062-
if (Idx >= 0) {
4063-
NewVal = NewPN->getIncomingValue(Idx);
4064-
} else {
4065-
Instruction *In = cast<Instruction>(PHI->getIncomingValue(I));
4071+
auto GetNewOps = [&](Value *PhiOp) {
4072+
SmallVector<Value *> NewOps;
4073+
for (Value *Op : GEPI.operands())
4074+
if (Op == Phi)
4075+
NewOps.push_back(PhiOp);
4076+
else
4077+
NewOps.push_back(Op);
4078+
return NewOps;
4079+
};
40664080

4067-
IRB.SetInsertPoint(In->getParent(), std::next(In->getIterator()));
4068-
Type *Ty = GEPI.getSourceElementType();
4069-
NewVal = IRB.CreateGEP(Ty, In, Index, In->getName() + ".sroa.gep",
4070-
IsInBounds);
4071-
}
4072-
NewPN->addIncoming(NewVal, B);
4081+
IRB.SetInsertPoint(Phi);
4082+
PHINode *NewPhi = IRB.CreatePHI(GEPI.getType(), Phi->getNumIncomingValues(),
4083+
Phi->getName() + ".sroa.phi");
4084+
4085+
bool IsInBounds = GEPI.isInBounds();
4086+
Type *SourceTy = GEPI.getSourceElementType();
4087+
// We only handle constants and static allocas here, so we can insert GEPs
4088+
// at the beginning of the function after static allocas.
4089+
IRB.SetInsertPointPastAllocas(GEPI.getFunction());
4090+
for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4091+
Value *Op = Phi->getIncomingValue(I);
4092+
BasicBlock *BB = Phi->getIncomingBlock(I);
4093+
SmallVector<Value *> NewOps = GetNewOps(Op);
4094+
4095+
Value *NewGEP =
4096+
IRB.CreateGEP(SourceTy, NewOps[0], ArrayRef(NewOps).drop_front(),
4097+
Phi->getName() + ".sroa.gep", IsInBounds);
4098+
NewPhi->addIncoming(NewGEP, BB);
40734099
}
40744100

40754101
Visited.erase(&GEPI);
4076-
GEPI.replaceAllUsesWith(NewPN);
4102+
GEPI.replaceAllUsesWith(NewPhi);
40774103
GEPI.eraseFromParent();
4078-
Visited.insert(NewPN);
4079-
enqueueUsers(*NewPN);
4104+
Visited.insert(NewPhi);
4105+
enqueueUsers(*NewPhi);
40804106

4081-
LLVM_DEBUG(for (Value *In : NewPN->incoming_values())
4082-
dbgs() << "\n " << *In;
4083-
dbgs() << "\n " << *NewPN << '\n');
4107+
LLVM_DEBUG(dbgs() << "\n to: " << *NewPhi << '\n');
40844108

40854109
return true;
40864110
}
@@ -4089,8 +4113,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
40894113
if (foldGEPSelect(GEPI))
40904114
return true;
40914115

4092-
if (isa<PHINode>(GEPI.getPointerOperand()) &&
4093-
foldGEPPhi(GEPI))
4116+
if (foldGEPPhi(GEPI))
40944117
return true;
40954118

40964119
enqueueUsers(GEPI);

‎llvm/test/Transforms/SROA/phi-and-select.ll

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -114,13 +114,13 @@ define i32 @test3(i32 %x) {
114114
; CHECK-LABEL: @test3(
115115
; CHECK-NEXT: entry:
116116
; CHECK-NEXT: switch i32 [[X:%.*]], label [[BB0:%.*]] [
117-
; CHECK-NEXT: i32 1, label [[BB1:%.*]]
118-
; CHECK-NEXT: i32 2, label [[BB2:%.*]]
119-
; CHECK-NEXT: i32 3, label [[BB3:%.*]]
120-
; CHECK-NEXT: i32 4, label [[BB4:%.*]]
121-
; CHECK-NEXT: i32 5, label [[BB5:%.*]]
122-
; CHECK-NEXT: i32 6, label [[BB6:%.*]]
123-
; CHECK-NEXT: i32 7, label [[BB7:%.*]]
117+
; CHECK-NEXT: i32 1, label [[BB1:%.*]]
118+
; CHECK-NEXT: i32 2, label [[BB2:%.*]]
119+
; CHECK-NEXT: i32 3, label [[BB3:%.*]]
120+
; CHECK-NEXT: i32 4, label [[BB4:%.*]]
121+
; CHECK-NEXT: i32 5, label [[BB5:%.*]]
122+
; CHECK-NEXT: i32 6, label [[BB6:%.*]]
123+
; CHECK-NEXT: i32 7, label [[BB7:%.*]]
124124
; CHECK-NEXT: ]
125125
; CHECK: bb0:
126126
; CHECK-NEXT: br label [[EXIT:%.*]]
@@ -733,6 +733,7 @@ define void @PR20822(i1 %c1, i1 %c2, ptr %ptr) {
733733
; CHECK-LABEL: @PR20822(
734734
; CHECK-NEXT: entry:
735735
; CHECK-NEXT: [[F_SROA_0:%.*]] = alloca i32, align 4
736+
; CHECK-NEXT: [[F1_SROA_GEP:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[PTR:%.*]], i32 0, i32 0
736737
; CHECK-NEXT: br i1 [[C1:%.*]], label [[IF_END:%.*]], label [[FOR_COND:%.*]]
737738
; CHECK: for.cond:
738739
; CHECK-NEXT: br label [[IF_END]]
@@ -742,9 +743,8 @@ define void @PR20822(i1 %c1, i1 %c2, ptr %ptr) {
742743
; CHECK: if.then2:
743744
; CHECK-NEXT: br label [[IF_THEN5]]
744745
; CHECK: if.then5:
745-
; CHECK-NEXT: [[F1:%.*]] = phi ptr [ [[PTR:%.*]], [[IF_THEN2]] ], [ [[F_SROA_0]], [[IF_END]] ]
746-
; CHECK-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[F1]], i32 0, i32 0
747-
; CHECK-NEXT: store i32 0, ptr [[DOTFCA_0_GEP]], align 4
746+
; CHECK-NEXT: [[F1_SROA_PHI:%.*]] = phi ptr [ [[F1_SROA_GEP]], [[IF_THEN2]] ], [ [[F_SROA_0]], [[IF_END]] ]
747+
; CHECK-NEXT: store i32 0, ptr [[F1_SROA_PHI]], align 4
748748
; CHECK-NEXT: ret void
749749
;
750750
entry:

‎llvm/test/Transforms/SROA/phi-gep.ll

Lines changed: 83 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,13 @@ end:
6565
define i32 @test_sroa_phi_gep_poison(i1 %cond) {
6666
; CHECK-LABEL: @test_sroa_phi_gep_poison(
6767
; CHECK-NEXT: entry:
68-
; CHECK-NEXT: [[A:%.*]] = alloca [[PAIR:%.*]], align 4
6968
; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[END:%.*]]
7069
; CHECK: if.then:
70+
; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN:%.*]] = load i32, ptr poison, align 4
7171
; CHECK-NEXT: br label [[END]]
7272
; CHECK: end:
73-
; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ poison, [[IF_THEN]] ]
74-
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1
75-
; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
76-
; CHECK-NEXT: ret i32 [[LOAD]]
73+
; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN]], [[IF_THEN]] ]
74+
; CHECK-NEXT: ret i32 [[PHI_SROA_PHI_SROA_SPECULATED]]
7775
;
7876
entry:
7977
%a = alloca %pair, align 4
@@ -94,17 +92,13 @@ end:
9492
define i32 @test_sroa_phi_gep_global(i1 %cond) {
9593
; CHECK-LABEL: @test_sroa_phi_gep_global(
9694
; CHECK-NEXT: entry:
97-
; CHECK-NEXT: [[A:%.*]] = alloca [[PAIR:%.*]], align 4
98-
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i32 0, i32 1
99-
; CHECK-NEXT: store i32 1, ptr [[GEP_A]], align 4
10095
; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[END:%.*]]
10196
; CHECK: if.then:
97+
; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN:%.*]] = load i32, ptr getelementptr inbounds ([[PAIR:%.*]], ptr @g, i32 0, i32 1), align 4
10298
; CHECK-NEXT: br label [[END]]
10399
; CHECK: end:
104-
; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ @g, [[IF_THEN]] ]
105-
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1
106-
; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
107-
; CHECK-NEXT: ret i32 [[LOAD]]
100+
; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN]], [[IF_THEN]] ]
101+
; CHECK-NEXT: ret i32 [[PHI_SROA_PHI_SROA_SPECULATED]]
108102
;
109103
entry:
110104
%a = alloca %pair, align 4
@@ -245,15 +239,15 @@ define i32 @test_sroa_invoke_phi_gep(i1 %cond) personality ptr @__gxx_personalit
245239
; CHECK-NEXT: br i1 [[COND:%.*]], label [[CALL:%.*]], label [[END:%.*]]
246240
; CHECK: call:
247241
; CHECK-NEXT: [[B:%.*]] = invoke ptr @foo()
248-
; CHECK-NEXT: to label [[END]] unwind label [[INVOKE_CATCH:%.*]]
242+
; CHECK-NEXT: to label [[END]] unwind label [[INVOKE_CATCH:%.*]]
249243
; CHECK: end:
250244
; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[CALL]] ]
251245
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1
252246
; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
253247
; CHECK-NEXT: ret i32 [[LOAD]]
254248
; CHECK: invoke_catch:
255249
; CHECK-NEXT: [[RES:%.*]] = landingpad { ptr, i32 }
256-
; CHECK-NEXT: catch ptr null
250+
; CHECK-NEXT: catch ptr null
257251
; CHECK-NEXT: ret i32 0
258252
;
259253
entry:
@@ -468,10 +462,10 @@ define i32 @test_sroa_phi_gep_multiple_values_from_same_block(i32 %arg) {
468462
; CHECK-LABEL: @test_sroa_phi_gep_multiple_values_from_same_block(
469463
; CHECK-NEXT: bb.1:
470464
; CHECK-NEXT: switch i32 [[ARG:%.*]], label [[BB_3:%.*]] [
471-
; CHECK-NEXT: i32 1, label [[BB_2:%.*]]
472-
; CHECK-NEXT: i32 2, label [[BB_2]]
473-
; CHECK-NEXT: i32 3, label [[BB_4:%.*]]
474-
; CHECK-NEXT: i32 4, label [[BB_4]]
465+
; CHECK-NEXT: i32 1, label [[BB_2:%.*]]
466+
; CHECK-NEXT: i32 2, label [[BB_2]]
467+
; CHECK-NEXT: i32 3, label [[BB_4:%.*]]
468+
; CHECK-NEXT: i32 4, label [[BB_4]]
475469
; CHECK-NEXT: ]
476470
; CHECK: bb.2:
477471
; CHECK-NEXT: br label [[BB_4]]
@@ -504,6 +498,77 @@ bb.4: ; preds = %bb.1, %bb.1, %bb
504498
ret i32 %load
505499
}
506500

501+
define i64 @test_phi_idx_mem2reg_const(i1 %arg) {
502+
; CHECK-LABEL: @test_phi_idx_mem2reg_const(
503+
; CHECK-NEXT: bb:
504+
; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
505+
; CHECK: bb1:
506+
; CHECK-NEXT: br label [[END:%.*]]
507+
; CHECK: bb2:
508+
; CHECK-NEXT: br label [[END]]
509+
; CHECK: end:
510+
; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i64 [ 2, [[BB1]] ], [ 3, [[BB2]] ]
511+
; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, [[BB1]] ], [ 1, [[BB2]] ]
512+
; CHECK-NEXT: ret i64 [[PHI_SROA_PHI_SROA_SPECULATED]]
513+
;
514+
bb:
515+
%alloca = alloca [2 x i64], align 8
516+
%gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
517+
store i64 2, ptr %alloca
518+
store i64 3, ptr %gep1
519+
br i1 %arg, label %bb1, label %bb2
520+
521+
bb1:
522+
br label %end
523+
524+
bb2:
525+
br label %end
526+
527+
end:
528+
%phi = phi i64 [ 0, %bb1 ], [ 1, %bb2 ]
529+
%getelementptr = getelementptr inbounds i64, ptr %alloca, i64 %phi
530+
%load = load i64, ptr %getelementptr
531+
ret i64 %load
532+
}
533+
534+
define i64 @test_phi_idx_mem2reg_not_const(i1 %arg, i64 %idx) {
535+
; CHECK-LABEL: @test_phi_idx_mem2reg_not_const(
536+
; CHECK-NEXT: bb:
537+
; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [2 x i64], align 8
538+
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 1
539+
; CHECK-NEXT: store i64 2, ptr [[ALLOCA]], align 4
540+
; CHECK-NEXT: store i64 3, ptr [[GEP1]], align 4
541+
; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
542+
; CHECK: bb1:
543+
; CHECK-NEXT: br label [[END:%.*]]
544+
; CHECK: bb2:
545+
; CHECK-NEXT: br label [[END]]
546+
; CHECK: end:
547+
; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, [[BB1]] ], [ [[IDX:%.*]], [[BB2]] ]
548+
; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 [[PHI]]
549+
; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GETELEMENTPTR]], align 4
550+
; CHECK-NEXT: ret i64 [[LOAD]]
551+
;
552+
bb:
553+
%alloca = alloca [2 x i64], align 8
554+
%gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
555+
store i64 2, ptr %alloca
556+
store i64 3, ptr %gep1
557+
br i1 %arg, label %bb1, label %bb2
558+
559+
bb1:
560+
br label %end
561+
562+
bb2:
563+
br label %end
564+
565+
end:
566+
%phi = phi i64 [ 0, %bb1 ], [ %idx, %bb2 ]
567+
%getelementptr = getelementptr inbounds i64, ptr %alloca, i64 %phi
568+
%load = load i64, ptr %getelementptr
569+
ret i64 %load
570+
}
571+
507572
declare ptr @foo()
508573

509574
declare i32 @__gxx_personality_v0(...)

0 commit comments

Comments
 (0)