Skip to content

Commit 8bd7b65

Browse files
fhahnfhahn
committed
!fixup rebase and fixup
1 parent 15e5ae1 commit 8bd7b65

File tree

4 files changed

+72
-136
lines changed

4 files changed

+72
-136
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 0 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -9222,87 +9222,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92229222
VPlanTransforms::clearReductionWrapFlags(*Plan);
92239223
}
92249224

9225-
void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
9226-
assert(IndDesc.getKind() == InductionDescriptor::IK_PtrInduction &&
9227-
"Not a pointer induction according to InductionDescriptor!");
9228-
assert(cast<PHINode>(getUnderlyingInstr())->getType()->isPointerTy() &&
9229-
"Unexpected type.");
9230-
assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
9231-
"Recipe should have been replaced");
9232-
9233-
auto *IVR = getParent()->getPlan()->getCanonicalIV();
9234-
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
9235-
unsigned CurrentPart = 0;
9236-
if (getNumOperands() == 5)
9237-
CurrentPart =
9238-
cast<ConstantInt>(getOperand(4)->getLiveInIRValue())->getZExtValue();
9239-
Type *PhiType = IndDesc.getStep()->getType();
9240-
9241-
// Build a pointer phi
9242-
Value *ScalarStartValue = getStartValue()->getLiveInIRValue();
9243-
Type *ScStValueType = ScalarStartValue->getType();
9244-
PHINode *NewPointerPhi = nullptr;
9245-
9246-
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
9247-
if (getNumOperands() == 5) {
9248-
auto *GEP = cast<GetElementPtrInst>(State.get(getOperand(3), 0));
9249-
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
9250-
} else {
9251-
NewPointerPhi =
9252-
PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV);
9253-
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
9254-
}
9255-
9256-
// A pointer induction, performed by using a gep
9257-
BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
9258-
unsigned UF = getNumOperands() == 2
9259-
? 1
9260-
: cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
9261-
->getZExtValue();
9262-
9263-
Value *ScalarStepValue = State.get(getOperand(1), VPIteration(0, 0));
9264-
Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
9265-
Value *NumUnrolledElems =
9266-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));
9267-
// Add induction update using an incorrect block temporarily. The phi node
9268-
// will be fixed after VPlan execution. Note that at this point the latch
9269-
// block cannot be used, as it does not exist yet.
9270-
// TODO: Model increment value in VPlan, by turning the recipe into a
9271-
// multi-def and a subclass of VPHeaderPHIRecipe.
9272-
if (getNumOperands() != 5) {
9273-
Value *InductionGEP = GetElementPtrInst::Create(
9274-
State.Builder.getInt8Ty(), NewPointerPhi,
9275-
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
9276-
InductionLoc);
9277-
9278-
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
9279-
}
9280-
9281-
// Create UF many actual address geps that use the pointer
9282-
// phi as base and a vectorized version of the step value
9283-
// (<step*0, ..., step*N>) as offset.
9284-
for (unsigned Part = 0; Part < State.UF; ++Part) {
9285-
Type *VecPhiType = VectorType::get(PhiType, State.VF);
9286-
Value *StartOffsetScalar = State.Builder.CreateMul(
9287-
RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
9288-
Value *StartOffset =
9289-
State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
9290-
// Create a vector of consecutive numbers from zero to VF.
9291-
StartOffset = State.Builder.CreateAdd(
9292-
StartOffset, State.Builder.CreateStepVector(VecPhiType));
9293-
9294-
assert(ScalarStepValue == State.get(getOperand(1), VPIteration(Part, 0)) &&
9295-
"scalar step must be the same across all parts");
9296-
Value *GEP = State.Builder.CreateGEP(
9297-
State.Builder.getInt8Ty(), NewPointerPhi,
9298-
State.Builder.CreateMul(
9299-
StartOffset,
9300-
State.Builder.CreateVectorSplat(State.VF, ScalarStepValue),
9301-
"vector.gep"));
9302-
State.set(this, GEP, Part);
9303-
}
9304-
}
9305-
93069225
void VPDerivedIVRecipe::execute(VPTransformState &State) {
93079226
assert(!State.Instance && "VPDerivedIVRecipe being replicated.");
93089227

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1682,8 +1682,7 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
16821682
bool onlyFirstPartUsed(const VPValue *Op) const override {
16831683
assert(is_contained(operands(), Op) &&
16841684
"Op must be an operand of the recipe");
1685-
assert(getNumOperands() == 1 && "must have a single operand");
1686-
return true;
1685+
return Op == getOperand(0);
16871686
}
16881687

16891688
VPVectorPointerRecipe *clone() override {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ unsigned VPInstruction::getInterleaveCount() const {
692692
? 1
693693
: cast<ConstantInt>(getOperand(1)->getLiveInIRValue())
694694
->getZExtValue();
695+
}
695696

696697
#if !defined(NDEBUG)
697698
bool VPInstruction::isFPMathOp() const {
@@ -2530,42 +2531,59 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
25302531

25312532
auto *IVR = getParent()->getPlan()->getCanonicalIV();
25322533
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
2534+
unsigned CurrentPart = 0;
2535+
if (getNumOperands() == 5)
2536+
CurrentPart =
2537+
cast<ConstantInt>(getOperand(4)->getLiveInIRValue())->getZExtValue();
25332538
Type *PhiType = IndDesc.getStep()->getType();
25342539

25352540
// Build a pointer phi
25362541
Value *ScalarStartValue = getStartValue()->getLiveInIRValue();
25372542
Type *ScStValueType = ScalarStartValue->getType();
2538-
PHINode *NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
2539-
CanonicalIV->getIterator());
2543+
PHINode *NewPointerPhi = nullptr;
25402544

25412545
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
2542-
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
2546+
if (getNumOperands() == 5) {
2547+
auto *GEP = cast<GetElementPtrInst>(State.get(getOperand(3), 0));
2548+
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
2549+
} else {
2550+
NewPointerPhi =
2551+
PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV);
2552+
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
2553+
}
25432554

25442555
// A pointer induction, performed by using a gep
25452556
BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
2557+
unsigned UF = getNumOperands() == 2
2558+
? 1
2559+
: cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
2560+
->getZExtValue();
25462561

25472562
Value *ScalarStepValue = State.get(getOperand(1), VPIteration(0, 0));
25482563
Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
25492564
Value *NumUnrolledElems =
2550-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF));
2551-
Value *InductionGEP = GetElementPtrInst::Create(
2552-
State.Builder.getInt8Ty(), NewPointerPhi,
2553-
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
2554-
InductionLoc);
2565+
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));
25552566
// Add induction update using an incorrect block temporarily. The phi node
25562567
// will be fixed after VPlan execution. Note that at this point the latch
25572568
// block cannot be used, as it does not exist yet.
25582569
// TODO: Model increment value in VPlan, by turning the recipe into a
25592570
// multi-def and a subclass of VPHeaderPHIRecipe.
2560-
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
2571+
if (getNumOperands() != 5) {
2572+
Value *InductionGEP = GetElementPtrInst::Create(
2573+
State.Builder.getInt8Ty(), NewPointerPhi,
2574+
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
2575+
InductionLoc);
2576+
2577+
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
2578+
}
25612579

25622580
// Create UF many actual address geps that use the pointer
25632581
// phi as base and a vectorized version of the step value
25642582
// (<step*0, ..., step*N>) as offset.
25652583
for (unsigned Part = 0; Part < State.UF; ++Part) {
25662584
Type *VecPhiType = VectorType::get(PhiType, State.VF);
2567-
Value *StartOffsetScalar =
2568-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, Part));
2585+
Value *StartOffsetScalar = State.Builder.CreateMul(
2586+
RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
25692587
Value *StartOffset =
25702588
State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
25712589
// Create a vector of consecutive numbers from zero to VF.

0 commit comments

Comments
 (0)