Skip to content

Commit 215e52a

Browse files
LiqinWengLiqinWeng
committed
[LV][VPlan] Extract the implementation of transform Recipe to EVLRecipe into a small function
1 parent dc936f3 commit 215e52a

File tree

1 file changed

+96
-98
lines changed

1 file changed

+96
-98
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 96 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,13 +1442,100 @@ void VPlanTransforms::addActiveLaneMask(
14421442
HeaderMask->replaceAllUsesWith(LaneMask);
14431443
}
14441444

1445+
static VPRecipeBase *createEVLRecipe(VPValue &EVL, VPValue *HeaderMask,
1446+
VPValue *AllOneMask,
1447+
VPRecipeBase *CurRecipe,
1448+
VPTypeAnalysis TypeInfo) {
1449+
using namespace llvm::VPlanPatternMatch;
1450+
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1451+
assert(OrigMask && "Unmasked recipe when folding tail");
1452+
return HeaderMask == OrigMask ? nullptr : OrigMask;
1453+
};
1454+
1455+
return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1456+
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1457+
VPValue *NewMask = GetNewMask(L->getMask());
1458+
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1459+
})
1460+
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1461+
VPValue *NewMask = GetNewMask(S->getMask());
1462+
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1463+
})
1464+
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1465+
unsigned Opcode = W->getOpcode();
1466+
if (!Instruction::isBinaryOp(Opcode) && !Instruction::isUnaryOp(Opcode))
1467+
return nullptr;
1468+
return new VPWidenEVLRecipe(*W, EVL);
1469+
})
1470+
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1471+
VPValue *NewMask = GetNewMask(Red->getCondOp());
1472+
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1473+
})
1474+
.Case<VPWidenIntrinsicRecipe>(
1475+
[&](VPWidenIntrinsicRecipe *CInst) -> VPRecipeBase * {
1476+
auto *CI = cast<CallInst>(CInst->getUnderlyingInstr());
1477+
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1478+
CI->getCalledFunction()->getIntrinsicID());
1479+
assert(VPID != Intrinsic::not_intrinsic &&
1480+
"Expected VP Instrinsic");
1481+
1482+
SmallVector<VPValue *> Ops(CInst->operands());
1483+
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1484+
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1485+
"Expected VP intrinsic");
1486+
1487+
Ops.push_back(AllOneMask);
1488+
Ops.push_back(&EVL);
1489+
return new VPWidenIntrinsicRecipe(*CI, VPID, Ops,
1490+
TypeInfo.inferScalarType(CInst),
1491+
CInst->getDebugLoc());
1492+
})
1493+
.Case<VPWidenCastRecipe>([&](VPWidenCastRecipe *CInst) -> VPRecipeBase * {
1494+
auto *CI = dyn_cast<CastInst>(CInst->getUnderlyingInstr());
1495+
Intrinsic::ID VPID = VPIntrinsic::getForOpcode(CI->getOpcode());
1496+
assert(VPID != Intrinsic::not_intrinsic &&
1497+
"Expected vp.casts Instrinsic");
1498+
1499+
SmallVector<VPValue *> Ops(CInst->operands());
1500+
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1501+
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1502+
"Expected VP intrinsic");
1503+
Ops.push_back(AllOneMask);
1504+
Ops.push_back(&EVL);
1505+
return new VPWidenIntrinsicRecipe(
1506+
VPID, Ops, TypeInfo.inferScalarType(CInst), CInst->getDebugLoc());
1507+
})
1508+
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1509+
SmallVector<VPValue *> Ops(Sel->operands());
1510+
Ops.push_back(&EVL);
1511+
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1512+
TypeInfo.inferScalarType(Sel),
1513+
Sel->getDebugLoc());
1514+
})
1515+
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1516+
VPValue *LHS, *RHS;
1517+
// Transform select with a header mask condition
1518+
// select(header_mask, LHS, RHS)
1519+
// into vector predication merge.
1520+
// vp.merge(all-true, LHS, RHS, EVL)
1521+
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1522+
m_VPValue(RHS))))
1523+
return nullptr;
1524+
// Use all true as the condition because this transformation is
1525+
// limited to selects whose condition is a header mask.
1526+
return new VPWidenIntrinsicRecipe(
1527+
Intrinsic::vp_merge, {AllOneMask, LHS, RHS, &EVL},
1528+
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1529+
})
1530+
.Default([&](VPRecipeBase *R) { return nullptr; });
1531+
}
1532+
14451533
/// Replace recipes with their EVL variants.
14461534
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
1447-
using namespace llvm::VPlanPatternMatch;
14481535
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
14491536
VPTypeAnalysis TypeInfo(CanonicalIVType);
14501537
LLVMContext &Ctx = CanonicalIVType->getContext();
1451-
SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan);
1538+
VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
14521539

14531540
for (VPUser *U : Plan.getVF().users()) {
14541541
if (auto *R = dyn_cast<VPReverseVectorPointerRecipe>(U))
@@ -1460,111 +1547,22 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
14601547
for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
14611548
for (VPUser *U : collectUsersRecursively(HeaderMask)) {
14621549
auto *CurRecipe = cast<VPRecipeBase>(U);
1463-
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1464-
assert(OrigMask && "Unmasked recipe when folding tail");
1465-
return HeaderMask == OrigMask ? nullptr : OrigMask;
1466-
};
1467-
1468-
VPRecipeBase *NewRecipe =
1469-
TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1470-
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1471-
VPValue *NewMask = GetNewMask(L->getMask());
1472-
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1473-
})
1474-
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1475-
VPValue *NewMask = GetNewMask(S->getMask());
1476-
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1477-
})
1478-
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1479-
unsigned Opcode = W->getOpcode();
1480-
if (!Instruction::isBinaryOp(Opcode) &&
1481-
!Instruction::isUnaryOp(Opcode))
1482-
return nullptr;
1483-
return new VPWidenEVLRecipe(*W, EVL);
1484-
})
1485-
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1486-
VPValue *NewMask = GetNewMask(Red->getCondOp());
1487-
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1488-
})
1489-
.Case<VPWidenIntrinsicRecipe>(
1490-
[&](VPWidenIntrinsicRecipe *CInst) -> VPRecipeBase * {
1491-
auto *CI = cast<CallInst>(CInst->getUnderlyingInstr());
1492-
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1493-
CI->getCalledFunction()->getIntrinsicID());
1494-
if (VPID == Intrinsic::not_intrinsic)
1495-
return nullptr;
1496-
1497-
SmallVector<VPValue *> Ops(CInst->operands());
1498-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1499-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1500-
"Expected VP intrinsic");
1501-
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(
1502-
IntegerType::getInt1Ty(CI->getContext())));
1503-
Ops.push_back(Mask);
1504-
Ops.push_back(&EVL);
1505-
return new VPWidenIntrinsicRecipe(
1506-
*CI, VPID, Ops, TypeInfo.inferScalarType(CInst),
1507-
CInst->getDebugLoc());
1508-
})
1509-
.Case<VPWidenCastRecipe>(
1510-
[&](VPWidenCastRecipe *CastR) -> VPRecipeBase * {
1511-
Intrinsic::ID VPID =
1512-
VPIntrinsic::getForOpcode(CastR->getOpcode());
1513-
assert(VPID != Intrinsic::not_intrinsic &&
1514-
"Expected vp.casts Instrinsic");
1515-
1516-
SmallVector<VPValue *> Ops(CastR->operands());
1517-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1518-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1519-
"Expected VP intrinsic");
1520-
VPValue *Mask =
1521-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1522-
Ops.push_back(Mask);
1523-
Ops.push_back(&EVL);
1524-
return new VPWidenIntrinsicRecipe(
1525-
VPID, Ops, TypeInfo.inferScalarType(CastR),
1526-
CastR->getDebugLoc());
1527-
})
1528-
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1529-
SmallVector<VPValue *> Ops(Sel->operands());
1530-
Ops.push_back(&EVL);
1531-
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1532-
TypeInfo.inferScalarType(Sel),
1533-
Sel->getDebugLoc());
1534-
})
1535-
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1536-
VPValue *LHS, *RHS;
1537-
// Transform select with a header mask condition
1538-
// select(header_mask, LHS, RHS)
1539-
// into vector predication merge.
1540-
// vp.merge(all-true, LHS, RHS, EVL)
1541-
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1542-
m_VPValue(RHS))))
1543-
return nullptr;
1544-
// Use all true as the condition because this transformation is
1545-
// limited to selects whose condition is a header mask.
1546-
VPValue *AllTrue =
1547-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1548-
return new VPWidenIntrinsicRecipe(
1549-
Intrinsic::vp_merge, {AllTrue, LHS, RHS, &EVL},
1550-
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1551-
})
1552-
.Default([&](VPRecipeBase *R) { return nullptr; });
1553-
1554-
if (!NewRecipe)
1550+
VPRecipeBase *EVLRecipe =
1551+
createEVLRecipe(EVL, HeaderMask, AllOneMask, CurRecipe, TypeInfo);
1552+
if (!EVLRecipe)
15551553
continue;
15561554

1557-
[[maybe_unused]] unsigned NumDefVal = NewRecipe->getNumDefinedValues();
1555+
[[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
15581556
assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
15591557
"New recipe must define the same number of values as the "
15601558
"original.");
15611559
assert(
15621560
NumDefVal <= 1 &&
15631561
"Only supports recipes with a single definition or without users.");
1564-
NewRecipe->insertBefore(CurRecipe);
1565-
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(NewRecipe)) {
1562+
EVLRecipe->insertBefore(CurRecipe);
1563+
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
15661564
VPValue *CurVPV = CurRecipe->getVPSingleValue();
1567-
CurVPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
1565+
CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
15681566
}
15691567
// Defer erasing recipes till the end so that we don't invalidate the
15701568
// VPTypeAnalysis cache.

0 commit comments

Comments
 (0)