Skip to content

[RISCV] Move performCombineVMergeAndVOps to RISCVVectorPeephole #144076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[RISCV] Move performCombineVMergeAndVOps to RISCVVectorPeephole #144076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
010a8f4
[RISCV] Move performCombineVMergeAndVOps to RISCVVectorPeephole
lukel97 Jun 13, 2025
f2f28f5
Update == RISCV::NoRegister checks
lukel97 Jun 13, 2025
8f566c4
Its -> it's, use operands_begin() to get operand iterator
lukel97 Jun 13, 2025
2d77d4f
Merge branch 'main' into vector-peephole/fold-vmerge-to-mask
lukel97 Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
214 changes: 0 additions & 214 deletions llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,6 @@ void RISCVDAGToDAGISel::PostprocessISelDAG() {

CurDAG->setRoot(Dummy.getValue());

MadeChange |= doPeepholeMergeVVMFold();

// After we're done with everything else, convert IMPLICIT_DEF
// passthru operands to NoRegister. This is required to workaround
// an optimization deficiency in MachineCSE. This really should
Expand Down Expand Up @@ -4069,218 +4067,6 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
return true;
}

static bool IsVMerge(SDNode *N) {
return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
}

// Try to fold away VMERGE_VVM instructions into their true operands:
//
// %true = PseudoVADD_VV ...
// %x = PseudoVMERGE_VVM %false, %false, %true, %mask
// ->
// %x = PseudoVADD_VV_MASK %false, ..., %mask
//
// We can only fold if vmerge's passthru operand, vmerge's false operand and
// %true's passthru operand (if it has one) are the same. This is because we
// have to consolidate them into one passthru operand in the result.
//
// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
// mask is all ones.
//
// The resulting VL is the minimum of the two VLs.
//
// The resulting policy is the effective policy the vmerge would have had,
// i.e. whether or not it's passthru operand was implicit-def.
bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
SDValue Passthru, False, True, VL, Mask;
assert(IsVMerge(N));
Passthru = N->getOperand(0);
False = N->getOperand(1);
True = N->getOperand(2);
Mask = N->getOperand(3);
VL = N->getOperand(4);

// If the EEW of True is different from vmerge's SEW, then we can't fold.
if (True.getSimpleValueType() != N->getSimpleValueType(0))
return false;

// We require that either passthru and false are the same, or that passthru
// is undefined.
if (Passthru != False && !isImplicitDef(Passthru))
return false;

assert(True.getResNo() == 0 &&
"Expect True is the first output of an instruction.");

// Need N is the exactly one using True.
if (!True.hasOneUse())
return false;

if (!True.isMachineOpcode())
return false;

unsigned TrueOpc = True.getMachineOpcode();
const MCInstrDesc &TrueMCID = TII->get(TrueOpc);
uint64_t TrueTSFlags = TrueMCID.TSFlags;
bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(TrueMCID);

const RISCV::RISCVMaskedPseudoInfo *Info =
RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc);
if (!Info)
return false;

// If True has a passthru operand then it needs to be the same as vmerge's
// False, since False will be used for the result's passthru operand.
if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
SDValue PassthruOpTrue = True->getOperand(0);
if (False != PassthruOpTrue)
return false;
}

// Skip if True has side effect.
if (TII->get(TrueOpc).hasUnmodeledSideEffects())
return false;

unsigned TrueChainOpIdx = True.getNumOperands() - 1;
bool HasChainOp =
True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other;

if (HasChainOp) {
// Avoid creating cycles in the DAG. We must ensure that none of the other
// operands depend on True through it's Chain.
SmallVector<const SDNode *, 4> LoopWorklist;
SmallPtrSet<const SDNode *, 16> Visited;
LoopWorklist.push_back(False.getNode());
LoopWorklist.push_back(Mask.getNode());
LoopWorklist.push_back(VL.getNode());
if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
return false;
}

// The vector policy operand may be present for masked intrinsics
bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags);
unsigned TrueVLIndex =
True.getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
SDValue TrueVL = True.getOperand(TrueVLIndex);
SDValue SEW = True.getOperand(TrueVLIndex + 1);

auto GetMinVL = [](SDValue LHS, SDValue RHS) {
if (LHS == RHS)
return LHS;
if (isAllOnesConstant(LHS))
return RHS;
if (isAllOnesConstant(RHS))
return LHS;
auto *CLHS = dyn_cast<ConstantSDNode>(LHS);
auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
if (!CLHS || !CRHS)
return SDValue();
return CLHS->getZExtValue() <= CRHS->getZExtValue() ? LHS : RHS;
};

// Because N and True must have the same passthru operand (or True's operand
// is implicit_def), the "effective" body is the minimum of their VLs.
SDValue OrigVL = VL;
VL = GetMinVL(TrueVL, VL);
if (!VL)
return false;

// Some operations produce different elementwise results depending on the
// active elements, like viota.m or vredsum. This transformation is illegal
// for these if we change the active elements (i.e. mask or VL).
const MCInstrDesc &TrueBaseMCID = TII->get(RISCV::getRVVMCOpcode(TrueOpc));
if (RISCVII::elementsDependOnVL(TrueBaseMCID.TSFlags) && (TrueVL != VL))
return false;
if (RISCVII::elementsDependOnMask(TrueBaseMCID.TSFlags) &&
(Mask && !usesAllOnesMask(Mask)))
return false;

// Make sure it doesn't raise any observable fp exceptions, since changing the
// active elements will affect how fflags is set.
if (mayRaiseFPException(True.getNode()) && !True->getFlags().hasNoFPExcept())
return false;

SDLoc DL(N);

unsigned MaskedOpc = Info->MaskedPseudo;
#ifndef NDEBUG
const MCInstrDesc &MaskedMCID = TII->get(MaskedOpc);
assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) &&
"Expected instructions with mask have policy operand.");
assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
MCOI::TIED_TO) == 0 &&
"Expected instructions with mask have a tied dest.");
#endif

// Use a tumu policy, relaxing it to tail agnostic provided that the passthru
// operand is undefined.
//
// However, if the VL became smaller than what the vmerge had originally, then
// elements past VL that were previously in the vmerge's body will have moved
// to the tail. In that case we always need to use tail undisturbed to
// preserve them.
bool MergeVLShrunk = VL != OrigVL;
uint64_t Policy = (isImplicitDef(Passthru) && !MergeVLShrunk)
? RISCVVType::TAIL_AGNOSTIC
: /*TUMU*/ 0;
SDValue PolicyOp =
CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());


SmallVector<SDValue, 8> Ops;
Ops.push_back(False);

const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
const unsigned NormalOpsEnd = TrueVLIndex - HasRoundingMode;
Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + NormalOpsEnd);

Ops.push_back(Mask);

// For unmasked "VOp" with rounding mode operand, that is interfaces like
// (..., rm, vl) or (..., rm, vl, policy).
// Its masked version is (..., vm, rm, vl, policy).
// Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td
if (HasRoundingMode)
Ops.push_back(True->getOperand(TrueVLIndex - 1));

Ops.append({VL, SEW, PolicyOp});

// Result node should have chain operand of True.
if (HasChainOp)
Ops.push_back(True.getOperand(TrueChainOpIdx));

MachineSDNode *Result =
CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
Result->setFlags(True->getFlags());

if (!cast<MachineSDNode>(True)->memoperands_empty())
CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(True)->memoperands());

// Replace vmerge.vvm node by Result.
ReplaceUses(SDValue(N, 0), SDValue(Result, 0));

// Replace another value of True. E.g. chain and VL.
for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));

return true;
}

bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
bool MadeChange = false;
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();

while (Position != CurDAG->allnodes_begin()) {
SDNode *N = &*--Position;
if (N->use_empty() || !N->isMachineOpcode())
continue;

if (IsVMerge(N))
MadeChange |= performCombineVMergeAndVOps(N);
}
return MadeChange;
}

/// If our passthru is an implicit_def, use noreg instead. This side
/// steps issues with MachineCSE not being able to CSE expressions with
/// IMPLICIT_DEF operands while preserving the semantic intent. See
Expand Down
1 change: 0 additions & 1 deletion llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,6 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
private:
bool doPeepholeSExtW(SDNode *Node);
bool doPeepholeMaskedRVV(MachineSDNode *Node);
bool doPeepholeMergeVVMFold();
bool doPeepholeNoRegPassThru();
bool performCombineVMergeAndVOps(SDNode *N);
bool selectImm64IfCheaper(int64_t Imm, int64_t OrigImm, SDValue N,
Expand Down
Loading
Loading