Skip to content

Commit 8004c2e

Browse files
committed
[DAGCombiner] Add DAG combine for PARTIAL_REDUCE_MLA when no mul op
Generic DAG combine for ISD::PARTIAL_REDUCE_U/SMLA to convert: PARTIAL_REDUCE_*MLA(Acc, ZEXT(UnextOp1), Splat(1)) into PARTIAL_REDUCE_UMLA(Acc, UnextOp1, TRUNC(Splat(1))) and PARTIAL_REDUCE_*MLA(Acc, SEXT(UnextOp1), Splat(1)) into PARTIAL_REDUCE_SMLA(Acc, UnextOp1, TRUNC(Splat(1))). # Conflicts: # llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
1 parent 543f112 commit 8004c2e

File tree

1 file changed

+53
-1
lines changed

1 file changed

+53
-1
lines changed

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,8 @@ namespace {
618618
SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
619619
SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
620620
const TargetLowering &TLI);
621+
SDValue foldPartialReduceMLAMulOp(SDNode *N);
622+
SDValue foldPartialReduceMLANoMulOp(SDNode *N);
621623

622624
SDValue CombineExtLoad(SDNode *N);
623625
SDValue CombineZExtLogicopShiftLoad(SDNode *N);
@@ -12612,13 +12614,21 @@ SDValue DAGCombiner::visitMHISTOGRAM(SDNode *N) {
1261212614
return SDValue();
1261312615
}
1261412616

12617+
SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
12618+
if (SDValue Res = foldPartialReduceMLAMulOp(N))
12619+
return Res;
12620+
if (SDValue Res = foldPartialReduceMLANoMulOp(N))
12621+
return Res;
12622+
return SDValue();
12623+
}
12624+
1261512625
// Makes PARTIAL_REDUCE_*MLA(Acc, MUL(ZEXT(LHSExtOp), ZEXT(RHSExtOp)),
1261612626
// Splat(1)) into
1261712627
// PARTIAL_REDUCE_UMLA(Acc, LHSExtOp, RHSExtOp).
1261812628
// Makes PARTIAL_REDUCE_*MLA(Acc, MUL(SEXT(LHSExtOp), SEXT(RHSExtOp)),
1261912629
// Splat(1)) into
1262012630
// PARTIAL_REDUCE_SMLA(Acc, LHSExtOp, RHSExtOp).
12621-
SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
12631+
SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
1262212632
SDLoc DL(N);
1262312633

1262412634
SDValue Acc = N->getOperand(0);
@@ -12669,6 +12679,48 @@ SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
1266912679
RHSExtOp);
1267012680
}
1267112681

12682+
// Makes PARTIAL_REDUCE_*MLA(Acc, ZEXT(UnextOp1), Splat(1)) into
12683+
// PARTIAL_REDUCE_UMLA(Acc, Op, TRUNC(Splat(1)))
12684+
// Makes PARTIAL_REDUCE_*MLA(Acc, SEXT(UnextOp1), Splat(1)) into
12685+
// PARTIAL_REDUCE_SMLA(Acc, Op, TRUNC(Splat(1)))
12686+
SDValue DAGCombiner::foldPartialReduceMLANoMulOp(SDNode *N) {
12687+
SDLoc DL(N);
12688+
SDValue Acc = N->getOperand(0);
12689+
SDValue Op1 = N->getOperand(1);
12690+
SDValue Op2 = N->getOperand(2);
12691+
12692+
APInt ConstantOne;
12693+
if (!ISD::isConstantSplatVector(Op2.getNode(), ConstantOne) ||
12694+
!ConstantOne.isOne())
12695+
return SDValue();
12696+
12697+
unsigned Op1Opcode = Op1.getOpcode();
12698+
if (!ISD::isExtOpcode(Op1Opcode))
12699+
return SDValue();
12700+
12701+
SDValue UnextOp1 = Op1.getOperand(0);
12702+
EVT UnextOp1VT = UnextOp1.getValueType();
12703+
12704+
if (!TLI.isPartialReduceMLALegalOrCustom(N->getValueType(0), UnextOp1VT))
12705+
return SDValue();
12706+
12707+
SDValue TruncOp2 = DAG.getNode(ISD::TRUNCATE, DL, UnextOp1VT, Op2);
12708+
12709+
bool Op1IsSigned = Op1Opcode == ISD::SIGN_EXTEND;
12710+
12711+
bool NodeIsSigned = N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA;
12712+
EVT AccElemVT = Acc.getValueType().getVectorElementType();
12713+
if (Op1IsSigned != NodeIsSigned &&
12714+
(Op1.getValueType().getVectorElementType() != AccElemVT ||
12715+
Op2.getValueType().getVectorElementType() != AccElemVT))
12716+
return SDValue();
12717+
12718+
unsigned NewOpcode =
12719+
Op1IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
12720+
return DAG.getNode(NewOpcode, DL, N->getValueType(0), Acc, UnextOp1,
12721+
TruncOp2);
12722+
}
12723+
1267212724
SDValue DAGCombiner::visitVP_STRIDED_LOAD(SDNode *N) {
1267312725
auto *SLD = cast<VPStridedLoadSDNode>(N);
1267412726
EVT EltVT = SLD->getValueType(0).getVectorElementType();

0 commit comments

Comments
 (0)