Skip to content

[SelectionDAG][RISCV] Promote VECREDUCE_{FMAX,FMIN,FMAXIMUM,FMINIMUM} #128800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 28, 2025
Merged

[SelectionDAG][RISCV] Promote VECREDUCE_{FMAX,FMIN,FMAXIMUM,FMINIMUM} #128800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
50cc58c
[SelectionDAG][RISCV] Promote VECREDUCE_{FMAX,FMIN,FMAXIMUM,FMINIMUM}
tclin914 Feb 24, 2025
496b064
clang-format
tclin914 Feb 26, 2025
e43532f
prmote -> promote
tclin914 Feb 26, 2025
2319dde
Add tests for bf16/zvfbfmin
tclin914 Feb 27, 2025
d1cff55
nxv4f16 -> nxv4bf16 for bf16 tests
tclin914 Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2913,31 +2913,34 @@ SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
}

SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
MVT VecVT = Node->getOperand(1).getSimpleValueType();
bool IsVPOpcode = ISD::isVPOpcode(Node->getOpcode());
MVT VecVT = IsVPOpcode ? Node->getOperand(1).getSimpleValueType()
: Node->getOperand(0).getSimpleValueType();
MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT);
MVT ScalarVT = Node->getSimpleValueType(0);
MVT NewScalarVT = NewVecVT.getVectorElementType();

SDLoc DL(Node);
SmallVector<SDValue, 4> Operands(Node->getNumOperands());

// promote the initial value.
// FIXME: Support integer.
assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
"Only FP promotion is supported");
Operands[0] =
DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0));

for (unsigned j = 1; j != Node->getNumOperands(); ++j)
for (unsigned j = 0; j != Node->getNumOperands(); ++j)
if (Node->getOperand(j).getValueType().isVector() &&
!(ISD::isVPOpcode(Node->getOpcode()) &&
!(IsVPOpcode &&
ISD::getVPMaskIdx(Node->getOpcode()) == j)) { // Skip mask operand.
// promote the vector operand.
// FIXME: Support integer.
assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
"Only FP promotion is supported");
Operands[j] =
DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j));
} else if (Node->getOperand(j).getValueType().isFloatingPoint()) {
// promote the initial value.
Operands[j] =
DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(j));
} else {
Operands[j] = Node->getOperand(j); // Skip VL operand.
}
Expand Down Expand Up @@ -5049,7 +5052,11 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Node->getOpcode() == ISD::SINT_TO_FP ||
Node->getOpcode() == ISD::SETCC ||
Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
Node->getOpcode() == ISD::VECREDUCE_FMAX ||
Node->getOpcode() == ISD::VECREDUCE_FMIN ||
Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
OVT = Node->getOperand(0).getSimpleValueType();
}
if (Node->getOpcode() == ISD::ATOMIC_STORE ||
Expand Down Expand Up @@ -5796,6 +5803,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
break;
}
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
case ISD::VP_REDUCE_FMAX:
case ISD::VP_REDUCE_FMIN:
case ISD::VP_REDUCE_FMAXIMUM:
Expand Down
8 changes: 7 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,13 +503,19 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::VECREDUCE_UMIN:
case ISD::VECREDUCE_FADD:
case ISD::VECREDUCE_FMUL:
case ISD::VECTOR_FIND_LAST_ACTIVE:
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(0).getValueType());
break;
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
case ISD::VECTOR_FIND_LAST_ACTIVE:
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(0).getValueType());
// Defer non-vector results to LegalizeDAG.
if (Action == TargetLowering::Promote)
Action = TargetLowering::Legal;
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Expand Down
36 changes: 29 additions & 7 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -959,13 +959,35 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,

// TODO: support more ops.
static const unsigned ZvfhminZvfbfminPromoteOps[] = {
ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, ISD::FSUB,
ISD::FMUL, ISD::FMA, ISD::FDIV, ISD::FSQRT,
ISD::FCEIL, ISD::FTRUNC, ISD::FFLOOR, ISD::FROUND,
ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT, ISD::IS_FPCLASS,
ISD::SETCC, ISD::FMAXIMUM, ISD::FMINIMUM, ISD::STRICT_FADD,
ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
ISD::STRICT_FMA};
ISD::FMINNUM,
ISD::FMAXNUM,
ISD::FADD,
ISD::FSUB,
ISD::FMUL,
ISD::FMA,
ISD::FDIV,
ISD::FSQRT,
ISD::FCEIL,
ISD::FTRUNC,
ISD::FFLOOR,
ISD::FROUND,
ISD::FROUNDEVEN,
ISD::FRINT,
ISD::FNEARBYINT,
ISD::IS_FPCLASS,
ISD::SETCC,
ISD::FMAXIMUM,
ISD::FMINIMUM,
ISD::STRICT_FADD,
ISD::STRICT_FSUB,
ISD::STRICT_FMUL,
ISD::STRICT_FDIV,
ISD::STRICT_FSQRT,
ISD::STRICT_FMA,
ISD::VECREDUCE_FMIN,
ISD::VECREDUCE_FMAX,
ISD::VECREDUCE_FMINIMUM,
ISD::VECREDUCE_FMAXIMUM};

// TODO: support more vp ops.
static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
Expand Down
136 changes: 136 additions & 0 deletions llvm/test/CodeGen/RISCV/rvv/vreductions-fp-sdnode-bf16.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=riscv32 -mattr=+d,+zfbfmin,+zvfbfmin,+v -target-abi=ilp32d \
; RUN: -verify-machineinstrs < %s | FileCheck %s
; RUN: llc -mtriple=riscv64 -mattr=+d,+zfbfmin,+zvfbfmin,+v -target-abi=lp64d \
; RUN: -verify-machineinstrs < %s | FileCheck %s

define bfloat @vreduce_fmin_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmin_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmin.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fmax_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmax_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmax.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fmin_nnan_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmin_nnan_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmin.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call nnan bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fmax_nnan_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmax_nnan_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmax.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call nnan bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fminimum_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fminimum_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vmfne.vv v8, v10, v10
; CHECK-NEXT: vcpop.m a0, v8
; CHECK-NEXT: beqz a0, .LBB4_2
; CHECK-NEXT: # %bb.1:
; CHECK-NEXT: lui a0, 523264
; CHECK-NEXT: fmv.w.x fa5, a0
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB4_2:
; CHECK-NEXT: vfredmin.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fmaximum_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmaximum_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vmfne.vv v8, v10, v10
; CHECK-NEXT: vcpop.m a0, v8
; CHECK-NEXT: beqz a0, .LBB5_2
; CHECK-NEXT: # %bb.1:
; CHECK-NEXT: lui a0, 523264
; CHECK-NEXT: fmv.w.x fa5, a0
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB5_2:
; CHECK-NEXT: vfredmax.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fminimum_nnan_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fminimum_nnan_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmin.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call nnan bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

define bfloat @vreduce_fmaximum_nnan_nxv4bf16(<vscale x 4 x bfloat> %val) {
; CHECK-LABEL: vreduce_fmaximum_nnan_nxv4bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v8
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vfredmax.vs v8, v10, v10
; CHECK-NEXT: vfmv.f.s fa5, v8
; CHECK-NEXT: fcvt.bf16.s fa0, fa5
; CHECK-NEXT: ret
%s = call nnan bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat> %val)
ret bfloat %s
}

Loading
Loading