Skip to content

[RISCV] Split long build_vector sequences to reduce critical path #81312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 14, 2024
Merged

[RISCV] Split long build_vector sequences to reduce critical path #81312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e6e52ee
[RISCV] Split long build_vector sequences to reduce critical path
preames Feb 8, 2024
3324531
Apply suggestions from code review
preames Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3877,6 +3877,47 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}

// For m1 vectors, if we have non-undef values in both halves of our vector,
// split the vector into low and high halves, build them separately, then
// use a vselect to combine them. For long vectors, this cuts the critical
// path of the vslide1down sequence in half, and gives us an opportunity
// to special case each half independently. Note that we don't change the
// length of the sub-vectors here, so if both fallback to the generic
// vslide1down path, we should be able to fold the vselect into the final
// vslidedown (for the undef tail) for the first half w/ masking.
unsigned NumElts = VT.getVectorNumElements();
unsigned NumUndefElts =
count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
unsigned NumDefElts = NumElts - NumUndefElts;
if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
ContainerVT.bitsLE(getLMUL1VT(ContainerVT))) {
SmallVector<SDValue> SubVecAOps, SubVecBOps;
SmallVector<SDValue> MaskVals;
SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
SubVecAOps.reserve(NumElts);
SubVecBOps.reserve(NumElts);
for (unsigned i = 0; i < NumElts; i++) {
SDValue Elem = Op->getOperand(i);
if (i < NumElts / 2) {
SubVecAOps.push_back(Elem);
SubVecBOps.push_back(UndefElem);
} else {
SubVecAOps.push_back(UndefElem);
SubVecBOps.push_back(Elem);
}
bool SelectMaskVal = (i < NumElts / 2);
MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
}
assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
MaskVals.size() == NumElts);

SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
}

// Cap the cost at a value linear to the number of elements in the vector.
// The default lowering is to use the stack. The vector store + scalar loads
// is linear in VL. However, at high lmuls vslide1down and vslidedown end up
Expand Down
16 changes: 10 additions & 6 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1399,15 +1399,17 @@ define <2 x double> @vid_step2_v2f64() {
define <8 x float> @buildvec_v8f32_zvl256(float %e0, float %e1, float %e2, float %e3, float %e4, float %e5, float %e6, float %e7) vscale_range(4, 128) {
; CHECK-LABEL: buildvec_v8f32_zvl256:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma
; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, mu
; CHECK-NEXT: vfmv.v.f v8, fa0
; CHECK-NEXT: vfslide1down.vf v8, v8, fa1
; CHECK-NEXT: vfslide1down.vf v8, v8, fa2
; CHECK-NEXT: vfslide1down.vf v8, v8, fa3
; CHECK-NEXT: vfslide1down.vf v8, v8, fa4
; CHECK-NEXT: vfslide1down.vf v9, v8, fa3
; CHECK-NEXT: vfmv.v.f v8, fa4
; CHECK-NEXT: vfslide1down.vf v8, v8, fa5
; CHECK-NEXT: vfslide1down.vf v8, v8, fa6
; CHECK-NEXT: vmv.v.i v0, 15
; CHECK-NEXT: vfslide1down.vf v8, v8, fa7
; CHECK-NEXT: vslidedown.vi v8, v9, 4, v0.t
; CHECK-NEXT: ret
%v0 = insertelement <8 x float> poison, float %e0, i64 0
%v1 = insertelement <8 x float> %v0, float %e1, i64 1
Expand Down Expand Up @@ -1448,15 +1450,17 @@ define <8 x double> @buildvec_v8f64_zvl256(double %e0, double %e1, double %e2, d
define <8 x double> @buildvec_v8f64_zvl512(double %e0, double %e1, double %e2, double %e3, double %e4, double %e5, double %e6, double %e7) vscale_range(8, 128) {
; CHECK-LABEL: buildvec_v8f64_zvl512:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e64, m1, ta, ma
; CHECK-NEXT: vsetivli zero, 8, e64, m1, ta, mu
; CHECK-NEXT: vfmv.v.f v8, fa0
; CHECK-NEXT: vfslide1down.vf v8, v8, fa1
; CHECK-NEXT: vfslide1down.vf v8, v8, fa2
; CHECK-NEXT: vfslide1down.vf v8, v8, fa3
; CHECK-NEXT: vfslide1down.vf v8, v8, fa4
; CHECK-NEXT: vfslide1down.vf v9, v8, fa3
; CHECK-NEXT: vfmv.v.f v8, fa4
; CHECK-NEXT: vfslide1down.vf v8, v8, fa5
; CHECK-NEXT: vfslide1down.vf v8, v8, fa6
; CHECK-NEXT: vmv.v.i v0, 15
; CHECK-NEXT: vfslide1down.vf v8, v8, fa7
; CHECK-NEXT: vslidedown.vi v8, v9, 4, v0.t
; CHECK-NEXT: ret
%v0 = insertelement <8 x double> poison, double %e0, i64 0
%v1 = insertelement <8 x double> %v0, double %e1, i64 1
Expand Down
112 changes: 60 additions & 52 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,44 +359,46 @@ define void @fp2si_v8f64_v8i8(ptr %x, ptr %y) {
; RV32-NEXT: feq.d a0, fa3, fa3
; RV32-NEXT: fmax.d fa3, fa3, fa5
; RV32-NEXT: fmin.d fa3, fa3, fa4
; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; RV32-NEXT: fld fa2, 40(sp)
; RV32-NEXT: fcvt.w.d a2, fa3, rtz
; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV32-NEXT: fld fa3, 32(sp)
; RV32-NEXT: neg a0, a0
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: vslide1down.vx v8, v10, a0
; RV32-NEXT: feq.d a0, fa3, fa3
; RV32-NEXT: fmax.d fa3, fa3, fa5
; RV32-NEXT: feq.d a2, fa2, fa2
; RV32-NEXT: fmax.d fa3, fa2, fa5
; RV32-NEXT: fmin.d fa3, fa3, fa4
; RV32-NEXT: fcvt.w.d a2, fa3, rtz
; RV32-NEXT: fld fa3, 40(sp)
; RV32-NEXT: neg a0, a0
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: feq.d a0, fa3, fa3
; RV32-NEXT: fcvt.w.d a3, fa3, rtz
; RV32-NEXT: fld fa3, 32(sp)
; RV32-NEXT: vslide1down.vx v8, v10, a0
; RV32-NEXT: neg a0, a2
; RV32-NEXT: and a0, a0, a3
; RV32-NEXT: feq.d a2, fa3, fa3
; RV32-NEXT: neg a2, a2
; RV32-NEXT: fmax.d fa3, fa3, fa5
; RV32-NEXT: fmin.d fa3, fa3, fa4
; RV32-NEXT: fcvt.w.d a2, fa3, rtz
; RV32-NEXT: fcvt.w.d a3, fa3, rtz
; RV32-NEXT: fld fa3, 48(sp)
; RV32-NEXT: neg a0, a0
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: and a2, a2, a3
; RV32-NEXT: vmv.v.x v9, a2
; RV32-NEXT: vslide1down.vx v9, v9, a0
; RV32-NEXT: feq.d a0, fa3, fa3
; RV32-NEXT: fmax.d fa3, fa3, fa5
; RV32-NEXT: fmin.d fa3, fa3, fa4
; RV32-NEXT: fcvt.w.d a2, fa3, rtz
; RV32-NEXT: fld fa3, 56(sp)
; RV32-NEXT: neg a0, a0
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: vslide1down.vx v9, v9, a0
; RV32-NEXT: feq.d a0, fa3, fa3
; RV32-NEXT: neg a0, a0
; RV32-NEXT: fmax.d fa5, fa3, fa5
; RV32-NEXT: fmin.d fa5, fa5, fa4
; RV32-NEXT: fcvt.w.d a2, fa5, rtz
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: vse8.v v8, (a1)
; RV32-NEXT: vmv.v.i v0, 15
; RV32-NEXT: vslide1down.vx v9, v9, a0
; RV32-NEXT: vslidedown.vi v9, v8, 4, v0.t
; RV32-NEXT: vse8.v v9, (a1)
; RV32-NEXT: addi sp, s0, -128
; RV32-NEXT: lw ra, 124(sp) # 4-byte Folded Reload
; RV32-NEXT: lw s0, 120(sp) # 4-byte Folded Reload
Expand Down Expand Up @@ -458,44 +460,46 @@ define void @fp2si_v8f64_v8i8(ptr %x, ptr %y) {
; RV64-NEXT: feq.d a0, fa3, fa3
; RV64-NEXT: fmax.d fa3, fa3, fa5
; RV64-NEXT: fmin.d fa3, fa3, fa4
; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; RV64-NEXT: fld fa2, 40(sp)
; RV64-NEXT: fcvt.l.d a2, fa3, rtz
; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV64-NEXT: fld fa3, 32(sp)
; RV64-NEXT: neg a0, a0
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: vslide1down.vx v8, v10, a0
; RV64-NEXT: feq.d a0, fa3, fa3
; RV64-NEXT: fmax.d fa3, fa3, fa5
; RV64-NEXT: feq.d a2, fa2, fa2
; RV64-NEXT: fmax.d fa3, fa2, fa5
; RV64-NEXT: fmin.d fa3, fa3, fa4
; RV64-NEXT: fcvt.l.d a2, fa3, rtz
; RV64-NEXT: fld fa3, 40(sp)
; RV64-NEXT: neg a0, a0
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: feq.d a0, fa3, fa3
; RV64-NEXT: fcvt.l.d a3, fa3, rtz
; RV64-NEXT: fld fa3, 32(sp)
; RV64-NEXT: vslide1down.vx v8, v10, a0
; RV64-NEXT: neg a0, a2
; RV64-NEXT: and a0, a0, a3
; RV64-NEXT: feq.d a2, fa3, fa3
; RV64-NEXT: negw a2, a2
; RV64-NEXT: fmax.d fa3, fa3, fa5
; RV64-NEXT: fmin.d fa3, fa3, fa4
; RV64-NEXT: fcvt.l.d a2, fa3, rtz
; RV64-NEXT: fcvt.l.d a3, fa3, rtz
; RV64-NEXT: fld fa3, 48(sp)
; RV64-NEXT: neg a0, a0
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: and a2, a2, a3
; RV64-NEXT: vmv.v.x v9, a2
; RV64-NEXT: vslide1down.vx v9, v9, a0
; RV64-NEXT: feq.d a0, fa3, fa3
; RV64-NEXT: fmax.d fa3, fa3, fa5
; RV64-NEXT: fmin.d fa3, fa3, fa4
; RV64-NEXT: fcvt.l.d a2, fa3, rtz
; RV64-NEXT: fld fa3, 56(sp)
; RV64-NEXT: neg a0, a0
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: vslide1down.vx v9, v9, a0
; RV64-NEXT: feq.d a0, fa3, fa3
; RV64-NEXT: neg a0, a0
; RV64-NEXT: fmax.d fa5, fa3, fa5
; RV64-NEXT: fmin.d fa5, fa5, fa4
; RV64-NEXT: fcvt.l.d a2, fa5, rtz
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: vse8.v v8, (a1)
; RV64-NEXT: vmv.v.i v0, 15
; RV64-NEXT: vslide1down.vx v9, v9, a0
; RV64-NEXT: vslidedown.vi v9, v8, 4, v0.t
; RV64-NEXT: vse8.v v9, (a1)
; RV64-NEXT: addi sp, s0, -128
; RV64-NEXT: ld ra, 120(sp) # 8-byte Folded Reload
; RV64-NEXT: ld s0, 112(sp) # 8-byte Folded Reload
Expand Down Expand Up @@ -553,11 +557,11 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
; RV32-NEXT: vslidedown.vi v8, v8, 3
; RV32-NEXT: vfmv.f.s fa4, v8
; RV32-NEXT: fmax.d fa4, fa4, fa3
; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV32-NEXT: fld fa2, 32(sp)
; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; RV32-NEXT: fld fa2, 40(sp)
; RV32-NEXT: fmin.d fa4, fa4, fa5
; RV32-NEXT: fcvt.wu.d a0, fa4, rtz
; RV32-NEXT: fld fa4, 40(sp)
; RV32-NEXT: fld fa4, 32(sp)
; RV32-NEXT: fmax.d fa2, fa2, fa3
; RV32-NEXT: fmin.d fa2, fa2, fa5
; RV32-NEXT: fcvt.wu.d a2, fa2, rtz
Expand All @@ -570,14 +574,16 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
; RV32-NEXT: fmin.d fa4, fa4, fa5
; RV32-NEXT: fcvt.wu.d a0, fa4, rtz
; RV32-NEXT: fld fa4, 56(sp)
; RV32-NEXT: vslide1down.vx v8, v8, a2
; RV32-NEXT: vslide1down.vx v8, v8, a3
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: vmv.v.x v9, a3
; RV32-NEXT: vslide1down.vx v9, v9, a2
; RV32-NEXT: vslide1down.vx v9, v9, a0
; RV32-NEXT: fmax.d fa4, fa4, fa3
; RV32-NEXT: fmin.d fa5, fa4, fa5
; RV32-NEXT: fcvt.wu.d a0, fa5, rtz
; RV32-NEXT: vslide1down.vx v8, v8, a0
; RV32-NEXT: vse8.v v8, (a1)
; RV32-NEXT: vmv.v.i v0, 15
; RV32-NEXT: vslide1down.vx v9, v9, a0
; RV32-NEXT: vslidedown.vi v9, v8, 4, v0.t
; RV32-NEXT: vse8.v v9, (a1)
; RV32-NEXT: addi sp, s0, -128
; RV32-NEXT: lw ra, 124(sp) # 4-byte Folded Reload
; RV32-NEXT: lw s0, 120(sp) # 4-byte Folded Reload
Expand Down Expand Up @@ -627,11 +633,11 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
; RV64-NEXT: vslidedown.vi v8, v8, 3
; RV64-NEXT: vfmv.f.s fa4, v8
; RV64-NEXT: fmax.d fa4, fa4, fa3
; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV64-NEXT: fld fa2, 32(sp)
; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; RV64-NEXT: fld fa2, 40(sp)
; RV64-NEXT: fmin.d fa4, fa4, fa5
; RV64-NEXT: fcvt.lu.d a0, fa4, rtz
; RV64-NEXT: fld fa4, 40(sp)
; RV64-NEXT: fld fa4, 32(sp)
; RV64-NEXT: fmax.d fa2, fa2, fa3
; RV64-NEXT: fmin.d fa2, fa2, fa5
; RV64-NEXT: fcvt.lu.d a2, fa2, rtz
Expand All @@ -644,14 +650,16 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
; RV64-NEXT: fmin.d fa4, fa4, fa5
; RV64-NEXT: fcvt.lu.d a0, fa4, rtz
; RV64-NEXT: fld fa4, 56(sp)
; RV64-NEXT: vslide1down.vx v8, v8, a2
; RV64-NEXT: vslide1down.vx v8, v8, a3
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: vmv.v.x v9, a3
; RV64-NEXT: vslide1down.vx v9, v9, a2
; RV64-NEXT: vslide1down.vx v9, v9, a0
; RV64-NEXT: fmax.d fa4, fa4, fa3
; RV64-NEXT: fmin.d fa5, fa4, fa5
; RV64-NEXT: fcvt.lu.d a0, fa5, rtz
; RV64-NEXT: vslide1down.vx v8, v8, a0
; RV64-NEXT: vse8.v v8, (a1)
; RV64-NEXT: vmv.v.i v0, 15
; RV64-NEXT: vslide1down.vx v9, v9, a0
; RV64-NEXT: vslidedown.vi v9, v8, 4, v0.t
; RV64-NEXT: vse8.v v9, (a1)
; RV64-NEXT: addi sp, s0, -128
; RV64-NEXT: ld ra, 120(sp) # 8-byte Folded Reload
; RV64-NEXT: ld s0, 112(sp) # 8-byte Folded Reload
Expand Down
Loading