Skip to content

Commit 657abbb

Browse files
tclin914tclin914
committed
[RISCV] Lower BUILD_VECTOR with i64 type to VID on RV32 if possible
1 parent 8d825cb commit 657abbb

File tree

5 files changed

+136
-174
lines changed

5 files changed

+136
-174
lines changed

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 83 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,6 +1281,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
12811281
if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
12821282
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
12831283
setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1284+
1285+
// Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1286+
setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
12841287
}
12851288

12861289
setOperationAction(
@@ -3601,6 +3604,78 @@ static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
36013604
return Gather;
36023605
}
36033606

3607+
static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
3608+
const RISCVSubtarget &Subtarget) {
3609+
MVT VT = Op.getSimpleValueType();
3610+
assert(VT.isFixedLengthVector() && "Unexpected vector!");
3611+
3612+
MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3613+
3614+
SDLoc DL(Op);
3615+
auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3616+
3617+
if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
3618+
int64_t StepNumerator = SimpleVID->StepNumerator;
3619+
unsigned StepDenominator = SimpleVID->StepDenominator;
3620+
int64_t Addend = SimpleVID->Addend;
3621+
3622+
assert(StepNumerator != 0 && "Invalid step");
3623+
bool Negate = false;
3624+
int64_t SplatStepVal = StepNumerator;
3625+
unsigned StepOpcode = ISD::MUL;
3626+
// Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
3627+
// anyway as the shift of 63 won't fit in uimm5.
3628+
if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
3629+
isPowerOf2_64(std::abs(StepNumerator))) {
3630+
Negate = StepNumerator < 0;
3631+
StepOpcode = ISD::SHL;
3632+
SplatStepVal = Log2_64(std::abs(StepNumerator));
3633+
}
3634+
3635+
// Only emit VIDs with suitably-small steps/addends. We use imm5 is a
3636+
// threshold since it's the immediate value many RVV instructions accept.
3637+
// There is no vmul.vi instruction so ensure multiply constant can fit in
3638+
// a single addi instruction.
3639+
if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
3640+
(StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
3641+
isPowerOf2_32(StepDenominator) &&
3642+
(SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3643+
MVT VIDVT =
3644+
VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
3645+
MVT VIDContainerVT =
3646+
getContainerForFixedLengthVector(DAG, VIDVT, Subtarget);
3647+
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3648+
// Convert right out of the scalable type so we can use standard ISD
3649+
// nodes for the rest of the computation. If we used scalable types with
3650+
// these, we'd lose the fixed-length vector info and generate worse
3651+
// vsetvli code.
3652+
VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
3653+
if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
3654+
(StepOpcode == ISD::SHL && SplatStepVal != 0)) {
3655+
SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
3656+
VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
3657+
}
3658+
if (StepDenominator != 1) {
3659+
SDValue SplatStep =
3660+
DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
3661+
VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3662+
}
3663+
if (Addend != 0 || Negate) {
3664+
SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
3665+
VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3666+
VID);
3667+
}
3668+
if (VT.isFloatingPoint()) {
3669+
// TODO: Use vfwcvt to reduce register pressure.
3670+
VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3671+
}
3672+
return VID;
3673+
}
3674+
}
3675+
3676+
return SDValue();
3677+
}
3678+
36043679
/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
36053680
/// which constitute a large proportion of the elements. In such cases we can
36063681
/// splat a vector with the dominant element and make up the shortfall with
@@ -3818,64 +3893,8 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
38183893
// Try and match index sequences, which we can lower to the vid instruction
38193894
// with optional modifications. An all-undef vector is matched by
38203895
// getSplatValue, above.
3821-
if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
3822-
int64_t StepNumerator = SimpleVID->StepNumerator;
3823-
unsigned StepDenominator = SimpleVID->StepDenominator;
3824-
int64_t Addend = SimpleVID->Addend;
3825-
3826-
assert(StepNumerator != 0 && "Invalid step");
3827-
bool Negate = false;
3828-
int64_t SplatStepVal = StepNumerator;
3829-
unsigned StepOpcode = ISD::MUL;
3830-
// Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
3831-
// anyway as the shift of 63 won't fit in uimm5.
3832-
if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
3833-
isPowerOf2_64(std::abs(StepNumerator))) {
3834-
Negate = StepNumerator < 0;
3835-
StepOpcode = ISD::SHL;
3836-
SplatStepVal = Log2_64(std::abs(StepNumerator));
3837-
}
3838-
3839-
// Only emit VIDs with suitably-small steps/addends. We use imm5 is a
3840-
// threshold since it's the immediate value many RVV instructions accept.
3841-
// There is no vmul.vi instruction so ensure multiply constant can fit in
3842-
// a single addi instruction.
3843-
if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
3844-
(StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
3845-
isPowerOf2_32(StepDenominator) &&
3846-
(SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3847-
MVT VIDVT =
3848-
VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
3849-
MVT VIDContainerVT =
3850-
getContainerForFixedLengthVector(DAG, VIDVT, Subtarget);
3851-
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3852-
// Convert right out of the scalable type so we can use standard ISD
3853-
// nodes for the rest of the computation. If we used scalable types with
3854-
// these, we'd lose the fixed-length vector info and generate worse
3855-
// vsetvli code.
3856-
VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
3857-
if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
3858-
(StepOpcode == ISD::SHL && SplatStepVal != 0)) {
3859-
SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
3860-
VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
3861-
}
3862-
if (StepDenominator != 1) {
3863-
SDValue SplatStep =
3864-
DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
3865-
VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3866-
}
3867-
if (Addend != 0 || Negate) {
3868-
SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
3869-
VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3870-
VID);
3871-
}
3872-
if (VT.isFloatingPoint()) {
3873-
// TODO: Use vfwcvt to reduce register pressure.
3874-
VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3875-
}
3876-
return VID;
3877-
}
3878-
}
3896+
if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
3897+
return Res;
38793898

38803899
// For very small build_vectors, use a single scalar insert of a constant.
38813900
// TODO: Base this on constant rematerialization cost, not size.
@@ -7473,8 +7492,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
74737492
return lowerVECTOR_REVERSE(Op, DAG);
74747493
case ISD::VECTOR_SPLICE:
74757494
return lowerVECTOR_SPLICE(Op, DAG);
7476-
case ISD::BUILD_VECTOR:
7495+
case ISD::BUILD_VECTOR: {
7496+
MVT VT = Op.getSimpleValueType();
7497+
MVT EltVT = VT.getVectorElementType();
7498+
if (!Subtarget.is64Bit() && EltVT == MVT::i64)
7499+
return lowerBuildVectorViaVID(Op, DAG, Subtarget);
74777500
return lowerBUILD_VECTOR(Op, DAG, Subtarget);
7501+
}
74787502
case ISD::SPLAT_VECTOR: {
74797503
MVT VT = Op.getSimpleValueType();
74807504
MVT EltVT = VT.getVectorElementType();

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -290,15 +290,11 @@ define void @buildvec_vid_stepn3_addn3_v4i32(ptr %z0, ptr %z1, ptr %z2, ptr %z3)
290290
ret void
291291
}
292292

293-
; FIXME: RV32 doesn't catch this pattern due to BUILD_VECTOR legalization.
294293
define <4 x i64> @buildvec_vid_step1_add0_v4i64() {
295294
; RV32-LABEL: buildvec_vid_step1_add0_v4i64:
296295
; RV32: # %bb.0:
297-
; RV32-NEXT: lui a0, %hi(.LCPI25_0)
298-
; RV32-NEXT: addi a0, a0, %lo(.LCPI25_0)
299-
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
300-
; RV32-NEXT: vle8.v v10, (a0)
301-
; RV32-NEXT: vsext.vf4 v8, v10
296+
; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
297+
; RV32-NEXT: vid.v v8
302298
; RV32-NEXT: ret
303299
;
304300
; RV64V-LABEL: buildvec_vid_step1_add0_v4i64:
@@ -323,11 +319,9 @@ define <4 x i64> @buildvec_vid_step1_add0_v4i64() {
323319
define <4 x i64> @buildvec_vid_step2_add0_v4i64() {
324320
; RV32-LABEL: buildvec_vid_step2_add0_v4i64:
325321
; RV32: # %bb.0:
326-
; RV32-NEXT: lui a0, %hi(.LCPI26_0)
327-
; RV32-NEXT: addi a0, a0, %lo(.LCPI26_0)
328-
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
329-
; RV32-NEXT: vle8.v v10, (a0)
330-
; RV32-NEXT: vsext.vf4 v8, v10
322+
; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma
323+
; RV32-NEXT: vid.v v8
324+
; RV32-NEXT: vadd.vv v8, v8, v8
331325
; RV32-NEXT: ret
332326
;
333327
; RV64V-LABEL: buildvec_vid_step2_add0_v4i64:

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,15 +1193,11 @@ define void @mulhu_v2i64(ptr %x) {
11931193
; RV32-NEXT: addi a1, a1, %lo(.LCPI69_0)
11941194
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
11951195
; RV32-NEXT: vle32.v v9, (a1)
1196-
; RV32-NEXT: lui a1, 32
1197-
; RV32-NEXT: addi a1, a1, 1
11981196
; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
11991197
; RV32-NEXT: vmulhu.vv v8, v8, v9
1200-
; RV32-NEXT: vmv.s.x v9, a1
1201-
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
1202-
; RV32-NEXT: vsext.vf4 v10, v9
1203-
; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
1204-
; RV32-NEXT: vsrl.vv v8, v8, v10
1198+
; RV32-NEXT: vid.v v9
1199+
; RV32-NEXT: vadd.vi v9, v9, 1
1200+
; RV32-NEXT: vsrl.vv v8, v8, v9
12051201
; RV32-NEXT: vse64.v v8, (a0)
12061202
; RV32-NEXT: ret
12071203
;
@@ -1348,27 +1344,21 @@ define void @mulhs_v2i64(ptr %x) {
13481344
; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
13491345
; RV32-NEXT: vle64.v v8, (a0)
13501346
; RV32-NEXT: lui a1, 349525
1351-
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
13521347
; RV32-NEXT: vid.v v9
13531348
; RV32-NEXT: addi a2, a1, 1365
1349+
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
13541350
; RV32-NEXT: vmv.v.x v10, a2
13551351
; RV32-NEXT: li a2, 63
13561352
; RV32-NEXT: addi a1, a1, 1366
1357-
; RV32-NEXT: vsetvli zero, zero, e32, m1, tu, ma
1358-
; RV32-NEXT: vmv.s.x v10, a1
1359-
; RV32-NEXT: lui a1, 16
1360-
; RV32-NEXT: vsetvli zero, zero, e32, m1, ta, ma
1361-
; RV32-NEXT: vsrl.vi v9, v9, 1
1362-
; RV32-NEXT: vrsub.vi v9, v9, 0
13631353
; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
1354+
; RV32-NEXT: vrsub.vi v11, v9, 0
1355+
; RV32-NEXT: vsetvli zero, zero, e32, mf2, tu, ma
1356+
; RV32-NEXT: vmv.s.x v10, a1
1357+
; RV32-NEXT: vsetvli zero, zero, e64, m1, ta, ma
13641358
; RV32-NEXT: vmulh.vv v10, v8, v10
1365-
; RV32-NEXT: vmadd.vv v9, v8, v10
1366-
; RV32-NEXT: vmv.s.x v8, a1
1367-
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
1368-
; RV32-NEXT: vsext.vf4 v10, v8
1369-
; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
1370-
; RV32-NEXT: vsrl.vx v8, v9, a2
1371-
; RV32-NEXT: vsra.vv v9, v9, v10
1359+
; RV32-NEXT: vmadd.vv v11, v8, v10
1360+
; RV32-NEXT: vsrl.vx v8, v11, a2
1361+
; RV32-NEXT: vsra.vv v9, v11, v9
13721362
; RV32-NEXT: vadd.vv v8, v9, v8
13731363
; RV32-NEXT: vse64.v v8, (a0)
13741364
; RV32-NEXT: ret

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-stepvector.ll

Lines changed: 22 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2-
; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
3-
; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
2+
; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s
3+
; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s
44

55
declare <2 x i8> @llvm.stepvector.v2i8()
66

@@ -161,83 +161,47 @@ define <16 x i32> @stepvector_v16i32() {
161161
declare <2 x i64> @llvm.stepvector.v2i64()
162162

163163
define <2 x i64> @stepvector_v2i64() {
164-
; RV32-LABEL: stepvector_v2i64:
165-
; RV32: # %bb.0:
166-
; RV32-NEXT: lui a0, 16
167-
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
168-
; RV32-NEXT: vmv.s.x v9, a0
169-
; RV32-NEXT: vsext.vf4 v8, v9
170-
; RV32-NEXT: ret
171-
;
172-
; RV64-LABEL: stepvector_v2i64:
173-
; RV64: # %bb.0:
174-
; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
175-
; RV64-NEXT: vid.v v8
176-
; RV64-NEXT: ret
164+
; CHECK-LABEL: stepvector_v2i64:
165+
; CHECK: # %bb.0:
166+
; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
167+
; CHECK-NEXT: vid.v v8
168+
; CHECK-NEXT: ret
177169
%v = call <2 x i64> @llvm.stepvector.v2i64()
178170
ret <2 x i64> %v
179171
}
180172

181173
declare <4 x i64> @llvm.stepvector.v4i64()
182174

183175
define <4 x i64> @stepvector_v4i64() {
184-
; RV32-LABEL: stepvector_v4i64:
185-
; RV32: # %bb.0:
186-
; RV32-NEXT: lui a0, %hi(.LCPI14_0)
187-
; RV32-NEXT: addi a0, a0, %lo(.LCPI14_0)
188-
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
189-
; RV32-NEXT: vle8.v v10, (a0)
190-
; RV32-NEXT: vsext.vf4 v8, v10
191-
; RV32-NEXT: ret
192-
;
193-
; RV64-LABEL: stepvector_v4i64:
194-
; RV64: # %bb.0:
195-
; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma
196-
; RV64-NEXT: vid.v v8
197-
; RV64-NEXT: ret
176+
; CHECK-LABEL: stepvector_v4i64:
177+
; CHECK: # %bb.0:
178+
; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
179+
; CHECK-NEXT: vid.v v8
180+
; CHECK-NEXT: ret
198181
%v = call <4 x i64> @llvm.stepvector.v4i64()
199182
ret <4 x i64> %v
200183
}
201184

202185
declare <8 x i64> @llvm.stepvector.v8i64()
203186

204187
define <8 x i64> @stepvector_v8i64() {
205-
; RV32-LABEL: stepvector_v8i64:
206-
; RV32: # %bb.0:
207-
; RV32-NEXT: lui a0, %hi(.LCPI15_0)
208-
; RV32-NEXT: addi a0, a0, %lo(.LCPI15_0)
209-
; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
210-
; RV32-NEXT: vle8.v v12, (a0)
211-
; RV32-NEXT: vsext.vf4 v8, v12
212-
; RV32-NEXT: ret
213-
;
214-
; RV64-LABEL: stepvector_v8i64:
215-
; RV64: # %bb.0:
216-
; RV64-NEXT: vsetivli zero, 8, e64, m4, ta, ma
217-
; RV64-NEXT: vid.v v8
218-
; RV64-NEXT: ret
188+
; CHECK-LABEL: stepvector_v8i64:
189+
; CHECK: # %bb.0:
190+
; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
191+
; CHECK-NEXT: vid.v v8
192+
; CHECK-NEXT: ret
219193
%v = call <8 x i64> @llvm.stepvector.v8i64()
220194
ret <8 x i64> %v
221195
}
222196

223197
declare <16 x i64> @llvm.stepvector.v16i64()
224198

225199
define <16 x i64> @stepvector_v16i64() {
226-
; RV32-LABEL: stepvector_v16i64:
227-
; RV32: # %bb.0:
228-
; RV32-NEXT: li a0, 32
229-
; RV32-NEXT: lui a1, %hi(.LCPI16_0)
230-
; RV32-NEXT: addi a1, a1, %lo(.LCPI16_0)
231-
; RV32-NEXT: vsetvli zero, a0, e32, m8, ta, ma
232-
; RV32-NEXT: vle8.v v16, (a1)
233-
; RV32-NEXT: vsext.vf4 v8, v16
234-
; RV32-NEXT: ret
235-
;
236-
; RV64-LABEL: stepvector_v16i64:
237-
; RV64: # %bb.0:
238-
; RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma
239-
; RV64-NEXT: vid.v v8
240-
; RV64-NEXT: ret
200+
; CHECK-LABEL: stepvector_v16i64:
201+
; CHECK: # %bb.0:
202+
; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma
203+
; CHECK-NEXT: vid.v v8
204+
; CHECK-NEXT: ret
241205
%v = call <16 x i64> @llvm.stepvector.v16i64()
242206
ret <16 x i64> %v
243207
}

0 commit comments

Comments
 (0)