Skip to content

Commit 99a0cd6

Browse files
topperctopperc
authored
[RISCV] Use TU policy for C reduction intrinsics. (#93970)
The C intrinsics should allow a value to be specified for the upper elements. This used to work before https://reviews.llvm.org/D146752 which should have only changed the behavior for the autovectorizer. It clearly changed all reductions.
1 parent 96d19fd commit 99a0cd6

18 files changed

+605
-607
lines changed

‎llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4442,7 +4442,7 @@ class VPatTernaryNoMask<string intrinsic,
44424442
op2_kind:$rs2,
44434443
GPR:$vl, sew)>;
44444444

4445-
class VPatTernaryNoMaskTA<string intrinsic,
4445+
class VPatTernaryNoMaskTU<string intrinsic,
44464446
string inst,
44474447
string kind,
44484448
ValueType result_type,
@@ -4462,19 +4462,19 @@ class VPatTernaryNoMaskTA<string intrinsic,
44624462
result_reg_class:$rs3,
44634463
(op1_type op1_reg_class:$rs1),
44644464
op2_kind:$rs2,
4465-
GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
4466-
4467-
class VPatTernaryNoMaskTARoundingMode<string intrinsic,
4468-
string inst,
4469-
string kind,
4470-
ValueType result_type,
4471-
ValueType op1_type,
4472-
ValueType op2_type,
4473-
int log2sew,
4474-
LMULInfo vlmul,
4475-
VReg result_reg_class,
4476-
RegisterClass op1_reg_class,
4477-
DAGOperand op2_kind> :
4465+
GPR:$vl, log2sew, TU_MU)>;
4466+
4467+
class VPatTernaryNoMaskTURoundingMode<string intrinsic,
4468+
string inst,
4469+
string kind,
4470+
ValueType result_type,
4471+
ValueType op1_type,
4472+
ValueType op2_type,
4473+
int log2sew,
4474+
LMULInfo vlmul,
4475+
VReg result_reg_class,
4476+
RegisterClass op1_reg_class,
4477+
DAGOperand op2_kind> :
44784478
Pat<(result_type (!cast<Intrinsic>(intrinsic)
44794479
(result_type result_reg_class:$rs3),
44804480
(op1_type op1_reg_class:$rs1),
@@ -4486,7 +4486,7 @@ class VPatTernaryNoMaskTARoundingMode<string intrinsic,
44864486
(op1_type op1_reg_class:$rs1),
44874487
op2_kind:$rs2,
44884488
(XLenVT timm:$round),
4489-
GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
4489+
GPR:$vl, log2sew, TU_MU)>;
44904490

44914491
class VPatTernaryNoMaskWithPolicy<string intrinsic,
44924492
string inst,
@@ -4617,7 +4617,7 @@ class VPatTernaryMaskPolicyRoundingMode<string intrinsic,
46174617
(XLenVT timm:$round),
46184618
GPR:$vl, log2sew, (XLenVT timm:$policy))>;
46194619

4620-
class VPatTernaryMaskTA<string intrinsic,
4620+
class VPatTernaryMaskTU<string intrinsic,
46214621
string inst,
46224622
string kind,
46234623
ValueType result_type,
@@ -4640,9 +4640,9 @@ class VPatTernaryMaskTA<string intrinsic,
46404640
(op1_type op1_reg_class:$rs1),
46414641
op2_kind:$rs2,
46424642
(mask_type V0),
4643-
GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
4643+
GPR:$vl, log2sew, TU_MU)>;
46444644

4645-
class VPatTernaryMaskTARoundingMode<string intrinsic,
4645+
class VPatTernaryMaskTURoundingMode<string intrinsic,
46464646
string inst,
46474647
string kind,
46484648
ValueType result_type,
@@ -4667,7 +4667,7 @@ class VPatTernaryMaskTARoundingMode<string intrinsic,
46674667
op2_kind:$rs2,
46684668
(mask_type V0),
46694669
(XLenVT timm:$round),
4670-
GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
4670+
GPR:$vl, log2sew, TU_MU)>;
46714671

46724672
multiclass VPatUnaryS_M<string intrinsic_name,
46734673
string inst> {
@@ -5643,7 +5643,7 @@ multiclass VPatTernaryWithPolicyRoundingMode<string intrinsic,
56435643
op2_kind, isSEWAware>;
56445644
}
56455645

5646-
multiclass VPatTernaryTA<string intrinsic,
5646+
multiclass VPatTernaryTU<string intrinsic,
56475647
string inst,
56485648
string kind,
56495649
ValueType result_type,
@@ -5655,15 +5655,15 @@ multiclass VPatTernaryTA<string intrinsic,
56555655
VReg result_reg_class,
56565656
RegisterClass op1_reg_class,
56575657
DAGOperand op2_kind> {
5658-
def : VPatTernaryNoMaskTA<intrinsic, inst, kind, result_type, op1_type,
5658+
def : VPatTernaryNoMaskTU<intrinsic, inst, kind, result_type, op1_type,
56595659
op2_type, log2sew, vlmul, result_reg_class,
56605660
op1_reg_class, op2_kind>;
5661-
def : VPatTernaryMaskTA<intrinsic, inst, kind, result_type, op1_type,
5661+
def : VPatTernaryMaskTU<intrinsic, inst, kind, result_type, op1_type,
56625662
op2_type, mask_type, log2sew, vlmul,
56635663
result_reg_class, op1_reg_class, op2_kind>;
56645664
}
56655665

5666-
multiclass VPatTernaryTARoundingMode<string intrinsic,
5666+
multiclass VPatTernaryTURoundingMode<string intrinsic,
56675667
string inst,
56685668
string kind,
56695669
ValueType result_type,
@@ -5675,10 +5675,10 @@ multiclass VPatTernaryTARoundingMode<string intrinsic,
56755675
VReg result_reg_class,
56765676
RegisterClass op1_reg_class,
56775677
DAGOperand op2_kind> {
5678-
def : VPatTernaryNoMaskTARoundingMode<intrinsic, inst, kind, result_type, op1_type,
5678+
def : VPatTernaryNoMaskTURoundingMode<intrinsic, inst, kind, result_type, op1_type,
56795679
op2_type, log2sew, vlmul, result_reg_class,
56805680
op1_reg_class, op2_kind>;
5681-
def : VPatTernaryMaskTARoundingMode<intrinsic, inst, kind, result_type, op1_type,
5681+
def : VPatTernaryMaskTURoundingMode<intrinsic, inst, kind, result_type, op1_type,
56825682
op2_type, mask_type, log2sew, vlmul,
56835683
result_reg_class, op1_reg_class, op2_kind>;
56845684
}
@@ -5856,15 +5856,15 @@ multiclass VPatReductionV_VS<string intrinsic, string instruction, bit IsFloat =
58565856
foreach vti = !if(IsFloat, NoGroupFloatVectors, NoGroupIntegerVectors) in {
58575857
defvar vectorM1 = !cast<VTypeInfo>(!if(IsFloat, "VF", "VI") # vti.SEW # "M1");
58585858
let Predicates = GetVTypePredicates<vti>.Predicates in
5859-
defm : VPatTernaryTA<intrinsic, instruction, "VS",
5859+
defm : VPatTernaryTU<intrinsic, instruction, "VS",
58605860
vectorM1.Vector, vti.Vector,
58615861
vectorM1.Vector, vti.Mask,
58625862
vti.Log2SEW, vti.LMul,
58635863
VR, vti.RegClass, VR>;
58645864
}
58655865
foreach gvti = !if(IsFloat, GroupFloatVectors, GroupIntegerVectors) in {
58665866
let Predicates = GetVTypePredicates<gvti>.Predicates in
5867-
defm : VPatTernaryTA<intrinsic, instruction, "VS",
5867+
defm : VPatTernaryTU<intrinsic, instruction, "VS",
58685868
gvti.VectorM1, gvti.Vector,
58695869
gvti.VectorM1, gvti.Mask,
58705870
gvti.Log2SEW, gvti.LMul,
@@ -5876,15 +5876,15 @@ multiclass VPatReductionV_VS_RM<string intrinsic, string instruction, bit IsFloa
58765876
foreach vti = !if(IsFloat, NoGroupFloatVectors, NoGroupIntegerVectors) in {
58775877
defvar vectorM1 = !cast<VTypeInfo>(!if(IsFloat, "VF", "VI") # vti.SEW # "M1");
58785878
let Predicates = GetVTypePredicates<vti>.Predicates in
5879-
defm : VPatTernaryTARoundingMode<intrinsic, instruction, "VS",
5879+
defm : VPatTernaryTURoundingMode<intrinsic, instruction, "VS",
58805880
vectorM1.Vector, vti.Vector,
58815881
vectorM1.Vector, vti.Mask,
58825882
vti.Log2SEW, vti.LMul,
58835883
VR, vti.RegClass, VR>;
58845884
}
58855885
foreach gvti = !if(IsFloat, GroupFloatVectors, GroupIntegerVectors) in {
58865886
let Predicates = GetVTypePredicates<gvti>.Predicates in
5887-
defm : VPatTernaryTARoundingMode<intrinsic, instruction, "VS",
5887+
defm : VPatTernaryTURoundingMode<intrinsic, instruction, "VS",
58885888
gvti.VectorM1, gvti.Vector,
58895889
gvti.VectorM1, gvti.Mask,
58905890
gvti.Log2SEW, gvti.LMul,
@@ -5898,7 +5898,7 @@ multiclass VPatReductionW_VS<string intrinsic, string instruction, bit IsFloat =
58985898
if !le(wtiSEW, 64) then {
58995899
defvar wtiM1 = !cast<VTypeInfo>(!if(IsFloat, "VF", "VI") # wtiSEW # "M1");
59005900
let Predicates = GetVTypePredicates<vti>.Predicates in
5901-
defm : VPatTernaryTA<intrinsic, instruction, "VS",
5901+
defm : VPatTernaryTU<intrinsic, instruction, "VS",
59025902
wtiM1.Vector, vti.Vector,
59035903
wtiM1.Vector, vti.Mask,
59045904
vti.Log2SEW, vti.LMul,
@@ -5914,7 +5914,7 @@ multiclass VPatReductionW_VS_RM<string intrinsic, string instruction, bit IsFloa
59145914
if !le(wtiSEW, 64) then {
59155915
defvar wtiM1 = !cast<VTypeInfo>(!if(IsFloat, "VF", "VI") # wtiSEW # "M1");
59165916
let Predicates = GetVTypePredicates<vti>.Predicates in
5917-
defm : VPatTernaryTARoundingMode<intrinsic, instruction, "VS",
5917+
defm : VPatTernaryTURoundingMode<intrinsic, instruction, "VS",
59185918
wtiM1.Vector, vti.Vector,
59195919
wtiM1.Vector, vti.Mask,
59205920
vti.Log2SEW, vti.LMul,

‎llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -943,9 +943,8 @@ define <vscale x 2 x i32> @vredsum(<vscale x 2 x i32> %passthru, <vscale x 2 x i
943943
; CHECK-LABEL: vredsum:
944944
; CHECK: # %bb.0:
945945
; CHECK-NEXT: vmv1r.v v11, v8
946-
; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
946+
; CHECK-NEXT: vsetvli zero, a0, e32, m1, tu, ma
947947
; CHECK-NEXT: vredsum.vs v11, v9, v10
948-
; CHECK-NEXT: vsetvli zero, zero, e32, m1, tu, ma
949948
; CHECK-NEXT: vmerge.vvm v8, v8, v11, v0
950949
; CHECK-NEXT: ret
951950
%a = call <vscale x 2 x i32> @llvm.riscv.vredsum.nxv2i32.nxv2i32(
@@ -968,9 +967,8 @@ define <vscale x 2 x float> @vfredusum(<vscale x 2 x float> %passthru, <vscale x
968967
; CHECK: # %bb.0:
969968
; CHECK-NEXT: fsrmi a1, 0
970969
; CHECK-NEXT: vmv1r.v v11, v8
971-
; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
970+
; CHECK-NEXT: vsetvli zero, a0, e32, m1, tu, ma
972971
; CHECK-NEXT: vfredusum.vs v11, v9, v10
973-
; CHECK-NEXT: vsetvli zero, zero, e32, m1, tu, ma
974972
; CHECK-NEXT: vmerge.vvm v8, v8, v11, v0
975973
; CHECK-NEXT: fsrm a1
976974
; CHECK-NEXT: ret

0 commit comments

Comments
 (0)