Skip to content

Commit febeaf9

Browse files
author
Francesco Petrogalli
committed
[llvm][SVE] IR intrinsic for LD1RO.
Reviewers: sdesmalen, efriedma Subscribers: tschuett, hiraditya, rkruppe, psnobl, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D80738
1 parent 007098d commit febeaf9

File tree

6 files changed

+110
-11
lines changed

6 files changed

+110
-11
lines changed

‎llvm/include/llvm/IR/IntrinsicsAArch64.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1311,6 +1311,7 @@ def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
13111311
def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
13121312

13131313
def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1314+
def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
13141315

13151316
//
13161317
// Stores

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1463,6 +1463,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
14631463
case AArch64ISD::LDFF1: return "AArch64ISD::LDFF1";
14641464
case AArch64ISD::LDFF1S: return "AArch64ISD::LDFF1S";
14651465
case AArch64ISD::LD1RQ: return "AArch64ISD::LD1RQ";
1466+
case AArch64ISD::LD1RO: return "AArch64ISD::LD1RO";
14661467
case AArch64ISD::GLD1: return "AArch64ISD::GLD1";
14671468
case AArch64ISD::GLD1_SCALED: return "AArch64ISD::GLD1_SCALED";
14681469
case AArch64ISD::GLD1_SXTW: return "AArch64ISD::GLD1_SXTW";
@@ -11885,7 +11886,10 @@ static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
1188511886
return L;
1188611887
}
1188711888

11888-
static SDValue performLD1RQCombine(SDNode *N, SelectionDAG &DAG) {
11889+
template <unsigned Opcode>
11890+
static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
11891+
static_assert(Opcode == AArch64ISD::LD1RQ || Opcode == AArch64ISD::LD1RO,
11892+
"Unsupported opcode.");
1188911893
SDLoc DL(N);
1189011894
EVT VT = N->getValueType(0);
1189111895

@@ -11894,13 +11898,13 @@ static SDValue performLD1RQCombine(SDNode *N, SelectionDAG &DAG) {
1189411898
LoadVT = VT.changeTypeToInteger();
1189511899

1189611900
SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
11897-
SDValue Load = DAG.getNode(AArch64ISD::LD1RQ, DL, {LoadVT, MVT::Other}, Ops);
11901+
SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
1189811902
SDValue LoadChain = SDValue(Load.getNode(), 1);
1189911903

1190011904
if (VT.isFloatingPoint())
1190111905
Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
1190211906

11903-
return DAG.getMergeValues({ Load, LoadChain }, DL);
11907+
return DAG.getMergeValues({Load, LoadChain}, DL);
1190411908
}
1190511909

1190611910
static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
@@ -13493,7 +13497,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
1349313497
case Intrinsic::aarch64_sve_ldnt1:
1349413498
return performLDNT1Combine(N, DAG);
1349513499
case Intrinsic::aarch64_sve_ld1rq:
13496-
return performLD1RQCombine(N, DAG);
13500+
return performLD1ReplicateCombine<AArch64ISD::LD1RQ>(N, DAG);
13501+
case Intrinsic::aarch64_sve_ld1ro:
13502+
return performLD1ReplicateCombine<AArch64ISD::LD1RO>(N, DAG);
1349713503
case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
1349813504
return performGatherLoadCombine(N, DAG, AArch64ISD::GLDNT1);
1349913505
case Intrinsic::aarch64_sve_ldnt1_gather:

‎llvm/lib/Target/AArch64/AArch64ISelLowering.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ enum NodeType : unsigned {
253253
LDFF1,
254254
LDFF1S,
255255
LD1RQ,
256+
LD1RO,
256257

257258
// Unsigned gather loads.
258259
GLD1,

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,13 @@ def AArch64ldff1s : SDNode<"AArch64ISD::LDFF1S", SDT_AArch64_LD1, [SDNPHasChain,
3131
// Contiguous load and replicate - node definitions
3232
//
3333

34-
def SDT_AArch64_LD1RQ : SDTypeProfile<1, 2, [
34+
def SDT_AArch64_LD1Replicate : SDTypeProfile<1, 2, [
3535
SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>,
3636
SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>
3737
]>;
3838

39-
def AArch64ld1rq : SDNode<"AArch64ISD::LD1RQ", SDT_AArch64_LD1RQ, [SDNPHasChain, SDNPMayLoad]>;
39+
def AArch64ld1rq : SDNode<"AArch64ISD::LD1RQ", SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
40+
def AArch64ld1ro : SDNode<"AArch64ISD::LD1RO", SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
4041

4142
// Gather loads - node definitions
4243
//
@@ -1434,6 +1435,7 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
14341435

14351436
def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>;
14361437
def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>;
1438+
def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
14371439
def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
14381440
def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>;
14391441
def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
@@ -1954,10 +1956,10 @@ let Predicates = [HasSVE, HasMatMulFP32] in {
19541956

19551957
let Predicates = [HasSVE, HasMatMulFP64] in {
19561958
defm FMMLA_ZZZ_D : sve_fp_matrix_mla<1, "fmmla", ZPR64, int_aarch64_sve_fmmla, nxv2f64>;
1957-
defm LD1RO_B_IMM : sve_mem_ldor_si<0b00, "ld1rob", Z_b, ZPR8>;
1958-
defm LD1RO_H_IMM : sve_mem_ldor_si<0b01, "ld1roh", Z_h, ZPR16>;
1959-
defm LD1RO_W_IMM : sve_mem_ldor_si<0b10, "ld1row", Z_s, ZPR32>;
1960-
defm LD1RO_D_IMM : sve_mem_ldor_si<0b11, "ld1rod", Z_d, ZPR64>;
1959+
defm LD1RO_B_IMM : sve_mem_ldor_si<0b00, "ld1rob", Z_b, ZPR8, nxv16i8, nxv16i1, AArch64ld1ro>;
1960+
defm LD1RO_H_IMM : sve_mem_ldor_si<0b01, "ld1roh", Z_h, ZPR16, nxv8i16, nxv8i1, AArch64ld1ro>;
1961+
defm LD1RO_W_IMM : sve_mem_ldor_si<0b10, "ld1row", Z_s, ZPR32, nxv4i32, nxv4i1, AArch64ld1ro>;
1962+
defm LD1RO_D_IMM : sve_mem_ldor_si<0b11, "ld1rod", Z_d, ZPR64, nxv2i64, nxv2i1, AArch64ld1ro>;
19611963
defm LD1RO_B : sve_mem_ldor_ss<0b00, "ld1rob", Z_b, ZPR8, GPR64NoXZRshifted8>;
19621964
defm LD1RO_H : sve_mem_ldor_ss<0b01, "ld1roh", Z_h, ZPR16, GPR64NoXZRshifted16>;
19631965
defm LD1RO_W : sve_mem_ldor_ss<0b10, "ld1row", Z_s, ZPR32, GPR64NoXZRshifted32>;

‎llvm/lib/Target/AArch64/SVEInstrFormats.td

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7663,14 +7663,19 @@ class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList>
76637663
}
76647664

76657665
multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty,
7666-
ZPRRegOp zprty> {
7666+
ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> {
76677667
def NAME : sve_mem_ldor_si<sz, asm, listty>;
76687668
def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
76697669
(!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
76707670
def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
76717671
(!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
76727672
def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]",
76737673
(!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>;
7674+
7675+
// Base addressing mode
7676+
def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), GPR64sp:$base)),
7677+
(!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, (i64 0))>;
7678+
76747679
}
76757680

76767681
class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList,

‎llvm/test/CodeGen/AArch64/sve-intrinsics-ld1ro.ll

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+f64mm -asm-verbose=0 < %s | FileCheck %s
2+
3+
;
4+
; LD1ROB
5+
;
6+
7+
define <vscale x 16 x i8> @ld1rob_i8(<vscale x 16 x i1> %pred, i8* %addr) nounwind {
8+
; CHECK-LABEL: ld1rob_i8:
9+
; CHECK-NEXT: ld1rob { z0.b }, p0/z, [x0]
10+
; CHECK-NEXT: ret
11+
%res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> %pred, i8* %addr)
12+
ret <vscale x 16 x i8> %res
13+
}
14+
15+
;
16+
; LD1ROH
17+
;
18+
19+
define <vscale x 8 x i16> @ld1roh_i16(<vscale x 8 x i1> %pred, i16* %addr) nounwind {
20+
; CHECK-LABEL: ld1roh_i16:
21+
; CHECK-NEXT: ld1roh { z0.h }, p0/z, [x0]
22+
; CHECK-NEXT: ret
23+
%res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1ro.nxv8i16(<vscale x 8 x i1> %pred, i16* %addr)
24+
ret <vscale x 8 x i16> %res
25+
}
26+
27+
define <vscale x 8 x half> @ld1roh_half(<vscale x 8 x i1> %pred, half* %addr) nounwind {
28+
; CHECK-LABEL: ld1roh_half:
29+
; CHECK-NEXT: ld1roh { z0.h }, p0/z, [x0]
30+
; CHECK-NEXT: ret
31+
%res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1ro.nxv8f16(<vscale x 8 x i1> %pred, half* %addr)
32+
ret <vscale x 8 x half> %res
33+
}
34+
35+
;
36+
; LD1ROW
37+
;
38+
39+
define <vscale x 4 x i32> @ld1row_i32(<vscale x 4 x i1> %pred, i32* %addr) nounwind {
40+
; CHECK-LABEL: ld1row_i32:
41+
; CHECK-NEXT: ld1row { z0.s }, p0/z, [x0]
42+
; CHECK-NEXT: ret
43+
%res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1ro.nxv4i32(<vscale x 4 x i1> %pred, i32* %addr)
44+
ret <vscale x 4 x i32> %res
45+
}
46+
47+
define <vscale x 4 x float> @ld1row_float(<vscale x 4 x i1> %pred, float* %addr) nounwind {
48+
; CHECK-LABEL: ld1row_float:
49+
; CHECK-NEXT: ld1row { z0.s }, p0/z, [x0]
50+
; CHECK-NEXT: ret
51+
%res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1ro.nxv4f32(<vscale x 4 x i1> %pred, float* %addr)
52+
ret <vscale x 4 x float> %res
53+
}
54+
55+
;
56+
; LD1ROD
57+
;
58+
59+
define <vscale x 2 x i64> @ld1rod_i64(<vscale x 2 x i1> %pred, i64* %addr) nounwind {
60+
; CHECK-LABEL: ld1rod_i64:
61+
; CHECK-NEXT: ld1rod { z0.d }, p0/z, [x0]
62+
; CHECK-NEXT: ret
63+
%res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1ro.nxv2i64(<vscale x 2 x i1> %pred, i64* %addr)
64+
ret <vscale x 2 x i64> %res
65+
}
66+
67+
define <vscale x 2 x double> @ld1rod_double(<vscale x 2 x i1> %pred, double* %addr) nounwind {
68+
; CHECK-LABEL: ld1rod_double:
69+
; CHECK-NEXT: ld1rod { z0.d }, p0/z, [x0]
70+
; CHECK-NEXT: ret
71+
%res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1ro.nxv2f64(<vscale x 2 x i1> %pred, double* %addr)
72+
ret <vscale x 2 x double> %res
73+
}
74+
75+
declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1>, i8*)
76+
77+
declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1ro.nxv8i16(<vscale x 8 x i1>, i16*)
78+
declare <vscale x 8 x half> @llvm.aarch64.sve.ld1ro.nxv8f16(<vscale x 8 x i1>, half*)
79+
80+
declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1ro.nxv4i32(<vscale x 4 x i1>, i32*)
81+
declare <vscale x 4 x float> @llvm.aarch64.sve.ld1ro.nxv4f32(<vscale x 4 x i1>, float*)
82+
83+
declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1ro.nxv2i64(<vscale x 2 x i1>, i64*)
84+
declare <vscale x 2 x double> @llvm.aarch64.sve.ld1ro.nxv2f64(<vscale x 2 x i1>, double*)

0 commit comments

Comments
 (0)