Skip to content

Commit 2593e80

Browse files
tangaactangaac
committed
lower vector shuffle to shift
1 parent a6d3662 commit 2593e80

File tree

6 files changed

+263
-333
lines changed

6 files changed

+263
-333
lines changed

‎llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

Lines changed: 129 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,121 @@ SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
519519
}
520520
}
521521

522+
/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
523+
/// instruction.
524+
// The funciton matches elements form one of the input vector shuffled to the
525+
// left or right with zeroable elements 'shifted in'. It handles both the
526+
// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
527+
// lane.
528+
// This is mainly copy from X86.
529+
static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
530+
unsigned ScalarSizeInBits, ArrayRef<int> Mask,
531+
int MaskOffset, const APInt &Zeroable) {
532+
int Size = Mask.size();
533+
unsigned SizeInBits = Size * ScalarSizeInBits;
534+
535+
auto CheckZeros = [&](int Shift, int Scale, bool Left) {
536+
for (int i = 0; i < Size; i += Scale)
537+
for (int j = 0; j < Shift; ++j)
538+
if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
539+
return false;
540+
541+
return true;
542+
};
543+
544+
auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
545+
int Step = 1) {
546+
for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
547+
if (!(Mask[i] == -1 || Mask[i] == Low))
548+
return false;
549+
return true;
550+
};
551+
552+
auto MatchShift = [&](int Shift, int Scale, bool Left) {
553+
for (int i = 0; i != Size; i += Scale) {
554+
unsigned Pos = Left ? i + Shift : i;
555+
unsigned Low = Left ? i : i + Shift;
556+
unsigned Len = Scale - Shift;
557+
if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
558+
return -1;
559+
}
560+
561+
int ShiftEltBits = ScalarSizeInBits * Scale;
562+
bool ByteShift = ShiftEltBits > 64;
563+
Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
564+
: (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
565+
int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
566+
567+
// Normalize the scale for byte shifts to still produce an i64 element
568+
// type.
569+
Scale = ByteShift ? Scale / 2 : Scale;
570+
571+
// We need to round trip through the appropriate type for the shift.
572+
MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
573+
ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
574+
: MVT::getVectorVT(ShiftSVT, Size / Scale);
575+
return (int)ShiftAmt;
576+
};
577+
578+
unsigned MaxWidth = 128;
579+
for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
580+
for (int Shift = 1; Shift != Scale; ++Shift)
581+
for (bool Left : {true, false})
582+
if (CheckZeros(Shift, Scale, Left)) {
583+
int ShiftAmt = MatchShift(Shift, Scale, Left);
584+
if (0 < ShiftAmt)
585+
return ShiftAmt;
586+
}
587+
588+
// no match
589+
return -1;
590+
}
591+
592+
/// Lower VECTOR_SHUFFLE as shift (if possible).
593+
///
594+
/// For example:
595+
/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
596+
/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
597+
/// is lowered to:
598+
/// (VBSLL_V $v0, $v0, 4)
599+
///
600+
/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
601+
/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
602+
/// is lowered to:
603+
/// (VSLLI_D $v0, $v0, 32)
604+
static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef<int> Mask,
605+
MVT VT, SDValue V1, SDValue V2,
606+
SelectionDAG &DAG,
607+
const APInt &Zeroable) {
608+
int Size = Mask.size();
609+
assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
610+
611+
MVT ShiftVT;
612+
SDValue V = V1;
613+
unsigned Opcode;
614+
615+
// Try to match shuffle against V1 shift.
616+
int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
617+
Mask, 0, Zeroable);
618+
619+
// If V1 failed, try to match shuffle against V2 shift.
620+
if (ShiftAmt < 0) {
621+
ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
622+
Mask, Size, Zeroable);
623+
V = V2;
624+
}
625+
626+
if (ShiftAmt < 0)
627+
return SDValue();
628+
629+
assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
630+
"Illegal integer vector type");
631+
V = DAG.getBitcast(ShiftVT, V);
632+
V = DAG.getNode(Opcode, DL, ShiftVT, V,
633+
DAG.getConstant(ShiftAmt, DL, MVT::i64));
634+
return DAG.getBitcast(VT, V);
635+
}
636+
522637
/// Determine whether a range fits a regular pattern of values.
523638
/// This function accounts for the possibility of jumping over the End iterator.
524639
template <typename ValType>
@@ -587,14 +702,12 @@ static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
587702
static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL,
588703
ArrayRef<int> Mask, MVT VT,
589704
SDValue V1, SDValue V2,
590-
SelectionDAG &DAG) {
705+
SelectionDAG &DAG,
706+
const APInt &Zeroable) {
591707
int Bits = VT.getSizeInBits();
592708
int EltBits = VT.getScalarSizeInBits();
593709
int NumElements = VT.getVectorNumElements();
594710

595-
APInt KnownUndef, KnownZero;
596-
computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
597-
APInt Zeroable = KnownUndef | KnownZero;
598711
if (Zeroable.isAllOnes())
599712
return DAG.getConstant(0, DL, VT);
600713

@@ -1056,6 +1169,10 @@ static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
10561169
"Unexpected mask size for shuffle!");
10571170
assert(Mask.size() % 2 == 0 && "Expected even mask size.");
10581171

1172+
APInt KnownUndef, KnownZero;
1173+
computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
1174+
APInt Zeroable = KnownUndef | KnownZero;
1175+
10591176
SDValue Result;
10601177
// TODO: Add more comparison patterns.
10611178
if (V2.isUndef()) {
@@ -1083,12 +1200,14 @@ static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
10831200
return Result;
10841201
if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
10851202
return Result;
1203+
if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
1204+
Zeroable)))
1205+
return Result;
10861206
if ((Result =
1087-
lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG)))
1207+
lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Zeroable)))
10881208
return Result;
10891209
if ((Result = lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG)))
10901210
return Result;
1091-
10921211
return SDValue();
10931212
}
10941213

@@ -4997,6 +5116,10 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
49975116
NODE_NAME_CASE(VANY_NONZERO)
49985117
NODE_NAME_CASE(FRECIPE)
49995118
NODE_NAME_CASE(FRSQRTE)
5119+
NODE_NAME_CASE(VSLLI)
5120+
NODE_NAME_CASE(VSRLI)
5121+
NODE_NAME_CASE(VBSLL)
5122+
NODE_NAME_CASE(VBSRL)
50005123
}
50015124
#undef NODE_NAME_CASE
50025125
return nullptr;

‎llvm/lib/Target/LoongArch/LoongArchISelLowering.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,15 @@ enum NodeType : unsigned {
147147

148148
// Floating point approximate reciprocal operation
149149
FRECIPE,
150-
FRSQRTE
150+
FRSQRTE,
151+
152+
// Vector logicial left / right shift by immediate
153+
VSLLI,
154+
VSRLI,
155+
156+
// Vector byte logicial left / right shift
157+
VBSLL,
158+
VBSRL
151159

152160
// Intrinsic operations end =============================================
153161
};

‎llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ def loongarch_vreplgr2vr: SDNode<"LoongArchISD::VREPLGR2VR", SDT_LoongArchVreplg
5858
def loongarch_vfrecipe: SDNode<"LoongArchISD::FRECIPE", SDT_LoongArchVFRECIPE>;
5959
def loongarch_vfrsqrte: SDNode<"LoongArchISD::FRSQRTE", SDT_LoongArchVFRSQRTE>;
6060

61+
def loongarch_vslli : SDNode<"LoongArchISD::VSLLI", SDT_LoongArchV1RUimm>;
62+
def loongarch_vsrli : SDNode<"LoongArchISD::VSRLI", SDT_LoongArchV1RUimm>;
63+
64+
def loongarch_vbsll : SDNode<"LoongArchISD::VBSLL", SDT_LoongArchV1RUimm>;
65+
def loongarch_vbsrl : SDNode<"LoongArchISD::VBSRL", SDT_LoongArchV1RUimm>;
66+
6167
def immZExt1 : ImmLeaf<i64, [{return isUInt<1>(Imm);}]>;
6268
def immZExt2 : ImmLeaf<i64, [{return isUInt<2>(Imm);}]>;
6369
def immZExt3 : ImmLeaf<i64, [{return isUInt<3>(Imm);}]>;
@@ -1494,15 +1500,59 @@ def : Pat<(or (v16i8 LSX128:$vj), (v16i8 (SplatPat_uimm8 uimm8:$imm))),
14941500
def : Pat<(xor (v16i8 LSX128:$vj), (v16i8 (SplatPat_uimm8 uimm8:$imm))),
14951501
(VXORI_B LSX128:$vj, uimm8:$imm)>;
14961502

1503+
// VBSLL_V
1504+
def : Pat<(loongarch_vbsll v16i8:$vj, uimm5:$imm), (VBSLL_V v16i8:$vj,
1505+
uimm5:$imm)>;
1506+
def : Pat<(loongarch_vbsll v8i16:$vj, uimm5:$imm), (VBSLL_V v8i16:$vj,
1507+
uimm5:$imm)>;
1508+
def : Pat<(loongarch_vbsll v4i32:$vj, uimm5:$imm), (VBSLL_V v4i32:$vj,
1509+
uimm5:$imm)>;
1510+
def : Pat<(loongarch_vbsll v2i64:$vj, uimm5:$imm), (VBSLL_V v2i64:$vj,
1511+
uimm5:$imm)>;
1512+
def : Pat<(loongarch_vbsll v4f32:$vj, uimm5:$imm), (VBSLL_V v4f32:$vj,
1513+
uimm5:$imm)>;
1514+
def : Pat<(loongarch_vbsll v2f64:$vj, uimm5:$imm), (VBSLL_V v2f64:$vj,
1515+
uimm5:$imm)>;
1516+
1517+
// VBSRL_V
1518+
def : Pat<(loongarch_vbsrl v16i8:$vj, uimm5:$imm), (VBSRL_V v16i8:$vj,
1519+
uimm5:$imm)>;
1520+
def : Pat<(loongarch_vbsrl v8i16:$vj, uimm5:$imm), (VBSRL_V v8i16:$vj,
1521+
uimm5:$imm)>;
1522+
def : Pat<(loongarch_vbsrl v4i32:$vj, uimm5:$imm), (VBSRL_V v4i32:$vj,
1523+
uimm5:$imm)>;
1524+
def : Pat<(loongarch_vbsrl v2i64:$vj, uimm5:$imm), (VBSRL_V v2i64:$vj,
1525+
uimm5:$imm)>;
1526+
def : Pat<(loongarch_vbsrl v4f32:$vj, uimm5:$imm), (VBSRL_V v4f32:$vj,
1527+
uimm5:$imm)>;
1528+
def : Pat<(loongarch_vbsrl v2f64:$vj, uimm5:$imm), (VBSRL_V v2f64:$vj,
1529+
uimm5:$imm)>;
1530+
14971531
// VSLL[I]_{B/H/W/D}
14981532
defm : PatVrVr<shl, "VSLL">;
14991533
defm : PatShiftVrVr<shl, "VSLL">;
15001534
defm : PatShiftVrUimm<shl, "VSLLI">;
1535+
def : Pat<(loongarch_vslli v16i8:$vj, uimm3:$imm), (VSLLI_B v16i8:$vj,
1536+
uimm3:$imm)>;
1537+
def : Pat<(loongarch_vslli v8i16:$vj, uimm4:$imm), (VSLLI_H v8i16:$vj,
1538+
uimm4:$imm)>;
1539+
def : Pat<(loongarch_vslli v4i32:$vj, uimm5:$imm), (VSLLI_W v4i32:$vj,
1540+
uimm5:$imm)>;
1541+
def : Pat<(loongarch_vslli v2i64:$vj, uimm6:$imm), (VSLLI_D v2i64:$vj,
1542+
uimm6:$imm)>;
15011543

15021544
// VSRL[I]_{B/H/W/D}
15031545
defm : PatVrVr<srl, "VSRL">;
15041546
defm : PatShiftVrVr<srl, "VSRL">;
15051547
defm : PatShiftVrUimm<srl, "VSRLI">;
1548+
def : Pat<(loongarch_vsrli v16i8:$vj, uimm3:$imm), (VSRLI_B v16i8:$vj,
1549+
uimm3:$imm)>;
1550+
def : Pat<(loongarch_vsrli v8i16:$vj, uimm4:$imm), (VSRLI_H v8i16:$vj,
1551+
uimm4:$imm)>;
1552+
def : Pat<(loongarch_vsrli v4i32:$vj, uimm5:$imm), (VSRLI_W v4i32:$vj,
1553+
uimm5:$imm)>;
1554+
def : Pat<(loongarch_vsrli v2i64:$vj, uimm6:$imm), (VSRLI_D v2i64:$vj,
1555+
uimm6:$imm)>;
15061556

15071557
// VSRA[I]_{B/H/W/D}
15081558
defm : PatVrVr<sra, "VSRA">;

‎llvm/test/CodeGen/LoongArch/lsx/build-vector.ll

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -374,11 +374,8 @@ define void @extract1_i32_zext_insert0_i64_undef(ptr %src, ptr %dst) nounwind {
374374
; CHECK-LABEL: extract1_i32_zext_insert0_i64_undef:
375375
; CHECK: # %bb.0:
376376
; CHECK-NEXT: vld $vr0, $a0, 0
377-
; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI24_0)
378-
; CHECK-NEXT: vld $vr1, $a0, %pc_lo12(.LCPI24_0)
379-
; CHECK-NEXT: vrepli.b $vr2, 0
380-
; CHECK-NEXT: vshuf.w $vr1, $vr2, $vr0
381-
; CHECK-NEXT: vst $vr1, $a1, 0
377+
; CHECK-NEXT: vsrli.d $vr0, $vr0, 32
378+
; CHECK-NEXT: vst $vr0, $a1, 0
382379
; CHECK-NEXT: ret
383380
%v = load volatile <4 x i32>, ptr %src
384381
%e = extractelement <4 x i32> %v, i32 1

0 commit comments

Comments
 (0)