Skip to content

Commit b76f40c

Browse files
davemgreendavemgreen
committed
[DAG][AArch64][ARM] Recognize avg (hadd) from wrapping flags
This slightly extends the creation of hadd nodes to allow them to be generated with the original type size if wrapping flags allow. https://alive2.llvm.org/ce/z/bPjakD https://alive2.llvm.org/ce/z/fa_gzb Differential Revision: https://reviews.llvm.org/D143371
1 parent 121452a commit b76f40c

File tree

4 files changed

+42
-41
lines changed

4 files changed

+42
-41
lines changed

‎llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -944,33 +944,37 @@ static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG,
944944

945945
SDValue ExtOpA = Add.getOperand(0);
946946
SDValue ExtOpB = Add.getOperand(1);
947-
auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3) {
947+
SDValue Add2;
948+
auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
948949
ConstantSDNode *ConstOp;
949950
if ((ConstOp = isConstOrConstSplat(Op1, DemandedElts)) &&
950951
ConstOp->isOne()) {
951952
ExtOpA = Op2;
952953
ExtOpB = Op3;
954+
Add2 = A;
953955
return true;
954956
}
955957
if ((ConstOp = isConstOrConstSplat(Op2, DemandedElts)) &&
956958
ConstOp->isOne()) {
957959
ExtOpA = Op1;
958960
ExtOpB = Op3;
961+
Add2 = A;
959962
return true;
960963
}
961964
if ((ConstOp = isConstOrConstSplat(Op3, DemandedElts)) &&
962965
ConstOp->isOne()) {
963966
ExtOpA = Op1;
964967
ExtOpB = Op2;
968+
Add2 = A;
965969
return true;
966970
}
967971
return false;
968972
};
969973
bool IsCeil =
970974
(ExtOpA.getOpcode() == ISD::ADD &&
971-
MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB)) ||
975+
MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) ||
972976
(ExtOpB.getOpcode() == ISD::ADD &&
973-
MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA));
977+
MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB));
974978

975979
// If the shift is signed (sra):
976980
// - Needs >= 2 sign bit for both operands.
@@ -1033,8 +1037,18 @@ static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG,
10331037
EVT NVT = EVT::getIntegerVT(*DAG.getContext(), llvm::bit_ceil(MinWidth));
10341038
if (VT.isVector())
10351039
NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount());
1036-
if (!TLI.isOperationLegalOrCustom(AVGOpc, NVT))
1037-
return SDValue();
1040+
if (!TLI.isOperationLegalOrCustom(AVGOpc, NVT)) {
1041+
// If we could not transform, and (both) adds are nuw/nsw, we can use the
1042+
// larger type size to do the transform.
1043+
if (((!IsSigned && Add->getFlags().hasNoUnsignedWrap() &&
1044+
(!Add2 || Add2->getFlags().hasNoUnsignedWrap())) ||
1045+
(IsSigned && Add->getFlags().hasNoSignedWrap() &&
1046+
(!Add2 || Add2->getFlags().hasNoSignedWrap()))) &&
1047+
TLI.isOperationLegalOrCustom(AVGOpc, VT)) {
1048+
NVT = VT;
1049+
} else
1050+
return SDValue();
1051+
}
10381052

10391053
SDLoc DL(Op);
10401054
SDValue ResultAVG =

‎llvm/test/CodeGen/AArch64/arm64-vhadd.ll

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -813,8 +813,8 @@ define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {
813813
; CHECK-NEXT: shl.4h v0, v0, #8
814814
; CHECK-NEXT: shl.4h v1, v1, #8
815815
; CHECK-NEXT: sshr.4h v0, v0, #8
816-
; CHECK-NEXT: ssra.4h v0, v1, #8
817-
; CHECK-NEXT: sshr.4h v0, v0, #1
816+
; CHECK-NEXT: sshr.4h v1, v1, #8
817+
; CHECK-NEXT: shadd.4h v0, v0, v1
818818
; CHECK-NEXT: ret
819819
%zextsrc1 = sext <4 x i8> %src1 to <4 x i16>
820820
%zextsrc2 = sext <4 x i8> %src2 to <4 x i16>
@@ -828,8 +828,7 @@ define <4 x i16> @hadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {
828828
; CHECK: // %bb.0:
829829
; CHECK-NEXT: bic.4h v0, #255, lsl #8
830830
; CHECK-NEXT: bic.4h v1, #255, lsl #8
831-
; CHECK-NEXT: add.4h v0, v0, v1
832-
; CHECK-NEXT: ushr.4h v0, v0, #1
831+
; CHECK-NEXT: uhadd.4h v0, v0, v1
833832
; CHECK-NEXT: ret
834833
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
835834
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>
@@ -859,8 +858,7 @@ define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {
859858
; CHECK: // %bb.0:
860859
; CHECK-NEXT: bic.4h v0, #255, lsl #8
861860
; CHECK-NEXT: bic.4h v1, #255, lsl #8
862-
; CHECK-NEXT: add.4h v0, v0, v1
863-
; CHECK-NEXT: ushr.4h v0, v0, #1
861+
; CHECK-NEXT: uhadd.4h v0, v0, v1
864862
; CHECK-NEXT: ret
865863
%zextsrc1 = zext <4 x i8> %src1 to <4 x i16>
866864
%zextsrc2 = zext <4 x i8> %src2 to <4 x i16>

‎llvm/test/CodeGen/AArch64/sve2-hadd.ll

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,10 @@ entry:
5151
define <vscale x 2 x i32> @haddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {
5252
; CHECK-LABEL: haddu_v2i32:
5353
; CHECK: // %bb.0: // %entry
54+
; CHECK-NEXT: ptrue p0.d
5455
; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
55-
; CHECK-NEXT: adr z0.d, [z0.d, z1.d, uxtw]
56-
; CHECK-NEXT: lsr z0.d, z0.d, #1
56+
; CHECK-NEXT: and z1.d, z1.d, #0xffffffff
57+
; CHECK-NEXT: uhadd z0.d, p0/m, z0.d, z1.d
5758
; CHECK-NEXT: ret
5859
entry:
5960
%s0s = zext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>
@@ -116,10 +117,10 @@ entry:
116117
define <vscale x 2 x i16> @haddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
117118
; CHECK-LABEL: haddu_v2i16:
118119
; CHECK: // %bb.0: // %entry
120+
; CHECK-NEXT: ptrue p0.d
119121
; CHECK-NEXT: and z0.d, z0.d, #0xffff
120122
; CHECK-NEXT: and z1.d, z1.d, #0xffff
121-
; CHECK-NEXT: add z0.d, z0.d, z1.d
122-
; CHECK-NEXT: lsr z0.d, z0.d, #1
123+
; CHECK-NEXT: uhadd z0.d, p0/m, z0.d, z1.d
123124
; CHECK-NEXT: ret
124125
entry:
125126
%s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>
@@ -151,10 +152,10 @@ entry:
151152
define <vscale x 4 x i16> @haddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {
152153
; CHECK-LABEL: haddu_v4i16:
153154
; CHECK: // %bb.0: // %entry
155+
; CHECK-NEXT: ptrue p0.s
154156
; CHECK-NEXT: and z0.s, z0.s, #0xffff
155157
; CHECK-NEXT: and z1.s, z1.s, #0xffff
156-
; CHECK-NEXT: add z0.s, z0.s, z1.s
157-
; CHECK-NEXT: lsr z0.s, z0.s, #1
158+
; CHECK-NEXT: uhadd z0.s, p0/m, z0.s, z1.s
158159
; CHECK-NEXT: ret
159160
entry:
160161
%s0s = zext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>
@@ -217,10 +218,10 @@ entry:
217218
define <vscale x 4 x i8> @haddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
218219
; CHECK-LABEL: haddu_v4i8:
219220
; CHECK: // %bb.0: // %entry
221+
; CHECK-NEXT: ptrue p0.s
220222
; CHECK-NEXT: and z0.s, z0.s, #0xff
221223
; CHECK-NEXT: and z1.s, z1.s, #0xff
222-
; CHECK-NEXT: add z0.s, z0.s, z1.s
223-
; CHECK-NEXT: lsr z0.s, z0.s, #1
224+
; CHECK-NEXT: uhadd z0.s, p0/m, z0.s, z1.s
224225
; CHECK-NEXT: ret
225226
entry:
226227
%s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>
@@ -252,10 +253,10 @@ entry:
252253
define <vscale x 8 x i8> @haddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {
253254
; CHECK-LABEL: haddu_v8i8:
254255
; CHECK: // %bb.0: // %entry
256+
; CHECK-NEXT: ptrue p0.h
255257
; CHECK-NEXT: and z0.h, z0.h, #0xff
256258
; CHECK-NEXT: and z1.h, z1.h, #0xff
257-
; CHECK-NEXT: add z0.h, z0.h, z1.h
258-
; CHECK-NEXT: lsr z0.h, z0.h, #1
259+
; CHECK-NEXT: uhadd z0.h, p0/m, z0.h, z1.h
259260
; CHECK-NEXT: ret
260261
entry:
261262
%s0s = zext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>
@@ -352,12 +353,10 @@ entry:
352353
define <vscale x 2 x i32> @rhaddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {
353354
; CHECK-LABEL: rhaddu_v2i32:
354355
; CHECK: // %bb.0: // %entry
355-
; CHECK-NEXT: mov z2.d, #-1 // =0xffffffffffffffff
356+
; CHECK-NEXT: ptrue p0.d
356357
; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
357358
; CHECK-NEXT: and z1.d, z1.d, #0xffffffff
358-
; CHECK-NEXT: eor z0.d, z0.d, z2.d
359-
; CHECK-NEXT: sub z0.d, z1.d, z0.d
360-
; CHECK-NEXT: lsr z0.d, z0.d, #1
359+
; CHECK-NEXT: urhadd z0.d, p0/m, z0.d, z1.d
361360
; CHECK-NEXT: ret
362361
entry:
363362
%s0s = zext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>
@@ -467,12 +466,10 @@ entry:
467466
define <vscale x 4 x i16> @rhaddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {
468467
; CHECK-LABEL: rhaddu_v4i16:
469468
; CHECK: // %bb.0: // %entry
470-
; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff
469+
; CHECK-NEXT: ptrue p0.s
471470
; CHECK-NEXT: and z0.s, z0.s, #0xffff
472471
; CHECK-NEXT: and z1.s, z1.s, #0xffff
473-
; CHECK-NEXT: eor z0.d, z0.d, z2.d
474-
; CHECK-NEXT: sub z0.s, z1.s, z0.s
475-
; CHECK-NEXT: lsr z0.s, z0.s, #1
472+
; CHECK-NEXT: urhadd z0.s, p0/m, z0.s, z1.s
476473
; CHECK-NEXT: ret
477474
entry:
478475
%s0s = zext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>
@@ -582,12 +579,10 @@ entry:
582579
define <vscale x 8 x i8> @rhaddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {
583580
; CHECK-LABEL: rhaddu_v8i8:
584581
; CHECK: // %bb.0: // %entry
585-
; CHECK-NEXT: mov z2.h, #-1 // =0xffffffffffffffff
582+
; CHECK-NEXT: ptrue p0.h
586583
; CHECK-NEXT: and z0.h, z0.h, #0xff
587584
; CHECK-NEXT: and z1.h, z1.h, #0xff
588-
; CHECK-NEXT: eor z0.d, z0.d, z2.d
589-
; CHECK-NEXT: sub z0.h, z1.h, z0.h
590-
; CHECK-NEXT: lsr z0.h, z0.h, #1
585+
; CHECK-NEXT: urhadd z0.h, p0/m, z0.h, z1.h
591586
; CHECK-NEXT: ret
592587
entry:
593588
%s0s = zext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>

‎llvm/test/CodeGen/Thumb2/mve-vhadd.ll

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -243,10 +243,7 @@ define arm_aapcs_vfpcc <4 x i16> @vrhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
243243
; CHECK: @ %bb.0: @ %entry
244244
; CHECK-NEXT: vmovlb.u16 q1, q1
245245
; CHECK-NEXT: vmovlb.u16 q0, q0
246-
; CHECK-NEXT: vadd.i32 q0, q0, q1
247-
; CHECK-NEXT: movs r0, #1
248-
; CHECK-NEXT: vadd.i32 q0, q0, r0
249-
; CHECK-NEXT: vshr.u32 q0, q0, #1
246+
; CHECK-NEXT: vrhadd.u32 q0, q0, q1
250247
; CHECK-NEXT: bx lr
251248
entry:
252249
%s0s = zext <4 x i16> %s0 to <4 x i32>
@@ -357,10 +354,7 @@ define arm_aapcs_vfpcc <8 x i8> @vrhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
357354
; CHECK: @ %bb.0: @ %entry
358355
; CHECK-NEXT: vmovlb.u8 q1, q1
359356
; CHECK-NEXT: vmovlb.u8 q0, q0
360-
; CHECK-NEXT: vadd.i16 q0, q0, q1
361-
; CHECK-NEXT: movs r0, #1
362-
; CHECK-NEXT: vadd.i16 q0, q0, r0
363-
; CHECK-NEXT: vshr.u16 q0, q0, #1
357+
; CHECK-NEXT: vrhadd.u16 q0, q0, q1
364358
; CHECK-NEXT: bx lr
365359
entry:
366360
%s0s = zext <8 x i8> %s0 to <8 x i16>

0 commit comments

Comments
 (0)