Skip to content

Commit 3f8d77b

Browse files
Dinar TemirbulatovDinar Temirbulatov
Dinar Temirbulatov
authored and
Dinar Temirbulatov
committed
Revert "[AArch64][SVE] Improve code quality of vector unsigned/signed add reductions. (#97339)"
This reverts commit b7b0071. The change caused regression in a performance testing.
1 parent f2eb7c7 commit 3f8d77b

File tree

7 files changed

+78
-330
lines changed

7 files changed

+78
-330
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 2 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -17576,71 +17576,6 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
1757617576
return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP);
1757717577
}
1757817578

17579-
// Turn [sign|zero]_extend(vecreduce_add()) into SVE's SADDV|UADDV
17580-
// instructions.
17581-
static SDValue
17582-
performVecReduceAddExtCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17583-
const AArch64TargetLowering &TLI) {
17584-
if (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
17585-
N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND)
17586-
return SDValue();
17587-
bool IsSigned = N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND;
17588-
17589-
SelectionDAG &DAG = DCI.DAG;
17590-
auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
17591-
SDValue VecOp = N->getOperand(0).getOperand(0);
17592-
EVT VecOpVT = VecOp.getValueType();
17593-
SDLoc DL(N);
17594-
17595-
// Split the input vectors if not legal, e.g.
17596-
// i32 (vecreduce_add (zext nxv32i8 %op to nxv32i32))
17597-
// ->
17598-
// i32 (add
17599-
// (i32 vecreduce_add (zext nxv16i8 %op.lo to nxv16i32)),
17600-
// (i32 vecreduce_add (zext nxv16i8 %op.hi to nxv16i32)))
17601-
if (TLI.getTypeAction(*DAG.getContext(), VecOpVT) ==
17602-
TargetLowering::TypeSplitVector) {
17603-
SDValue Lo, Hi;
17604-
std::tie(Lo, Hi) = DAG.SplitVector(VecOp, DL);
17605-
unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
17606-
EVT HalfVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
17607-
*DAG.getContext());
17608-
Lo = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
17609-
DAG.getNode(ExtOpc, DL, HalfVT, Lo));
17610-
Hi = DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0),
17611-
DAG.getNode(ExtOpc, DL, HalfVT, Hi));
17612-
return DAG.getNode(ISD::ADD, DL, N->getValueType(0), Lo, Hi);
17613-
}
17614-
17615-
if (!TLI.isTypeLegal(VecOpVT))
17616-
return SDValue();
17617-
17618-
if (VecOpVT.isFixedLengthVector() &&
17619-
!TLI.useSVEForFixedLengthVectorVT(VecOpVT, !Subtarget.isNeonAvailable()))
17620-
return SDValue();
17621-
17622-
// The input type is legal so map VECREDUCE_ADD to UADDV/SADDV, e.g.
17623-
// i32 (vecreduce_add (zext nxv16i8 %op to nxv16i32))
17624-
// ->
17625-
// i32 (UADDV nxv16i8:%op)
17626-
EVT ElemType = N->getValueType(0);
17627-
SDValue Pg = getPredicateForVector(DAG, DL, VecOpVT);
17628-
if (VecOpVT.isFixedLengthVector()) {
17629-
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VecOpVT);
17630-
VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
17631-
}
17632-
SDValue Res =
17633-
DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
17634-
DAG.getConstant(IsSigned ? Intrinsic::aarch64_sve_saddv
17635-
: Intrinsic::aarch64_sve_uaddv,
17636-
DL, MVT::i64),
17637-
Pg, VecOp);
17638-
if (ElemType != MVT::i64)
17639-
Res = DAG.getAnyExtOrTrunc(Res, DL, ElemType);
17640-
17641-
return Res;
17642-
}
17643-
1764417579
// Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
1764517580
// vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
1764617581
// vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
@@ -25326,11 +25261,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
2532625261
return performInsertVectorEltCombine(N, DCI);
2532725262
case ISD::EXTRACT_VECTOR_ELT:
2532825263
return performExtractVectorEltCombine(N, DCI, Subtarget);
25329-
case ISD::VECREDUCE_ADD: {
25330-
if (SDValue Val = performVecReduceAddCombine(N, DCI.DAG, Subtarget))
25331-
return Val;
25332-
return performVecReduceAddExtCombine(N, DCI, *this);
25333-
}
25264+
case ISD::VECREDUCE_ADD:
25265+
return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
2533425266
case AArch64ISD::UADDV:
2533525267
return performUADDVCombine(N, DAG);
2533625268
case AArch64ISD::SMULL:

‎llvm/test/CodeGen/AArch64/double_reduct.ll

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,11 @@ define i16 @add_ext_i16(<16 x i8> %a, <16 x i8> %b) {
145145
define i16 @add_ext_v32i16(<32 x i8> %a, <16 x i8> %b) {
146146
; CHECK-LABEL: add_ext_v32i16:
147147
; CHECK: // %bb.0:
148-
; CHECK-NEXT: uaddlp v1.8h, v1.16b
149-
; CHECK-NEXT: uadalp v1.8h, v0.16b
150-
; CHECK-NEXT: uadalp v1.8h, v2.16b
151-
; CHECK-NEXT: addv h0, v1.8h
148+
; CHECK-NEXT: uaddl2 v3.8h, v0.16b, v1.16b
149+
; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b
150+
; CHECK-NEXT: add v0.8h, v0.8h, v3.8h
151+
; CHECK-NEXT: uadalp v0.8h, v2.16b
152+
; CHECK-NEXT: addv h0, v0.8h
152153
; CHECK-NEXT: fmov w0, s0
153154
; CHECK-NEXT: ret
154155
%ae = zext <32 x i8> %a to <32 x i16>

‎llvm/test/CodeGen/AArch64/sve-doublereduct.ll

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,17 @@ define i32 @add_i32(<vscale x 8 x i32> %a, <vscale x 4 x i32> %b) {
103103
define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
104104
; CHECK-LABEL: add_ext_i16:
105105
; CHECK: // %bb.0:
106-
; CHECK-NEXT: ptrue p0.b
107-
; CHECK-NEXT: uaddv d0, p0, z0.b
108-
; CHECK-NEXT: uaddv d1, p0, z1.b
109-
; CHECK-NEXT: fmov w8, s0
110-
; CHECK-NEXT: fmov w9, s1
111-
; CHECK-NEXT: add w0, w8, w9
106+
; CHECK-NEXT: uunpkhi z2.h, z0.b
107+
; CHECK-NEXT: uunpklo z0.h, z0.b
108+
; CHECK-NEXT: uunpkhi z3.h, z1.b
109+
; CHECK-NEXT: uunpklo z1.h, z1.b
110+
; CHECK-NEXT: ptrue p0.h
111+
; CHECK-NEXT: add z0.h, z0.h, z2.h
112+
; CHECK-NEXT: add z1.h, z1.h, z3.h
113+
; CHECK-NEXT: add z0.h, z0.h, z1.h
114+
; CHECK-NEXT: uaddv d0, p0, z0.h
115+
; CHECK-NEXT: fmov x0, d0
116+
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
112117
; CHECK-NEXT: ret
113118
%ae = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
114119
%be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
@@ -121,15 +126,21 @@ define i16 @add_ext_i16(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
121126
define i16 @add_ext_v32i16(<vscale x 32 x i8> %a, <vscale x 16 x i8> %b) {
122127
; CHECK-LABEL: add_ext_v32i16:
123128
; CHECK: // %bb.0:
124-
; CHECK-NEXT: ptrue p0.b
125-
; CHECK-NEXT: uaddv d1, p0, z1.b
126-
; CHECK-NEXT: uaddv d0, p0, z0.b
127-
; CHECK-NEXT: uaddv d2, p0, z2.b
128-
; CHECK-NEXT: fmov w8, s1
129-
; CHECK-NEXT: fmov w9, s0
130-
; CHECK-NEXT: add w8, w9, w8
131-
; CHECK-NEXT: fmov w9, s2
132-
; CHECK-NEXT: add w0, w8, w9
129+
; CHECK-NEXT: uunpklo z3.h, z1.b
130+
; CHECK-NEXT: uunpklo z4.h, z0.b
131+
; CHECK-NEXT: uunpkhi z1.h, z1.b
132+
; CHECK-NEXT: uunpkhi z0.h, z0.b
133+
; CHECK-NEXT: uunpkhi z5.h, z2.b
134+
; CHECK-NEXT: uunpklo z2.h, z2.b
135+
; CHECK-NEXT: ptrue p0.h
136+
; CHECK-NEXT: add z0.h, z0.h, z1.h
137+
; CHECK-NEXT: add z1.h, z4.h, z3.h
138+
; CHECK-NEXT: add z0.h, z1.h, z0.h
139+
; CHECK-NEXT: add z1.h, z2.h, z5.h
140+
; CHECK-NEXT: add z0.h, z0.h, z1.h
141+
; CHECK-NEXT: uaddv d0, p0, z0.h
142+
; CHECK-NEXT: fmov x0, d0
143+
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
133144
; CHECK-NEXT: ret
134145
%ae = zext <vscale x 32 x i8> %a to <vscale x 32 x i16>
135146
%be = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>

‎llvm/test/CodeGen/AArch64/sve-fixed-vector-zext.ll

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
21

32
; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=256 -verify-machineinstrs | FileCheck %s --check-prefixes=SVE256
43
; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v1 -O3 -aarch64-sve-vector-bits-min=128 -verify-machineinstrs | FileCheck %s --check-prefixes=NEON
@@ -7,31 +6,24 @@
76

87
define internal i32 @test(ptr nocapture readonly %p1, i32 %i1, ptr nocapture readonly %p2, i32 %i2) {
98
; SVE256-LABEL: test:
10-
; SVE256: // %bb.0: // %L.entry
11-
; SVE256-NEXT: ptrue p0.h, vl16
12-
; SVE256-NEXT: mov w9, wzr
13-
; SVE256-NEXT: mov w10, wzr
14-
; SVE256-NEXT: mov w8, wzr
15-
; SVE256-NEXT: mov w11, #-16 // =0xfffffff0
16-
; SVE256-NEXT: .p2align 5, , 16
17-
; SVE256-NEXT: .LBB0_1: // %L1
18-
; SVE256-NEXT: // =>This Inner Loop Header: Depth=1
19-
; SVE256-NEXT: sxtw x12, w9
20-
; SVE256-NEXT: sxtw x13, w10
21-
; SVE256-NEXT: adds w11, w11, #1
22-
; SVE256-NEXT: add w10, w10, w3
23-
; SVE256-NEXT: ld1b { z0.h }, p0/z, [x0, x12]
24-
; SVE256-NEXT: ld1b { z1.h }, p0/z, [x2, x13]
25-
; SVE256-NEXT: add w9, w9, w1
26-
; SVE256-NEXT: sub z0.h, z0.h, z1.h
27-
; SVE256-NEXT: saddv d0, p0, z0.h
28-
; SVE256-NEXT: fmov w12, s0
29-
; SVE256-NEXT: add w8, w12, w8
30-
; SVE256-NEXT: b.lo .LBB0_1
31-
; SVE256-NEXT: // %bb.2: // %L2
32-
; SVE256-NEXT: mov w0, w8
33-
; SVE256-NEXT: ret
9+
; SVE256: ld1b { z0.h }, p0/z,
10+
; SVE256: ld1b { z1.h }, p0/z,
11+
; SVE256: sub z0.h, z0.h, z1.h
12+
; SVE256-NEXT: sunpklo z1.s, z0.h
13+
; SVE256-NEXT: ext z0.b, z0.b, z0.b, #16
14+
; SVE256-NEXT: sunpklo z0.s, z0.h
15+
; SVE256-NEXT: add z0.s, z1.s, z0.s
16+
; SVE256-NEXT: uaddv d0, p1, z0.s
3417

18+
; NEON-LABEL: test:
19+
; NEON: ldr q0, [x0, w9, sxtw]
20+
; NEON: ldr q1, [x2, w10, sxtw]
21+
; NEON: usubl2 v2.8h, v0.16b, v1.16b
22+
; NEON-NEXT: usubl v0.8h, v0.8b, v1.8b
23+
; NEON: saddl2 v1.4s, v0.8h, v2.8h
24+
; NEON-NEXT: saddl v0.4s, v0.4h, v2.4h
25+
; NEON-NEXT: add v0.4s, v0.4s, v1.4s
26+
; NEON-NEXT: addv s0, v0.4s
3527

3628
L.entry:
3729
br label %L1
@@ -63,5 +55,3 @@ L2: ; preds = %L1
6355
}
6456

6557
declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
66-
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
67-
; NEON: {{.*}}

‎llvm/test/CodeGen/AArch64/sve-int-reduce.ll

Lines changed: 0 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -188,94 +188,6 @@ define i64 @uaddv_nxv2i64(<vscale x 2 x i64> %a) {
188188
ret i64 %res
189189
}
190190

191-
define i32 @uaddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
192-
; CHECK-LABEL: uaddv_nxv16i8_nxv16i32:
193-
; CHECK: // %bb.0:
194-
; CHECK-NEXT: ptrue p0.b
195-
; CHECK-NEXT: uaddv d0, p0, z0.b
196-
; CHECK-NEXT: fmov x0, d0
197-
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
198-
; CHECK-NEXT: ret
199-
%1 = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>
200-
%2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
201-
ret i32 %2
202-
}
203-
204-
define i64 @uaddv_nxv16i16_nxv16i64(<vscale x 16 x i16> %a) {
205-
; CHECK-LABEL: uaddv_nxv16i16_nxv16i64:
206-
; CHECK: // %bb.0:
207-
; CHECK-NEXT: ptrue p0.h
208-
; CHECK-NEXT: uaddv d1, p0, z1.h
209-
; CHECK-NEXT: uaddv d0, p0, z0.h
210-
; CHECK-NEXT: fmov x8, d1
211-
; CHECK-NEXT: fmov x9, d0
212-
; CHECK-NEXT: add x0, x9, x8
213-
; CHECK-NEXT: ret
214-
%1 = zext <vscale x 16 x i16> %a to <vscale x 16 x i64>
215-
%2 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> %1)
216-
ret i64 %2
217-
}
218-
219-
define i32 @uaddv_nxv16i16_nxv16i32(<vscale x 32 x i16> %a) {
220-
; CHECK-LABEL: uaddv_nxv16i16_nxv16i32:
221-
; CHECK: // %bb.0:
222-
; CHECK-NEXT: ptrue p0.h
223-
; CHECK-NEXT: uaddv d3, p0, z3.h
224-
; CHECK-NEXT: uaddv d2, p0, z2.h
225-
; CHECK-NEXT: uaddv d1, p0, z1.h
226-
; CHECK-NEXT: uaddv d0, p0, z0.h
227-
; CHECK-NEXT: fmov w8, s3
228-
; CHECK-NEXT: fmov w9, s2
229-
; CHECK-NEXT: fmov w10, s1
230-
; CHECK-NEXT: fmov w11, s0
231-
; CHECK-NEXT: add w8, w9, w8
232-
; CHECK-NEXT: add w9, w11, w10
233-
; CHECK-NEXT: add w0, w9, w8
234-
; CHECK-NEXT: ret
235-
%1 = zext <vscale x 32 x i16> %a to <vscale x 32 x i32>
236-
%2 = call i32 @llvm.vector.reduce.add.nxv32i64(<vscale x 32 x i32> %1)
237-
ret i32 %2
238-
}
239-
240-
define i32 @saddv_nxv16i8_nxv16i32(<vscale x 16 x i8> %a) {
241-
; CHECK-LABEL: saddv_nxv16i8_nxv16i32:
242-
; CHECK: // %bb.0:
243-
; CHECK-NEXT: ptrue p0.b
244-
; CHECK-NEXT: saddv d0, p0, z0.b
245-
; CHECK-NEXT: fmov x0, d0
246-
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
247-
; CHECK-NEXT: ret
248-
%1 = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>
249-
%2 = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> %1)
250-
ret i32 %2
251-
}
252-
253-
define i32 @uaddv_nxv32i16_nxv32i32(ptr %a) {
254-
; CHECK-LABEL: uaddv_nxv32i16_nxv32i32:
255-
; CHECK: // %bb.0:
256-
; CHECK-NEXT: ptrue p0.h
257-
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, #3, mul vl]
258-
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, #2, mul vl]
259-
; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, #1, mul vl]
260-
; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0]
261-
; CHECK-NEXT: uaddv d0, p0, z0.h
262-
; CHECK-NEXT: uaddv d1, p0, z1.h
263-
; CHECK-NEXT: uaddv d2, p0, z2.h
264-
; CHECK-NEXT: uaddv d3, p0, z3.h
265-
; CHECK-NEXT: fmov w8, s0
266-
; CHECK-NEXT: fmov w9, s1
267-
; CHECK-NEXT: fmov w10, s2
268-
; CHECK-NEXT: fmov w11, s3
269-
; CHECK-NEXT: add w8, w9, w8
270-
; CHECK-NEXT: add w9, w11, w10
271-
; CHECK-NEXT: add w0, w9, w8
272-
; CHECK-NEXT: ret
273-
%1 = load <vscale x 32 x i16>, ptr %a, align 16
274-
%2 = zext <vscale x 32 x i16> %1 to <vscale x 32 x i32>
275-
%3 = call i32 @llvm.vector.reduce.add.nxv32i32(<vscale x 32 x i32> %2)
276-
ret i32 %3
277-
}
278-
279191
; UMINV
280192

281193
define i8 @umin_nxv16i8(<vscale x 16 x i8> %a) {

0 commit comments

Comments
 (0)