Skip to content

Commit 7b4100d

Browse files
lukel97lukel97
committed
[RISCV] Widen i1 AnyOf reductions
1 parent f590430 commit 7b4100d

File tree

3 files changed

+104
-29
lines changed

3 files changed

+104
-29
lines changed

‎llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "llvm/IR/PatternMatch.h"
2626
#include "llvm/InitializePasses.h"
2727
#include "llvm/Pass.h"
28+
#include "llvm/Transforms/Utils/Local.h"
2829

2930
using namespace llvm;
3031

@@ -58,6 +59,7 @@ class RISCVCodeGenPrepare : public FunctionPass,
5859
bool visitAnd(BinaryOperator &BO);
5960
bool visitIntrinsicInst(IntrinsicInst &I);
6061
bool expandVPStrideLoad(IntrinsicInst &I);
62+
bool widenVPMerge(IntrinsicInst &I);
6163
};
6264

6365
} // end anonymous namespace
@@ -103,6 +105,76 @@ bool RISCVCodeGenPrepare::visitAnd(BinaryOperator &BO) {
103105
return true;
104106
}
105107

108+
// With EVL tail folding, an AnyOf reduction will generate an i1 vp.merge like
109+
// follows:
110+
//
111+
// loop:
112+
// %phi = phi <vscale x 4 x i1> [ zeroinitializer, %entry ], [ %rec, %loop ]
113+
// %cmp = icmp ...
114+
// %rec = call <vscale x 4 x i1> @llvm.vp.merge(%cmp, i1 true, %phi, %evl)
115+
// ...
116+
// middle:
117+
// %res = call i1 @llvm.vector.reduce.or(<vscale x 4 x i1> %rec)
118+
//
119+
// However RVV doesn't have any tail undisturbed mask instructions and so we
120+
// need a convoluted sequence of mask instructions to lower the i1 vp.merge: see
121+
// llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll.
122+
//
123+
// To avoid that this widens the i1 vp.merge to an i8 vp.merge, which will
124+
// usually be folded into a masked vor.vv.
125+
//
126+
// loop:
127+
// %phi = phi <vscale x 4 x i8> [ zeroinitializer, %entry ], [ %rec, %loop ]
128+
// %cmp = icmp ...
129+
// %rec = call <vscale x 4 x i8> @llvm.vp.merge(%cmp, i8 true, %phi, %evl)
130+
// %trunc = trunc <vscale x 4 x i8> %rec to <vscale x 4 x i1>
131+
// ...
132+
// middle:
133+
// %res = call i1 @llvm.vector.reduce.or(<vscale x 4 x i1> %rec)
134+
//
135+
// The trunc will normally be sunk outside of the loop, but even if there are
136+
// users inside the loop it is still profitable.
137+
bool RISCVCodeGenPrepare::widenVPMerge(IntrinsicInst &II) {
138+
if (!II.getType()->getScalarType()->isIntegerTy(1))
139+
return false;
140+
141+
Value *Mask, *True, *PhiV, *EVL;
142+
using namespace PatternMatch;
143+
if (!match(&II,
144+
m_Intrinsic<Intrinsic::vp_merge>(m_Value(Mask), m_Value(True),
145+
m_Value(PhiV), m_Value(EVL))))
146+
return false;
147+
148+
auto *Phi = dyn_cast<PHINode>(PhiV);
149+
if (!Phi || Phi->getNumUses() > 2 || Phi->getNumIncomingValues() != 2 ||
150+
!match(Phi->getIncomingValue(0), m_Zero()) ||
151+
Phi->getIncomingValue(1) != &II)
152+
return false;
153+
154+
Type *WideTy =
155+
VectorType::get(IntegerType::getInt8Ty(II.getContext()),
156+
cast<VectorType>(II.getType())->getElementCount());
157+
158+
IRBuilder<> Builder(Phi);
159+
PHINode *WidePhi = Builder.CreatePHI(WideTy, 2);
160+
WidePhi->addIncoming(ConstantAggregateZero::get(WideTy),
161+
Phi->getIncomingBlock(0));
162+
Builder.SetInsertPoint(&II);
163+
Value *WideTrue = Builder.CreateZExt(True, WideTy);
164+
Value *WideMerge = Builder.CreateIntrinsic(Intrinsic::vp_merge, {WideTy},
165+
{Mask, WideTrue, WidePhi, EVL});
166+
WidePhi->addIncoming(WideMerge, Phi->getIncomingBlock(1));
167+
Value *Trunc = Builder.CreateTrunc(WideMerge, II.getType());
168+
169+
II.replaceAllUsesWith(Trunc);
170+
171+
// Break the cycle and delete the old chain.
172+
Phi->setIncomingValue(1, Phi->getIncomingValue(0));
173+
llvm::RecursivelyDeleteTriviallyDeadInstructions(&II);
174+
175+
return true;
176+
}
177+
106178
// LLVM vector reduction intrinsics return a scalar result, but on RISC-V vector
107179
// reduction instructions write the result in the first element of a vector
108180
// register. So when a reduction in a loop uses a scalar phi, we end up with
@@ -138,6 +210,9 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
138210
if (expandVPStrideLoad(I))
139211
return true;
140212

213+
if (widenVPMerge(I))
214+
return true;
215+
141216
if (I.getIntrinsicID() != Intrinsic::vector_reduce_fadd &&
142217
!isa<VPReductionIntrinsic>(&I))
143218
return false;

‎llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -132,26 +132,25 @@ define i1 @widen_anyof_rdx(ptr %p, i64 %n) {
132132
; CHECK-LABEL: widen_anyof_rdx:
133133
; CHECK: # %bb.0: # %entry
134134
; CHECK-NEXT: li a2, 0
135-
; CHECK-NEXT: vsetvli a3, zero, e64, m4, ta, ma
136-
; CHECK-NEXT: vmclr.m v12
137-
; CHECK-NEXT: vid.v v8
135+
; CHECK-NEXT: vsetvli a3, zero, e8, mf2, ta, ma
136+
; CHECK-NEXT: vmv.v.i v8, 0
138137
; CHECK-NEXT: .LBB2_1: # %loop
139138
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
140139
; CHECK-NEXT: sub a3, a1, a2
141140
; CHECK-NEXT: slli a4, a2, 2
142-
; CHECK-NEXT: vsetvli a3, a3, e8, mf2, ta, ma
141+
; CHECK-NEXT: vsetvli a3, a3, e32, m2, ta, ma
143142
; CHECK-NEXT: add a4, a0, a4
144-
; CHECK-NEXT: vle32.v v14, (a4)
145-
; CHECK-NEXT: vsetvli a4, zero, e32, m2, ta, ma
146-
; CHECK-NEXT: vmsne.vi v13, v14, 0
147-
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma
148-
; CHECK-NEXT: vmsltu.vx v14, v8, a3
149-
; CHECK-NEXT: vmand.mm v13, v13, v14
143+
; CHECK-NEXT: vle32.v v10, (a4)
144+
; CHECK-NEXT: vmsne.vi v0, v10, 0
150145
; CHECK-NEXT: add a2, a2, a3
151-
; CHECK-NEXT: vmor.mm v12, v12, v13
146+
; CHECK-NEXT: vsetvli zero, zero, e8, mf2, tu, ma
147+
; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
152148
; CHECK-NEXT: blt a2, a1, .LBB2_1
153149
; CHECK-NEXT: # %bb.2: # %exit
154-
; CHECK-NEXT: vcpop.m a0, v12
150+
; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma
151+
; CHECK-NEXT: vand.vi v8, v8, 1
152+
; CHECK-NEXT: vmsne.vi v8, v8, 0
153+
; CHECK-NEXT: vcpop.m a0, v8
155154
; CHECK-NEXT: snez a0, a0
156155
; CHECK-NEXT: ret
157156
entry:
@@ -181,27 +180,26 @@ define i1 @widen_anyof_rdx_use_in_loop(ptr %p, i64 %n) {
181180
; CHECK-LABEL: widen_anyof_rdx_use_in_loop:
182181
; CHECK: # %bb.0: # %entry
183182
; CHECK-NEXT: li a2, 0
184-
; CHECK-NEXT: vsetvli a3, zero, e64, m4, ta, ma
185-
; CHECK-NEXT: vmclr.m v12
186-
; CHECK-NEXT: vid.v v8
183+
; CHECK-NEXT: vsetvli a3, zero, e8, mf2, ta, ma
184+
; CHECK-NEXT: vmv.v.i v8, 0
187185
; CHECK-NEXT: .LBB3_1: # %loop
188186
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
189187
; CHECK-NEXT: sub a3, a1, a2
190188
; CHECK-NEXT: slli a4, a2, 2
191-
; CHECK-NEXT: vsetvli a3, a3, e8, mf2, ta, ma
189+
; CHECK-NEXT: vsetvli a3, a3, e32, m2, ta, ma
192190
; CHECK-NEXT: add a4, a0, a4
193-
; CHECK-NEXT: vle32.v v14, (a4)
194-
; CHECK-NEXT: vsetvli a5, zero, e32, m2, ta, ma
195-
; CHECK-NEXT: vmsne.vi v13, v14, 0
196-
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma
197-
; CHECK-NEXT: vmsltu.vx v14, v8, a3
198-
; CHECK-NEXT: vmand.mm v13, v13, v14
199-
; CHECK-NEXT: vmor.mm v12, v12, v13
191+
; CHECK-NEXT: vle32.v v10, (a4)
192+
; CHECK-NEXT: vmsne.vi v0, v10, 0
193+
; CHECK-NEXT: vsetvli zero, zero, e8, mf2, tu, ma
194+
; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
195+
; CHECK-NEXT: vsetvli a5, zero, e8, mf2, ta, ma
196+
; CHECK-NEXT: vand.vi v9, v8, 1
197+
; CHECK-NEXT: vmsne.vi v9, v9, 0
200198
; CHECK-NEXT: add a2, a2, a3
201-
; CHECK-NEXT: vsm.v v12, (a4)
199+
; CHECK-NEXT: vsm.v v9, (a4)
202200
; CHECK-NEXT: blt a2, a1, .LBB3_1
203201
; CHECK-NEXT: # %bb.2: # %exit
204-
; CHECK-NEXT: vcpop.m a0, v12
202+
; CHECK-NEXT: vcpop.m a0, v9
205203
; CHECK-NEXT: snez a0, a0
206204
; CHECK-NEXT: ret
207205
entry:

‎llvm/test/CodeGen/RISCV/riscv-codegenprepare.ll

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,14 @@ define i1 @widen_anyof_rdx(ptr %p, i64 %n) {
110110
; CHECK-NEXT: br label [[LOOP:%.*]]
111111
; CHECK: loop:
112112
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
113-
; CHECK-NEXT: [[PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[ENTRY]] ], [ [[TMP4:%.*]], [[LOOP]] ]
113+
; CHECK-NEXT: [[TMP0:%.*]] = phi <vscale x 4 x i8> [ zeroinitializer, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
114114
; CHECK-NEXT: [[AVL:%.*]] = sub i64 [[N:%.*]], [[IV]]
115115
; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
116116
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[IV]]
117117
; CHECK-NEXT: [[X:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
118118
; CHECK-NEXT: [[CMP:%.*]] = icmp ne <vscale x 4 x i32> [[X]], zeroinitializer
119-
; CHECK-NEXT: [[TMP4]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[CMP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[PHI]], i32 [[EVL]])
119+
; CHECK-NEXT: [[TMP1]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> [[CMP]], <vscale x 4 x i8> splat (i8 1), <vscale x 4 x i8> [[TMP0]], i32 [[EVL]])
120+
; CHECK-NEXT: [[TMP4:%.*]] = trunc <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i1>
120121
; CHECK-NEXT: [[EVL_ZEXT:%.*]] = zext i32 [[EVL]] to i64
121122
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], [[EVL_ZEXT]]
122123
; CHECK-NEXT: [[DONE:%.*]] = icmp sge i64 [[IV_NEXT]], [[N]]
@@ -154,13 +155,14 @@ define i1 @widen_anyof_rdx_use_in_loop(ptr %p, i64 %n) {
154155
; CHECK-NEXT: br label [[LOOP:%.*]]
155156
; CHECK: loop:
156157
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
157-
; CHECK-NEXT: [[PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[ENTRY]] ], [ [[REC:%.*]], [[LOOP]] ]
158+
; CHECK-NEXT: [[TMP0:%.*]] = phi <vscale x 4 x i8> [ zeroinitializer, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
158159
; CHECK-NEXT: [[AVL:%.*]] = sub i64 [[N:%.*]], [[IV]]
159160
; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
160161
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[IV]]
161162
; CHECK-NEXT: [[X:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
162163
; CHECK-NEXT: [[CMP:%.*]] = icmp ne <vscale x 4 x i32> [[X]], zeroinitializer
163-
; CHECK-NEXT: [[REC]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[CMP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[PHI]], i32 [[EVL]])
164+
; CHECK-NEXT: [[TMP1]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> [[CMP]], <vscale x 4 x i8> splat (i8 1), <vscale x 4 x i8> [[TMP0]], i32 [[EVL]])
165+
; CHECK-NEXT: [[REC:%.*]] = trunc <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i1>
164166
; CHECK-NEXT: store <vscale x 4 x i1> [[REC]], ptr [[GEP]], align 1
165167
; CHECK-NEXT: [[EVL_ZEXT:%.*]] = zext i32 [[EVL]] to i64
166168
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], [[EVL_ZEXT]]

0 commit comments

Comments
 (0)