Skip to content

Commit 514b38c

Browse files
preamespreames
committed
[RISCV] Remove mask size restriction on single source and dual src shuffle costing (try 2)
Some callers pass in an empty mask to represent "unknown". We should use the generic costs for these cases. We can add VL=1 costing seperately if desired. Reapplying after revert. A new test had been added, and I'd missed updating it when rebasing before. This is a great happy accident as I hadn't figured out how to get SLP to exercise this case, I'd merely noticed it via inspection.
1 parent 5eb44df commit 514b38c

File tree

2 files changed

+71
-47
lines changed

2 files changed

+71
-47
lines changed

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -324,36 +324,33 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
324324
return LT.first * getLMULCost(LT.second);
325325
}
326326
}
327-
328-
// vrgather + cost of generating the mask constant.
329-
// We model this for an unknown mask with a single vrgather.
330-
if (LT.first == 1 &&
331-
(LT.second.getScalarSizeInBits() != 8 ||
332-
LT.second.getVectorNumElements() <= 256)) {
333-
VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext());
334-
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
335-
return IndexCost + getVRGatherVVCost(LT.second);
336-
}
327+
}
328+
// vrgather + cost of generating the mask constant.
329+
// We model this for an unknown mask with a single vrgather.
330+
if (LT.second.isFixedLengthVector() && LT.first == 1 &&
331+
(LT.second.getScalarSizeInBits() != 8 ||
332+
LT.second.getVectorNumElements() <= 256)) {
333+
VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext());
334+
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
335+
return IndexCost + getVRGatherVVCost(LT.second);
337336
}
338337
[[fallthrough]];
339338
}
340339
case TTI::SK_Transpose:
341340
case TTI::SK_PermuteTwoSrc: {
342-
if (Mask.size() >= 2 && LT.second.isFixedLengthVector()) {
343-
// 2 x (vrgather + cost of generating the mask constant) + cost of mask
344-
// register for the second vrgather. We model this for an unknown
345-
// (shuffle) mask.
346-
if (LT.first == 1 &&
347-
(LT.second.getScalarSizeInBits() != 8 ||
348-
LT.second.getVectorNumElements() <= 256)) {
349-
auto &C = Tp->getContext();
350-
auto EC = Tp->getElementCount();
351-
VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C);
352-
VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC);
353-
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
354-
InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);
355-
return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost;
356-
}
341+
// 2 x (vrgather + cost of generating the mask constant) + cost of mask
342+
// register for the second vrgather. We model this for an unknown
343+
// (shuffle) mask.
344+
if (LT.second.isFixedLengthVector() && LT.first == 1 &&
345+
(LT.second.getScalarSizeInBits() != 8 ||
346+
LT.second.getVectorNumElements() <= 256)) {
347+
auto &C = Tp->getContext();
348+
auto EC = Tp->getElementCount();
349+
VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C);
350+
VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC);
351+
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
352+
InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);
353+
return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost;
357354
}
358355
[[fallthrough]];
359356
}

‎llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
22
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \
3-
; RUN: -riscv-v-vector-bits-min=128 -riscv-v-slp-max-vf=0 -S \
4-
; RUN: | FileCheck %s --check-prefixes=CHECK
5-
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \
6-
; RUN: -riscv-v-vector-bits-min=256 -riscv-v-slp-max-vf=0 -S \
7-
; RUN: | FileCheck %s --check-prefixes=CHECK
8-
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \
9-
; RUN: -riscv-v-vector-bits-min=512 -riscv-v-slp-max-vf=0 -S \
10-
; RUN: | FileCheck %s --check-prefixes=CHECK
3+
; RUN: -riscv-v-slp-max-vf=0 -S \
4+
; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL128
5+
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl256b \
6+
; RUN: -riscv-v-slp-max-vf=0 -S \
7+
; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL256
8+
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl512b \
9+
; RUN: -riscv-v-slp-max-vf=0 -S \
10+
; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL512
1111

1212
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
1313
target triple = "riscv64"
@@ -274,20 +274,47 @@ entry:
274274
}
275275

276276
define void @reduce_or_2() {
277-
; CHECK-LABEL: @reduce_or_2(
278-
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 0, 0
279-
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison>, i64 [[TMP1]], i32 15
280-
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer
281-
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 6
282-
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer
283-
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
284-
; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
285-
; CHECK-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]]
286-
; CHECK-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]]
287-
; CHECK: 8:
288-
; CHECK-NEXT: ret void
289-
; CHECK: 9:
290-
; CHECK-NEXT: ret void
277+
; ZVL128-LABEL: @reduce_or_2(
278+
; ZVL128-NEXT: [[TMP1:%.*]] = shl i64 0, 0
279+
; ZVL128-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison>, i64 [[TMP1]], i32 15
280+
; ZVL128-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer
281+
; ZVL128-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 6
282+
; ZVL128-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer
283+
; ZVL128-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
284+
; ZVL128-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
285+
; ZVL128-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]]
286+
; ZVL128-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]]
287+
; ZVL128: 8:
288+
; ZVL128-NEXT: ret void
289+
; ZVL128: 9:
290+
; ZVL128-NEXT: ret void
291+
;
292+
; ZVL256-LABEL: @reduce_or_2(
293+
; ZVL256-NEXT: [[TMP1:%.*]] = shl i64 0, 0
294+
; ZVL256-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison>, i64 [[TMP1]], i32 15
295+
; ZVL256-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer
296+
; ZVL256-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 6
297+
; ZVL256-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer
298+
; ZVL256-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
299+
; ZVL256-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
300+
; ZVL256-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]]
301+
; ZVL256-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]]
302+
; ZVL256: 8:
303+
; ZVL256-NEXT: ret void
304+
; ZVL256: 9:
305+
; ZVL256-NEXT: ret void
306+
;
307+
; ZVL512-LABEL: @reduce_or_2(
308+
; ZVL512-NEXT: [[TMP1:%.*]] = shl i64 0, 0
309+
; ZVL512-NEXT: [[TMP2:%.*]] = insertelement <32 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 15
310+
; ZVL512-NEXT: [[TMP3:%.*]] = shufflevector <32 x i64> [[TMP2]], <32 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 15, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
311+
; ZVL512-NEXT: [[TMP4:%.*]] = icmp ult <32 x i64> [[TMP3]], zeroinitializer
312+
; ZVL512-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP4]])
313+
; ZVL512-NEXT: br i1 [[TMP5]], label [[TMP7:%.*]], label [[TMP6:%.*]]
314+
; ZVL512: 6:
315+
; ZVL512-NEXT: ret void
316+
; ZVL512: 7:
317+
; ZVL512-NEXT: ret void
291318
;
292319
%1 = shl i64 0, 0
293320
%2 = icmp ult i64 0, 0

0 commit comments

Comments
 (0)