Skip to content

[SLP][REVEC] Make getAltInstrMask and getGatherCost vectorize vector instructions. #99461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 8, 2024
Merged

[SLP][REVEC] Make getAltInstrMask and getGatherCost vectorize vector instructions. #99461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
147fbb6
[SLP][REVEC] Pre-commit test.
HanKuanChen Jul 24, 2024
5812ce1
[SLP][REVEC] Make getAltInstrMask support vector instructions.
HanKuanChen Jun 28, 2024
49355ac
[SLP][REVEC] Pre-commit test.
HanKuanChen Jul 24, 2024
dd30da7
[SLP][REVEC] Make getGatherCost support vector instructions.
HanKuanChen Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1047,10 +1047,13 @@ static void fixupOrderingIndices(MutableArrayRef<unsigned> Order) {
/// Opcode1.
SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, unsigned Opcode0,
unsigned Opcode1) {
SmallBitVector OpcodeMask(VL.size(), false);
Type *ScalarTy = VL[0]->getType();
unsigned ScalarTyNumElements = getNumElements(ScalarTy);
SmallBitVector OpcodeMask(VL.size() * ScalarTyNumElements, false);
for (unsigned Lane : seq<unsigned>(VL.size()))
if (cast<Instruction>(VL[Lane])->getOpcode() == Opcode1)
OpcodeMask.set(Lane);
OpcodeMask.set(Lane * ScalarTyNumElements,
Lane * ScalarTyNumElements + ScalarTyNumElements);
return OpcodeMask;
}

Expand Down Expand Up @@ -11487,7 +11490,8 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
// Find the cost of inserting/extracting values from the vector.
// Check if the same elements are inserted several times and count them as
// shuffle candidates.
APInt ShuffledElements = APInt::getZero(VL.size());
unsigned ScalarTyNumElements = getNumElements(ScalarTy);
APInt ShuffledElements = APInt::getZero(VecTy->getNumElements());
DenseMap<Value *, unsigned> UniqueElements;
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
InstructionCost Cost;
Expand All @@ -11507,7 +11511,8 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
Value *V = VL[I];
// No need to shuffle duplicates for constants.
if ((ForPoisonSrc && isConstant(V)) || isa<UndefValue>(V)) {
ShuffledElements.setBit(I);
ShuffledElements.setBits(I * ScalarTyNumElements,
I * ScalarTyNumElements + ScalarTyNumElements);
ShuffleMask[I] = isa<PoisonValue>(V) ? PoisonMaskElem : I;
continue;
}
Expand All @@ -11520,7 +11525,8 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
}

DuplicateNonConst = true;
ShuffledElements.setBit(I);
ShuffledElements.setBits(I * ScalarTyNumElements,
I * ScalarTyNumElements + ScalarTyNumElements);
ShuffleMask[I] = Res.first->second;
}
if (ForPoisonSrc)
Expand Down
59 changes: 59 additions & 0 deletions llvm/test/Transforms/SLPVectorizer/X86/revec-reduction-logical.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=slp-vectorizer -slp-revec -mtriple=x86_64 -S | FileCheck %s

define i1 @logical_and_icmp_diff_preds(<4 x i32> %x) {
; CHECK-LABEL: @logical_and_icmp_diff_preds(
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, <4 x i32> <i32 1, i32 3, i32 6, i32 0>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i32> [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: ret i1 [[TMP7]]
;
%x0 = extractelement <4 x i32> %x, i32 0
%x1 = extractelement <4 x i32> %x, i32 1
%x2 = extractelement <4 x i32> %x, i32 2
%x3 = extractelement <4 x i32> %x, i32 3
%c0 = icmp ult i32 %x0, 0
%c1 = icmp slt i32 %x1, 0
%c2 = icmp sgt i32 %x2, 0
%c3 = icmp slt i32 %x3, 0
%s1 = select i1 %c0, i1 %c1, i1 false
%s2 = select i1 %s1, i1 %c2, i1 false
%s3 = select i1 %s2, i1 %c3, i1 false
ret i1 %s3
}

define i1 @logical_and_icmp_clamp(<4 x i32> %x) {
; CHECK-LABEL: @logical_and_icmp_clamp(
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP1]], <i32 17, i32 17, i32 17, i32 17, i32 42, i32 42, i32 42, i32 42>
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[TMP1]], <i32 17, i32 17, i32 17, i32 17, i32 42, i32 42, i32 42, i32 42>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP5:%.*]] = freeze <8 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP5]])
; CHECK-NEXT: ret i1 [[TMP6]]
;
%x0 = extractelement <4 x i32> %x, i32 0
%x1 = extractelement <4 x i32> %x, i32 1
%x2 = extractelement <4 x i32> %x, i32 2
%x3 = extractelement <4 x i32> %x, i32 3
%c0 = icmp slt i32 %x0, 42
%c1 = icmp slt i32 %x1, 42
%c2 = icmp slt i32 %x2, 42
%c3 = icmp slt i32 %x3, 42
%d0 = icmp sgt i32 %x0, 17
%d1 = icmp sgt i32 %x1, 17
%d2 = icmp sgt i32 %x2, 17
%d3 = icmp sgt i32 %x3, 17
%s1 = select i1 %c0, i1 %c1, i1 false
%s2 = select i1 %s1, i1 %c2, i1 false
%s3 = select i1 %s2, i1 %c3, i1 false
%s4 = select i1 %s3, i1 %d0, i1 false
%s5 = select i1 %s4, i1 %d1, i1 false
%s6 = select i1 %s5, i1 %d2, i1 false
%s7 = select i1 %s6, i1 %d3, i1 false
ret i1 %s7
}
Loading