Skip to content

Commit d3a76b0

Browse files
jroelofsjroelofs
authored
[llvm][SLPVectorizer] Fix a bad cast assertion (#97621)
Fixes: rdar://128092379
1 parent eaabd76 commit d3a76b0

File tree

2 files changed

+90
-32
lines changed

2 files changed

+90
-32
lines changed

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 51 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -361,12 +361,14 @@ static bool isCommutative(Instruction *I) {
361361
return I->isCommutative();
362362
}
363363

364-
/// \returns inserting index of InsertElement or InsertValue instruction,
365-
/// using Offset as base offset for index.
366-
static std::optional<unsigned> getInsertIndex(const Value *InsertInst,
367-
unsigned Offset = 0) {
364+
template <typename T>
365+
static std::optional<unsigned> getInsertExtractIndex(const Value *Inst,
366+
unsigned Offset) {
367+
static_assert(std::is_same_v<T, InsertElementInst> ||
368+
std::is_same_v<T, ExtractElementInst>,
369+
"unsupported T");
368370
int Index = Offset;
369-
if (const auto *IE = dyn_cast<InsertElementInst>(InsertInst)) {
371+
if (const auto *IE = dyn_cast<T>(Inst)) {
370372
const auto *VT = dyn_cast<FixedVectorType>(IE->getType());
371373
if (!VT)
372374
return std::nullopt;
@@ -379,8 +381,25 @@ static std::optional<unsigned> getInsertIndex(const Value *InsertInst,
379381
Index += CI->getZExtValue();
380382
return Index;
381383
}
384+
return std::nullopt;
385+
}
386+
387+
/// \returns inserting or extracting index of InsertElement, ExtractElement or
388+
/// InsertValue instruction, using Offset as base offset for index.
389+
/// \returns std::nullopt if the index is not an immediate.
390+
static std::optional<unsigned> getElementIndex(const Value *Inst,
391+
unsigned Offset = 0) {
392+
if (auto Index = getInsertExtractIndex<InsertElementInst>(Inst, Offset))
393+
return Index;
394+
if (auto Index = getInsertExtractIndex<ExtractElementInst>(Inst, Offset))
395+
return Index;
396+
397+
int Index = Offset;
398+
399+
const auto *IV = dyn_cast<InsertValueInst>(Inst);
400+
if (!IV)
401+
return std::nullopt;
382402

383-
const auto *IV = cast<InsertValueInst>(InsertInst);
384403
Type *CurrentType = IV->getType();
385404
for (unsigned I : IV->indices()) {
386405
if (const auto *ST = dyn_cast<StructType>(CurrentType)) {
@@ -454,7 +473,7 @@ static SmallBitVector isUndefVector(const Value *V,
454473
Base = II->getOperand(0);
455474
if (isa<T>(II->getOperand(1)))
456475
continue;
457-
std::optional<unsigned> Idx = getInsertIndex(II);
476+
std::optional<unsigned> Idx = getElementIndex(II);
458477
if (!Idx) {
459478
Res.reset();
460479
return Res;
@@ -4718,8 +4737,8 @@ static bool areTwoInsertFromSameBuildVector(
47184737
return false;
47194738
auto *IE1 = VU;
47204739
auto *IE2 = V;
4721-
std::optional<unsigned> Idx1 = getInsertIndex(IE1);
4722-
std::optional<unsigned> Idx2 = getInsertIndex(IE2);
4740+
std::optional<unsigned> Idx1 = getElementIndex(IE1);
4741+
std::optional<unsigned> Idx2 = getElementIndex(IE2);
47234742
if (Idx1 == std::nullopt || Idx2 == std::nullopt)
47244743
return false;
47254744
// Go through the vector operand of insertelement instructions trying to find
@@ -4734,7 +4753,7 @@ static bool areTwoInsertFromSameBuildVector(
47344753
if (IE1 == V && !IE2)
47354754
return V->hasOneUse();
47364755
if (IE1 && IE1 != V) {
4737-
unsigned Idx1 = getInsertIndex(IE1).value_or(*Idx2);
4756+
unsigned Idx1 = getElementIndex(IE1).value_or(*Idx2);
47384757
IsReusedIdx |= ReusedIdx.test(Idx1);
47394758
ReusedIdx.set(Idx1);
47404759
if ((IE1 != VU && !IE1->hasOneUse()) || IsReusedIdx)
@@ -4743,7 +4762,7 @@ static bool areTwoInsertFromSameBuildVector(
47434762
IE1 = dyn_cast_or_null<InsertElementInst>(GetBaseOperand(IE1));
47444763
}
47454764
if (IE2 && IE2 != VU) {
4746-
unsigned Idx2 = getInsertIndex(IE2).value_or(*Idx1);
4765+
unsigned Idx2 = getElementIndex(IE2).value_or(*Idx1);
47474766
IsReusedIdx |= ReusedIdx.test(Idx2);
47484767
ReusedIdx.set(Idx2);
47494768
if ((IE2 != V && !IE2->hasOneUse()) || IsReusedIdx)
@@ -4902,13 +4921,13 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
49024921
IE1, IE2,
49034922
[](InsertElementInst *II) { return II->getOperand(0); }))
49044923
return I1 < I2;
4905-
return getInsertIndex(IE1) < getInsertIndex(IE2);
4924+
return getElementIndex(IE1) < getElementIndex(IE2);
49064925
}
49074926
if (auto *EE1 = dyn_cast<ExtractElementInst>(FirstUserOfPhi1))
49084927
if (auto *EE2 = dyn_cast<ExtractElementInst>(FirstUserOfPhi2)) {
49094928
if (EE1->getOperand(0) != EE2->getOperand(0))
49104929
return I1 < I2;
4911-
return getInsertIndex(EE1) < getInsertIndex(EE2);
4930+
return getElementIndex(EE1) < getElementIndex(EE2);
49124931
}
49134932
return I1 < I2;
49144933
};
@@ -6162,7 +6181,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
61626181
ValueSet SourceVectors;
61636182
for (Value *V : VL) {
61646183
SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
6165-
assert(getInsertIndex(V) != std::nullopt &&
6184+
assert(getElementIndex(V) != std::nullopt &&
61666185
"Non-constant or undef index?");
61676186
}
61686187

@@ -6929,7 +6948,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
69296948
decltype(OrdCompare)>
69306949
Indices(OrdCompare);
69316950
for (int I = 0, E = VL.size(); I < E; ++I) {
6932-
unsigned Idx = *getInsertIndex(VL[I]);
6951+
unsigned Idx = *getElementIndex(VL[I]);
69336952
Indices.emplace(Idx, I);
69346953
}
69356954
OrdersType CurrentOrder(VL.size(), VL.size());
@@ -9308,11 +9327,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
93089327
unsigned NumOfParts = TTI->getNumberOfParts(SrcVecTy);
93099328

93109329
SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
9311-
unsigned OffsetBeg = *getInsertIndex(VL.front());
9330+
unsigned OffsetBeg = *getElementIndex(VL.front());
93129331
unsigned OffsetEnd = OffsetBeg;
93139332
InsertMask[OffsetBeg] = 0;
93149333
for (auto [I, V] : enumerate(VL.drop_front())) {
9315-
unsigned Idx = *getInsertIndex(V);
9334+
unsigned Idx = *getElementIndex(V);
93169335
if (OffsetBeg > Idx)
93179336
OffsetBeg = Idx;
93189337
else if (OffsetEnd < Idx)
@@ -9353,7 +9372,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
93539372
SmallVector<int> PrevMask(InsertVecSz, PoisonMaskElem);
93549373
Mask.swap(PrevMask);
93559374
for (unsigned I = 0; I < NumScalars; ++I) {
9356-
unsigned InsertIdx = *getInsertIndex(VL[PrevMask[I]]);
9375+
unsigned InsertIdx = *getElementIndex(VL[PrevMask[I]]);
93579376
DemandedElts.setBit(InsertIdx);
93589377
IsIdentity &= InsertIdx - OffsetBeg == I;
93599378
Mask[InsertIdx - OffsetBeg] = I;
@@ -10103,8 +10122,8 @@ static bool isFirstInsertElement(const InsertElementInst *IE1,
1010310122
const auto *I2 = IE2;
1010410123
const InsertElementInst *PrevI1;
1010510124
const InsertElementInst *PrevI2;
10106-
unsigned Idx1 = *getInsertIndex(IE1);
10107-
unsigned Idx2 = *getInsertIndex(IE2);
10125+
unsigned Idx1 = *getElementIndex(IE1);
10126+
unsigned Idx2 = *getElementIndex(IE2);
1010810127
do {
1010910128
if (I2 == IE1)
1011010129
return true;
@@ -10113,10 +10132,10 @@ static bool isFirstInsertElement(const InsertElementInst *IE1,
1011310132
PrevI1 = I1;
1011410133
PrevI2 = I2;
1011510134
if (I1 && (I1 == IE1 || I1->hasOneUse()) &&
10116-
getInsertIndex(I1).value_or(Idx2) != Idx2)
10135+
getElementIndex(I1).value_or(Idx2) != Idx2)
1011710136
I1 = dyn_cast<InsertElementInst>(I1->getOperand(0));
1011810137
if (I2 && ((I2 == IE2 || I2->hasOneUse())) &&
10119-
getInsertIndex(I2).value_or(Idx1) != Idx1)
10138+
getElementIndex(I2).value_or(Idx1) != Idx1)
1012010139
I2 = dyn_cast<InsertElementInst>(I2->getOperand(0));
1012110140
} while ((I1 && PrevI1 != I1) || (I2 && PrevI2 != I2));
1012210141
llvm_unreachable("Two different buildvectors not expected.");
@@ -10308,7 +10327,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
1030810327
if (auto *FTy = dyn_cast<FixedVectorType>(VU->getType())) {
1030910328
if (!UsedInserts.insert(VU).second)
1031010329
continue;
10311-
std::optional<unsigned> InsertIdx = getInsertIndex(VU);
10330+
std::optional<unsigned> InsertIdx = getElementIndex(VU);
1031210331
if (InsertIdx) {
1031310332
const TreeEntry *ScalarTE = getTreeEntry(EU.Scalar);
1031410333
auto *It = find_if(
@@ -10334,14 +10353,14 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
1033410353
while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
1033510354
if (IEBase != EU.User &&
1033610355
(!IEBase->hasOneUse() ||
10337-
getInsertIndex(IEBase).value_or(*InsertIdx) == *InsertIdx))
10356+
getElementIndex(IEBase).value_or(*InsertIdx) == *InsertIdx))
1033810357
break;
1033910358
// Build the mask for the vectorized insertelement instructions.
1034010359
if (const TreeEntry *E = getTreeEntry(IEBase)) {
1034110360
VU = IEBase;
1034210361
do {
1034310362
IEBase = cast<InsertElementInst>(Base);
10344-
int Idx = *getInsertIndex(IEBase);
10363+
int Idx = *getElementIndex(IEBase);
1034510364
assert(Mask[Idx] == PoisonMaskElem &&
1034610365
"InsertElementInstruction used already.");
1034710366
Mask[Idx] = Idx;
@@ -12755,7 +12774,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
1275512774
cast<FixedVectorType>(FirstInsert->getType())->getNumElements();
1275612775
const unsigned NumScalars = E->Scalars.size();
1275712776

12758-
unsigned Offset = *getInsertIndex(VL0);
12777+
unsigned Offset = *getElementIndex(VL0);
1275912778
assert(Offset < NumElts && "Failed to find vector index offset");
1276012779

1276112780
// Create shuffle to resize vector
@@ -12773,7 +12792,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
1277312792
Mask.swap(PrevMask);
1277412793
for (unsigned I = 0; I < NumScalars; ++I) {
1277512794
Value *Scalar = E->Scalars[PrevMask[I]];
12776-
unsigned InsertIdx = *getInsertIndex(Scalar);
12795+
unsigned InsertIdx = *getElementIndex(Scalar);
1277712796
IsIdentity &= InsertIdx - Offset == I;
1277812797
Mask[InsertIdx - Offset] = I;
1277912798
}
@@ -12786,7 +12805,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
1278612805
// sequence.
1278712806
InsertElementInst *Ins = cast<InsertElementInst>(VL0);
1278812807
do {
12789-
std::optional<unsigned> InsertIdx = getInsertIndex(Ins);
12808+
std::optional<unsigned> InsertIdx = getElementIndex(Ins);
1279012809
if (!InsertIdx)
1279112810
break;
1279212811
if (InsertMask[*InsertIdx] == PoisonMaskElem)
@@ -13835,7 +13854,7 @@ Value *BoUpSLP::vectorizeTree(
1383513854
}
1383613855
}
1383713856

13838-
std::optional<unsigned> InsertIdx = getInsertIndex(VU);
13857+
std::optional<unsigned> InsertIdx = getElementIndex(VU);
1383913858
if (InsertIdx) {
1384013859
auto *It =
1384113860
find_if(ShuffledInserts, [VU](const ShuffledInsertData &Data) {
@@ -13858,13 +13877,13 @@ Value *BoUpSLP::vectorizeTree(
1385813877
while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
1385913878
if (IEBase != User &&
1386013879
(!IEBase->hasOneUse() ||
13861-
getInsertIndex(IEBase).value_or(Idx) == Idx))
13880+
getElementIndex(IEBase).value_or(Idx) == Idx))
1386213881
break;
1386313882
// Build the mask for the vectorized insertelement instructions.
1386413883
if (const TreeEntry *E = getTreeEntry(IEBase)) {
1386513884
do {
1386613885
IEBase = cast<InsertElementInst>(Base);
13867-
int IEIdx = *getInsertIndex(IEBase);
13886+
int IEIdx = *getElementIndex(IEBase);
1386813887
assert(Mask[IEIdx] == PoisonMaskElem &&
1386913888
"InsertElementInstruction used already.");
1387013889
Mask[IEIdx] = IEIdx;
@@ -17822,7 +17841,7 @@ static void findBuildAggregate_rec(Instruction *LastInsertInst,
1782217841
do {
1782317842
Value *InsertedOperand = LastInsertInst->getOperand(1);
1782417843
std::optional<unsigned> OperandIndex =
17825-
getInsertIndex(LastInsertInst, OperandOffset);
17844+
getElementIndex(LastInsertInst, OperandOffset);
1782617845
if (!OperandIndex)
1782717846
return;
1782817847
if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {

‎llvm/test/Transforms/SLPVectorizer/rdar128092379.ll

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -passes=slp-vectorizer < %s -o - -S | FileCheck %s
3+
4+
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
5+
target triple = "x86_64-apple-macosx15.4.0"
6+
7+
define fastcc i32 @rdar128092379(i8 %index) {
8+
; CHECK-LABEL: define fastcc i32 @rdar128092379(
9+
; CHECK-SAME: i8 [[INDEX:%.*]]) {
10+
; CHECK-NEXT: [[BLOCK:.*]]:
11+
; CHECK-NEXT: [[ZEXT:%.*]] = zext i8 [[INDEX]] to i64
12+
; CHECK-NEXT: [[ZEXT1:%.*]] = zext i8 [[INDEX]] to i64
13+
; CHECK-NEXT: br label %[[BLOCK3:.*]]
14+
; CHECK: [[BLOCK2:.*]]:
15+
; CHECK-NEXT: br label %[[BLOCK3]]
16+
; CHECK: [[BLOCK3]]:
17+
; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, %[[BLOCK2]] ], [ [[ZEXT1]], %[[BLOCK]] ]
18+
; CHECK-NEXT: [[PHI4:%.*]] = phi i64 [ 0, %[[BLOCK2]] ], [ [[ZEXT]], %[[BLOCK]] ]
19+
; CHECK-NEXT: [[EXTRACTELEMENT:%.*]] = extractelement <16 x i32> zeroinitializer, i64 [[PHI4]]
20+
; CHECK-NEXT: [[EXTRACTELEMENT5:%.*]] = extractelement <16 x i32> zeroinitializer, i64 [[PHI]]
21+
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[EXTRACTELEMENT]], [[EXTRACTELEMENT5]]
22+
; CHECK-NEXT: ret i32 [[SUM]]
23+
;
24+
block:
25+
%zext = zext i8 %index to i64
26+
%zext1 = zext i8 %index to i64
27+
br label %block3
28+
29+
block2:
30+
br label %block3
31+
32+
block3:
33+
%phi = phi i64 [ 0, %block2 ], [ %zext1, %block ]
34+
%phi4 = phi i64 [ 0, %block2 ], [ %zext, %block ]
35+
%extractelement = extractelement <16 x i32> zeroinitializer, i64 %phi4
36+
%extractelement5 = extractelement <16 x i32> zeroinitializer, i64 %phi
37+
%sum = add i32 %extractelement, %extractelement5
38+
ret i32 %sum
39+
}

0 commit comments

Comments
 (0)