9
9
//
10
10
// ===----------------------------------------------------------------------===//
11
11
12
- #include " llvm/Transforms/Vectorize.h"
13
12
#include " llvm/ADT/MapVector.h"
14
13
#include " llvm/ADT/PostOrderIterator.h"
15
14
#include " llvm/ADT/SetVector.h"
31
30
#include " llvm/Support/CommandLine.h"
32
31
#include " llvm/Support/Debug.h"
33
32
#include " llvm/Support/raw_ostream.h"
33
+ #include " llvm/Transforms/Vectorize.h"
34
34
35
35
using namespace llvm ;
36
36
@@ -60,9 +60,8 @@ class Vectorizer {
60
60
public:
61
61
Vectorizer (Function &F, AliasAnalysis &AA, DominatorTree &DT,
62
62
ScalarEvolution &SE, TargetTransformInfo &TTI)
63
- : F(F), AA(AA), DT(DT), SE(SE), TTI(TTI),
64
- DL (F.getParent()->getDataLayout()),
65
- Builder(SE.getContext()) {}
63
+ : F(F), AA(AA), DT(DT), SE(SE), TTI(TTI),
64
+ DL (F.getParent()->getDataLayout()), Builder(SE.getContext()) {}
66
65
67
66
bool run ();
68
67
@@ -178,8 +177,8 @@ bool LoadStoreVectorizer::runOnFunction(Function &F) {
178
177
AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults ();
179
178
DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree ();
180
179
ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE ();
181
- TargetTransformInfo &TTI
182
- = getAnalysis<TargetTransformInfoWrapperPass>().getTTI (F);
180
+ TargetTransformInfo &TTI =
181
+ getAnalysis<TargetTransformInfoWrapperPass>().getTTI (F);
183
182
184
183
Vectorizer V (F, AA, DT, SE, TTI);
185
184
return V.run ();
@@ -233,7 +232,7 @@ bool Vectorizer::isConsecutiveAccess(Value *A, Value *B) {
233
232
if (PtrA == PtrB ||
234
233
DL.getTypeStoreSize (PtrATy) != DL.getTypeStoreSize (PtrBTy) ||
235
234
DL.getTypeStoreSize (PtrATy->getScalarType ()) !=
236
- DL.getTypeStoreSize (PtrBTy->getScalarType ()))
235
+ DL.getTypeStoreSize (PtrBTy->getScalarType ()))
237
236
return false ;
238
237
239
238
APInt Size (PtrBitWidth, DL.getTypeStoreSize (PtrATy));
@@ -410,9 +409,9 @@ bool Vectorizer::isVectorizable(ArrayRef<Value *> Chain,
410
409
for (auto I = From, E = To; I != E; ++I, ++Idx) {
411
410
if (isa<LoadInst>(I) || isa<StoreInst>(I)) {
412
411
if (!is_contained (Chain, &*I))
413
- MemoryInstrs.push_back ({ &*I, Idx });
412
+ MemoryInstrs.push_back ({&*I, Idx});
414
413
else
415
- ChainInstrs.push_back ({ &*I, Idx });
414
+ ChainInstrs.push_back ({&*I, Idx});
416
415
} else if (I->mayHaveSideEffects ()) {
417
416
DEBUG (dbgs () << " LSV: Found side-effecting operation: " << *I << ' \n ' );
418
417
return false ;
@@ -446,16 +445,16 @@ bool Vectorizer::isVectorizable(ArrayRef<Value *> Chain,
446
445
Instruction *M1 = cast<Instruction>(VV);
447
446
448
447
if (!AA.isNoAlias (MemoryLocation::get (M0), MemoryLocation::get (M1))) {
449
- DEBUG (
448
+ DEBUG ({
450
449
Value *Ptr0 = getPointerOperand (M0);
451
450
Value *Ptr1 = getPointerOperand (M1);
452
451
453
452
dbgs () << " LSV: Found alias.\n "
454
453
" Aliasing instruction and pointer:\n "
455
- << *V << " aliases " << *Ptr0 << ' \n '
456
- << " Aliased instruction and pointer:\n "
457
- << *VV << " aliases " << *Ptr1 << ' \n '
458
- );
454
+ << *V << " aliases " << *Ptr0 << ' \n '
455
+ << " Aliased instruction and pointer:\n "
456
+ << *VV << " aliases " << *Ptr1 << ' \n '
457
+ } );
459
458
460
459
return false ;
461
460
}
@@ -496,8 +495,7 @@ void Vectorizer::collectInstructions(BasicBlock *BB) {
496
495
continue ;
497
496
498
497
// Make sure all the users of a vector are constant-index extracts.
499
- if (isa<VectorType>(Ty) &&
500
- !all_of (LI->users (), [LI](const User *U) {
498
+ if (isa<VectorType>(Ty) && !all_of (LI->users (), [LI](const User *U) {
501
499
const Instruction *UI = cast<Instruction>(U);
502
500
return isa<ExtractElementInst>(UI) &&
503
501
isa<ConstantInt>(UI->getOperand (1 ));
@@ -530,8 +528,7 @@ void Vectorizer::collectInstructions(BasicBlock *BB) {
530
528
if (TySize > VecRegSize / 2 )
531
529
continue ;
532
530
533
- if (isa<VectorType>(Ty) &&
534
- !all_of (SI->users (), [SI](const User *U) {
531
+ if (isa<VectorType>(Ty) && !all_of (SI->users (), [SI](const User *U) {
535
532
const Instruction *UI = cast<Instruction>(U);
536
533
return isa<ExtractElementInst>(UI) &&
537
534
isa<ConstantInt>(UI->getOperand (1 ));
@@ -685,11 +682,11 @@ bool Vectorizer::vectorizeStoreChain(ArrayRef<Value *> Chain) {
685
682
vectorizeStoreChain (Chain.slice (VF));
686
683
}
687
684
688
- DEBUG (
685
+ DEBUG ({
689
686
dbgs () << " LSV: Stores to vectorize:\n " ;
690
687
for (Value *V : Chain)
691
688
V->dump ();
692
- );
689
+ } );
693
690
694
691
// Check alignment restrictions.
695
692
unsigned Alignment = getAlignment (S0);
@@ -734,8 +731,8 @@ bool Vectorizer::vectorizeStoreChain(ArrayRef<Value *> Chain) {
734
731
if (Extract->getType () != StoreTy->getScalarType ())
735
732
Extract = Builder.CreateBitCast (Extract, StoreTy->getScalarType ());
736
733
737
- Value *Insert = Builder. CreateInsertElement (Vec, Extract,
738
- Builder.getInt32 (NewIdx));
734
+ Value *Insert =
735
+ Builder. CreateInsertElement (Vec, Extract, Builder.getInt32 (NewIdx));
739
736
Vec = Insert;
740
737
}
741
738
}
@@ -744,16 +741,17 @@ bool Vectorizer::vectorizeStoreChain(ArrayRef<Value *> Chain) {
744
741
StoreInst *Store = cast<StoreInst>(Chain[I]);
745
742
Value *Extract = Store->getValueOperand ();
746
743
if (Extract->getType () != StoreTy->getScalarType ())
747
- Extract = Builder.CreateBitOrPointerCast (Extract, StoreTy->getScalarType ());
744
+ Extract =
745
+ Builder.CreateBitOrPointerCast (Extract, StoreTy->getScalarType ());
748
746
749
- Value *Insert = Builder. CreateInsertElement (Vec, Extract,
750
- Builder.getInt32 (I));
747
+ Value *Insert =
748
+ Builder. CreateInsertElement (Vec, Extract, Builder.getInt32 (I));
751
749
Vec = Insert;
752
750
}
753
751
}
754
752
755
753
Value *Bitcast =
756
- Builder.CreateBitCast (S0->getPointerOperand (), VecTy->getPointerTo (AS));
754
+ Builder.CreateBitCast (S0->getPointerOperand (), VecTy->getPointerTo (AS));
757
755
StoreInst *SI = cast<StoreInst>(Builder.CreateStore (Vec, Bitcast));
758
756
propagateMetadata (SI, Chain);
759
757
SI->setAlignment (Alignment);
@@ -779,7 +777,6 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
779
777
DL.getTypeSizeInBits (LoadTy));
780
778
break ;
781
779
}
782
-
783
780
}
784
781
785
782
unsigned Sz = DL.getTypeSizeInBits (LoadTy);
@@ -795,7 +792,8 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
795
792
// TODO: Should size constraint be a target hook?
796
793
unsigned SzInBytes = (Sz / 8 ) * ChainSize;
797
794
if (SzInBytes > 2 && SzInBytes % 4 != 0 ) {
798
- DEBUG (dbgs () << " LSV: Size should be 1B, 2B or multiple of 4B. Splitting.\n " );
795
+ DEBUG (dbgs () << " LSV: Size should be 1B, 2B "
796
+ " or multiple of 4B. Splitting.\n " );
799
797
if (SzInBytes == 3 )
800
798
return vectorizeLoadChain (Chain.slice (0 , ChainSize - 1 ));
801
799
auto Chains = splitOddVectorElts (Chain, Sz);
@@ -840,11 +838,11 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
840
838
}
841
839
}
842
840
843
- DEBUG (
841
+ DEBUG ({
844
842
dbgs () << " LSV: Loads to vectorize:\n " ;
845
843
for (Value *V : Chain)
846
844
V->dump ();
847
- );
845
+ } );
848
846
849
847
BasicBlock::iterator First, Last;
850
848
std::tie (First, Last) = getBoundaryInstrs (Chain);
@@ -856,7 +854,7 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
856
854
Builder.SetInsertPoint (&*Last);
857
855
858
856
Value *Bitcast =
859
- Builder.CreateBitCast (L0->getPointerOperand (), VecTy->getPointerTo (AS));
857
+ Builder.CreateBitCast (L0->getPointerOperand (), VecTy->getPointerTo (AS));
860
858
861
859
LoadInst *LI = cast<LoadInst>(Builder.CreateLoad (Bitcast));
862
860
propagateMetadata (LI, Chain);
@@ -875,8 +873,8 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
875
873
Value *V = Builder.CreateExtractElement (LI, Builder.getInt32 (NewIdx));
876
874
Instruction *Extracted = cast<Instruction>(V);
877
875
if (Extracted->getType () != UI->getType ())
878
- Extracted =
879
- cast<Instruction>( Builder.CreateBitCast (Extracted, UI->getType ()));
876
+ Extracted = cast<Instruction>(
877
+ Builder.CreateBitCast (Extracted, UI->getType ()));
880
878
881
879
// Replace the old instruction.
882
880
UI->replaceAllUsesWith (Extracted);
@@ -898,8 +896,8 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
898
896
Instruction *Extracted = cast<Instruction>(V);
899
897
Instruction *UI = cast<Instruction>(Chain[I]);
900
898
if (Extracted->getType () != UI->getType ()) {
901
- Extracted =
902
- cast<Instruction>( Builder.CreateBitOrPointerCast (Extracted, UI->getType ()));
899
+ Extracted = cast<Instruction>(
900
+ Builder.CreateBitOrPointerCast (Extracted, UI->getType ()));
903
901
}
904
902
905
903
// Replace the old instruction.
0 commit comments