@@ -577,13 +577,6 @@ class InnerLoopVectorizer {
577
577
// / able to vectorize with strict in-order reductions for the given RdxDesc.
578
578
bool useOrderedReductions (const RecurrenceDescriptor &RdxDesc);
579
579
580
- // / Create a broadcast instruction. This method generates a broadcast
581
- // / instruction (shuffle) for loop invariant values and for the induction
582
- // / value. If this is the induction variable then we extend it to N, N+1, ...
583
- // / this is needed because each iteration in the loop corresponds to a SIMD
584
- // / element.
585
- virtual Value *getBroadcastInstrs (Value *V);
586
-
587
580
// Returns the resume value (bc.merge.rdx) for a reduction as
588
581
// generated by fixReduction.
589
582
PHINode *getReductionResumeValue (const RecurrenceDescriptor &RdxDesc);
@@ -820,9 +813,6 @@ class InnerLoopUnroller : public InnerLoopVectorizer {
820
813
ElementCount::getFixed (1 ),
821
814
ElementCount::getFixed(1 ), UnrollFactor, LVL, CM,
822
815
BFI, PSI, Check) {}
823
-
824
- private:
825
- Value *getBroadcastInstrs (Value *V) override ;
826
816
};
827
817
828
818
// / Encapsulate information regarding vectorization of a loop and its epilogue.
@@ -2266,25 +2256,6 @@ static void collectSupportedLoops(Loop &L, LoopInfo *LI,
2266
2256
// LoopVectorizationCostModel and LoopVectorizationPlanner.
2267
2257
// ===----------------------------------------------------------------------===//
2268
2258
2269
- Value *InnerLoopVectorizer::getBroadcastInstrs (Value *V) {
2270
- // We need to place the broadcast of invariant variables outside the loop,
2271
- // but only if it's proven safe to do so. Else, broadcast will be inside
2272
- // vector loop body.
2273
- Instruction *Instr = dyn_cast<Instruction>(V);
2274
- bool SafeToHoist = OrigLoop->isLoopInvariant (V) &&
2275
- (!Instr ||
2276
- DT->dominates (Instr->getParent (), LoopVectorPreHeader));
2277
- // Place the code for broadcasting invariant variables in the new preheader.
2278
- IRBuilder<>::InsertPointGuard Guard (Builder);
2279
- if (SafeToHoist)
2280
- Builder.SetInsertPoint (LoopVectorPreHeader->getTerminator ());
2281
-
2282
- // Broadcast the scalar into all locations in the vector.
2283
- Value *Shuf = Builder.CreateVectorSplat (VF, V, " broadcast" );
2284
-
2285
- return Shuf;
2286
- }
2287
-
2288
2259
// / This function adds
2289
2260
// / (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
2290
2261
// / to each vector element of Val. The sequence starts at StartIndex.
@@ -7849,8 +7820,6 @@ void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
7849
7820
}
7850
7821
#endif
7851
7822
7852
- Value *InnerLoopUnroller::getBroadcastInstrs (Value *V) { return V; }
7853
-
7854
7823
// ===--------------------------------------------------------------------===//
7855
7824
// EpilogueVectorizerMainLoop
7856
7825
// ===--------------------------------------------------------------------===//
@@ -9881,9 +9850,29 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part) {
9881
9850
if (hasVectorValue (Def, Part))
9882
9851
return Data.PerPartOutput [Def][Part];
9883
9852
9853
+ auto GetBroadcastInstrs = [this , Def](Value *V) {
9854
+ bool SafeToHoist = Def->isDefinedOutsideVectorRegions ();
9855
+ if (VF.isScalar ())
9856
+ return V;
9857
+ // Place the code for broadcasting invariant variables in the new preheader.
9858
+ IRBuilder<>::InsertPointGuard Guard (Builder);
9859
+ if (SafeToHoist) {
9860
+ BasicBlock *LoopVectorPreHeader = CFG.VPBB2IRBB [cast<VPBasicBlock>(
9861
+ Plan->getVectorLoopRegion ()->getSinglePredecessor ())];
9862
+ if (LoopVectorPreHeader)
9863
+ Builder.SetInsertPoint (LoopVectorPreHeader->getTerminator ());
9864
+ }
9865
+
9866
+ // Place the code for broadcasting invariant variables in the new preheader.
9867
+ // Broadcast the scalar into all locations in the vector.
9868
+ Value *Shuf = Builder.CreateVectorSplat (VF, V, " broadcast" );
9869
+
9870
+ return Shuf;
9871
+ };
9872
+
9884
9873
if (!hasScalarValue (Def, {Part, 0 })) {
9885
9874
Value *IRV = Def->getLiveInIRValue ();
9886
- Value *B = ILV-> getBroadcastInstrs (IRV);
9875
+ Value *B = GetBroadcastInstrs (IRV);
9887
9876
set (Def, B, Part);
9888
9877
return B;
9889
9878
}
@@ -9930,7 +9919,7 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part) {
9930
9919
// State, we will only generate the insertelements once.
9931
9920
Value *VectorValue = nullptr ;
9932
9921
if (IsUniform) {
9933
- VectorValue = ILV-> getBroadcastInstrs (ScalarValue);
9922
+ VectorValue = GetBroadcastInstrs (ScalarValue);
9934
9923
set (Def, VectorValue, Part);
9935
9924
} else {
9936
9925
// Initialize packing with insertelements to start from undef.
0 commit comments