|
25 | 25 | #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
|
26 | 26 |
|
27 | 27 | #include "VPlan.h"
|
| 28 | +#include "llvm/ADT/SmallSet.h" |
28 | 29 | #include "llvm/Support/InstructionCost.h"
|
29 | 30 |
|
30 | 31 | namespace llvm {
|
@@ -217,6 +218,16 @@ struct VectorizationFactor {
|
217 | 218 | }
|
218 | 219 | };
|
219 | 220 |
|
| 221 | +/// ElementCountComparator creates a total ordering for ElementCount |
| 222 | +/// for the purposes of using it in a set structure. |
| 223 | +struct ElementCountComparator { |
| 224 | + bool operator()(const ElementCount &LHS, const ElementCount &RHS) const { |
| 225 | + return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) < |
| 226 | + std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue()); |
| 227 | + } |
| 228 | +}; |
| 229 | +using ElementCountSet = SmallSet<ElementCount, 16, ElementCountComparator>; |
| 230 | + |
220 | 231 | /// A class that represents two vectorization factors (initialized with 0 by
|
221 | 232 | /// default). One for fixed-width vectorization and one for scalable
|
222 | 233 | /// vectorization. This can be used by the vectorizer to choose from a range of
|
@@ -280,6 +291,9 @@ class LoopVectorizationPlanner {
|
280 | 291 |
|
281 | 292 | SmallVector<VPlanPtr, 4> VPlans;
|
282 | 293 |
|
| 294 | + /// Profitable vector factors. |
| 295 | + SmallVector<VectorizationFactor, 8> ProfitableVFs; |
| 296 | + |
283 | 297 | /// A builder used to construct the current plan.
|
284 | 298 | VPBuilder Builder;
|
285 | 299 |
|
@@ -342,6 +356,12 @@ class LoopVectorizationPlanner {
|
342 | 356 | /// Check if the number of runtime checks exceeds the threshold.
|
343 | 357 | bool requiresTooManyRuntimeChecks() const;
|
344 | 358 |
|
| 359 | + /// \return The most profitable vectorization factor and the cost of that VF |
| 360 | + /// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if |
| 361 | + /// epilogue vectorization is not supported for the loop. |
| 362 | + VectorizationFactor |
| 363 | + selectEpilogueVectorizationFactor(const ElementCount MaxVF); |
| 364 | + |
345 | 365 | protected:
|
346 | 366 | /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
|
347 | 367 | /// according to the information gathered by Legal when it checked if it is
|
@@ -376,6 +396,20 @@ class LoopVectorizationPlanner {
|
376 | 396 | void adjustRecipesForReductions(VPBasicBlock *LatchVPBB, VPlanPtr &Plan,
|
377 | 397 | VPRecipeBuilder &RecipeBuilder,
|
378 | 398 | ElementCount MinVF);
|
| 399 | + |
| 400 | + /// \return The most profitable vectorization factor and the cost of that VF. |
| 401 | + /// This method checks every VF in \p CandidateVFs. |
| 402 | + VectorizationFactor |
| 403 | + selectVectorizationFactor(const ElementCountSet &CandidateVFs); |
| 404 | + |
| 405 | + /// Returns true if the per-lane cost of VectorizationFactor A is lower than |
| 406 | + /// that of B. |
| 407 | + bool isMoreProfitable(const VectorizationFactor &A, |
| 408 | + const VectorizationFactor &B) const; |
| 409 | + |
| 410 | + /// Determines if we have the infrastructure to vectorize the loop and its |
| 411 | + /// epilogue, assuming the main loop is vectorized by \p VF. |
| 412 | + bool isCandidateForEpilogueVectorization(const ElementCount VF) const; |
379 | 413 | };
|
380 | 414 |
|
381 | 415 | } // namespace llvm
|
|
0 commit comments