[VPlan] Manage Sentinel value for FindLastIV in VPlan. #142291

fhahn · 2025-05-31T21:19:07Z

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan.

llvmbot · 2025-05-31T21:19:40Z

@llvm/pr-subscribers-llvm-transforms

@llvm/pr-subscribers-vectorizers

Author: Florian Hahn (fhahn)

Changes

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan.

Full diff: https://github.com/llvm/llvm-project/pull/142291.diff

5 Files Affected:

(modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+20-8)
(modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+19)
(modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+4-4)
(modified) llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp (+1-1)
(modified) llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll (+1-1)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e9ace195684b3..6d6ed5d7d2a55 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7517,9 +7517,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
     using namespace llvm::PatternMatch;
     Value *Cmp, *OrigResumeV, *CmpOp;
     bool IsExpectedPattern =
-        match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
-                                        m_Specific(RdxDesc.getSentinelValue()),
-                                        m_Value(OrigResumeV))) &&
+        match(MainResumeValue,
+              m_Select(
+                  m_OneUse(m_Value(Cmp)),
+                  m_Specific(EpiRedResult->getOperand(2)->getLiveInIRValue()),
+                  m_Value(OrigResumeV))) &&
         (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
                                    m_Value(CmpOp))) &&
          (match(CmpOp,
@@ -9509,9 +9511,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
             RdxDesc.getRecurrenceKind())) {
       VPValue *Start = PhiR->getStartValue();
-      FinalReductionResult =
-          Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
-                               {PhiR, Start, NewExitingVPV}, ExitDL);
+      FinalReductionResult = Builder.createNaryOp(
+          VPInstruction::ComputeFindLastIVResult,
+          {PhiR, Start, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()),
+           NewExitingVPV},
+          ExitDL);
     } else {
       VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind(
                             RdxDesc.getRecurrenceKind())
@@ -10040,6 +10044,14 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
     Value *ResumeV = nullptr;
     // TODO: Move setting of resume values to prepareToExecute.
     if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+      auto *RdxResult =
+          cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
+            auto *VPI = dyn_cast<VPInstruction>(U);
+            return VPI &&
+                   (VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
+                    VPI->getOpcode() == VPInstruction::ComputeFindLastIVResult);
+          }));
+
       ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
                     ->getIncomingValueForBlock(L->getLoopPreheader());
       const RecurrenceDescriptor &RdxDesc =
@@ -10068,8 +10080,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
         IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
         Value *Cmp = Builder.CreateICmpEQ(
             ResumeV, ToFrozen[RdxDesc.getRecurrenceStartValue()]);
-        ResumeV =
-            Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
+        ResumeV = Builder.CreateSelect(
+            Cmp, RdxResult->getOperand(2)->getLiveInIRValue(), ResumeV);
       }
     } else {
       // Retrieve the induction resume values for wide inductions from
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f2a7f16e19a79..6b257a56c88ab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -318,6 +318,25 @@ m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
       {Op0, Op1, Op2});
 }
 
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+          unsigned Opcode, bool Commutative, typename... RecipeTys>
+using Recipe4Op_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t, Op3_t>,
+                                     Opcode, Commutative, RecipeTys...>;
+
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+          unsigned Opcode>
+using VPInstruction4Op_match =
+    Recipe4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode, /*Commutative*/ false,
+                    VPInstruction>;
+
+template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t,
+          typename Op3_t>
+inline VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>
+m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2,
+                const Op3_t &Op3) {
+  return VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>(
+      {Op0, Op1, Op2, Op3});
+}
 template <typename Op0_t>
 inline UnaryVPInstruction_match<Op0_t, VPInstruction::Not>
 m_Not(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a4831ea7c11f7..8bca0265b406a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -618,16 +618,16 @@ Value *VPInstruction::generate(VPTransformState &State) {
 
     // The recipe's operands are the reduction phi, followed by one operand for
     // each part of the reduction.
-    unsigned UF = getNumOperands() - 2;
-    Value *ReducedPartRdx = State.get(getOperand(2));
+    unsigned UF = getNumOperands() - 3;
+    Value *ReducedPartRdx = State.get(getOperand(3));
     for (unsigned Part = 1; Part < UF; ++Part) {
       ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
-                                      State.get(getOperand(2 + Part)));
+                                      State.get(getOperand(3 + Part)));
     }
 
     return createFindLastIVReduction(Builder, ReducedPartRdx,
                                      State.get(getOperand(1), true),
-                                     RdxDesc.getSentinelValue());
+                                     getOperand(2)->getLiveInIRValue());
   }
   case VPInstruction::ComputeReductionResult: {
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index e1fb3d476c58d..cba8b5f1454a3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -330,7 +330,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
     if (match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
                       m_VPValue(), m_VPValue(Op1))) ||
         match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
-                      m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
+                      m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
       addUniformForAllParts(cast<VPInstruction>(&R));
       for (unsigned Part = 1; Part != UF; ++Part)
         R.addOperand(getValueForPart(Op1, Part));
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index c22ecda266fde..96aa1ce4028e9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -237,7 +237,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<%cond>
+; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
 ; CHECK-NEXT:   EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
 ; CHECK-NEXT:   EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<%cmp.n>

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm#141860 llvm#141932 llvm#142290 llvm#142291

david-arm

LGTM! Seems reasonable and looks like we're closer to removing all dependencies on RdxDesc in the vplan recipes code.

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm#141860 llvm#141932 llvm#142290 llvm#142291

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan.

…42291) Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan. PR: llvm/llvm-project#142291

ayalz

Post-commit nits.

ayalz · 2025-06-13T20:44:57Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7266,9 +7266,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
    using namespace llvm::PatternMatch;
    Value *Cmp, *OrigResumeV, *CmpOp;


Value *Sentinel = ‏EpiRedResult->getOperand(2)->getLiveInIRValue();

Perhaps worth a Value *vputils::getSentinel(VPInstruction*)?

Done, thanks!

ayalz · 2025-06-13T20:47:05Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -9235,9 +9237,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
            RdxDesc.getRecurrenceKind())) {
      VPValue *Start = PhiR->getStartValue();


VPValue *Sentinel = ...

Done thanks

ayalz · 2025-06-13T20:48:01Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -9825,8 +9829,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
        BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
        IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
        Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);


Value *Sentinel = ...

Done thanks

ayalz · 2025-06-13T20:49:55Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -645,16 +645,16 @@ Value *VPInstruction::generate(VPTransformState &State) {

    // The recipe's operands are the reduction phi, followed by one operand for
    // each part of the reduction.


Comment needs updating? Operands better be explaining under /// somewhere?

Done thanks

ayalz · 2025-06-13T20:51:59Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

    for (unsigned Part = 1; Part < UF; ++Part) {
      ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
-                                      State.get(getOperand(2 + Part)));
+                                      State.get(getOperand(3 + Part)));
    }



Value *Sentinel = ...

Done, also for Start

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from #142291.

fhahn

Thanks @ayalz! should be addressed in 732ebf8

fhahn · 2025-06-14T09:31:19Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -9235,9 +9237,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
            RdxDesc.getRecurrenceKind())) {
      VPValue *Start = PhiR->getStartValue();


Done thanks

fhahn · 2025-06-14T09:32:11Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7266,9 +7266,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
    using namespace llvm::PatternMatch;
    Value *Cmp, *OrigResumeV, *CmpOp;


Done, thanks!

fhahn · 2025-06-14T09:34:40Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -9825,8 +9829,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
        BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
        IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
        Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);


Done thanks

fhahn · 2025-06-14T09:36:59Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -645,16 +645,16 @@ Value *VPInstruction::generate(VPTransformState &State) {

    // The recipe's operands are the reduction phi, followed by one operand for
    // each part of the reduction.


Done thanks

fhahn · 2025-06-14T09:37:22Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

    for (unsigned Part = 1; Part < UF; ++Part) {
      ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
-                                      State.get(getOperand(2 + Part)));
+                                      State.get(getOperand(3 + Part)));
    }



Done, also for Start

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from llvm/llvm-project#142291.

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan. PR: llvm#142291

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from llvm#142291.

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm#141860 llvm#141932 llvm#142290 llvm#142291

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan. PR: llvm#142291

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from llvm#142291.

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm#141860 llvm#141932 llvm#142290 llvm#142291

fhahn requested review from Mel-Chen, ayalz and aniragil May 31, 2025 21:19

llvmbot added vectorizers llvm:transforms labels May 31, 2025

fhahn force-pushed the vplan-findlastiv-sentinel-value branch from 0dde7da to 10f9da0 Compare June 1, 2025 21:17

This was referenced Jun 1, 2025

[VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). #142322

Open

[VPlan] Add ComputeAnyOfResult VPInstruction (NFC) #141932

Merged

fhahn force-pushed the vplan-findlastiv-sentinel-value branch from 10f9da0 to 413dc5a Compare June 3, 2025 14:13

fhahn force-pushed the vplan-findlastiv-sentinel-value branch from 413dc5a to fb8a70b Compare June 8, 2025 11:59

david-arm approved these changes Jun 9, 2025

View reviewed changes

[VPlan] Manage Sentinel value for FindLastIV in VPlan.

abc7c36

Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan.

fhahn force-pushed the vplan-findlastiv-sentinel-value branch from fb8a70b to abc7c36 Compare June 9, 2025 21:55

Merge branch 'main' into vplan-findlastiv-sentinel-value

abb0072

fhahn merged commit f688480 into llvm:main Jun 13, 2025
7 checks passed

fhahn deleted the vplan-findlastiv-sentinel-value branch June 13, 2025 18:17

ayalz reviewed Jun 13, 2025

View reviewed changes

fhahn added a commit that referenced this pull request Jun 14, 2025

[VPlan] Address post-commit comments for f688480.

732ebf8

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from #142291.

fhahn commented Jun 14, 2025

View reviewed changes

tomtor pushed a commit to tomtor/llvm-project that referenced this pull request Jun 14, 2025

[VPlan] Address post-commit comments for f688480.

3b1cd6a

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from llvm#142291.

akuhlens pushed a commit to akuhlens/llvm-project that referenced this pull request Jun 24, 2025

[VPlan] Address post-commit comments for f688480.

0574ea1

Assign sentinel value to named variable to clarify naming and update comments. Addresses post-commit comments from llvm#142291.

		@@ -7266,9 +7266,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
		using namespace llvm::PatternMatch;
		Value Cmp, OrigResumeV, *CmpOp;

		@@ -645,16 +645,16 @@ Value *VPInstruction::generate(VPTransformState &State) {

		// The recipe's operands are the reduction phi, followed by one operand for
		// each part of the reduction.

[VPlan] Manage Sentinel value for FindLastIV in VPlan. #142291

[VPlan] Manage Sentinel value for FindLastIV in VPlan. #142291

Uh oh!

Conversation

fhahn commented May 31, 2025

Uh oh!

llvmbot commented May 31, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

david-arm left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

ayalz left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

fhahn left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

llvmbot commented May 31, 2025 •

edited

Loading