!fixup address latest comments, thanks

Author: fhahn

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -8492,7 +8492,7 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
     Builder.insert(VectorPtr);
     Ptr = VectorPtr;
   }
-  auto Metadata = getMetadataToPropagate(I);
+  auto Metadata = getRecipeMetadata(I);
   if (LoadInst *Load = dyn_cast<LoadInst>(I))
     return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
                                  Metadata, I->getDebugLoc());
@@ -8872,7 +8872,8 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
   assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
           (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
          "Should not predicate a uniform recipe");
-  auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask, getMetadataToPropagate(I));
+  auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
+                                       getRecipeMetadata(I));
   return Recipe;
 }
 
@@ -8993,18 +8994,18 @@ bool VPRecipeBuilder::getScaledReductions(
   return false;
 }
 
-SmallVector<std::pair<unsigned, MDNode *>>
-VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
+VPIRMetadata VPRecipeBuilder::getRecipeMetadata(Instruction *I) const {
   SmallVector<std::pair<unsigned, MDNode *>> Metadata;
   ::getMetadataToPropagate(I, Metadata);
-  if (LVer && isa<LoadInst, StoreInst>(I)) {
-    const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
-    if (AliasScopeMD)
-      Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
-    if (NoAliasMD)
-      Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
-  }
-  return Metadata;
+  if (!LVer || !isa<LoadInst, StoreInst>(I))
+    return {};
+
+  const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
+  if (AliasScopeMD)
+    Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
+  if (NoAliasMD)
+    Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
+  return {Metadata};
 }
 
 VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
@@ -9576,7 +9577,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
         if (Legal->isInvariantStoreOfReduction(SI)) {
           auto *Recipe = new VPReplicateRecipe(
               SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
-              RecipeBuilder.getMetadataToPropagate(SI));
+              RecipeBuilder.getRecipeMetadata(SI));
           Recipe->insertBefore(*MiddleVPBB, MBIP);
         }
         R.eraseFromParent();

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

@@ -241,8 +241,7 @@ class VPRecipeBuilder {
 
   /// Returns the metatadata that can be preserved from the original instruction
   /// \p I, including noalias metadata guaranteed by runtime checks.
-  SmallVector<std::pair<unsigned, MDNode *>>
-  getMetadataToPropagate(Instruction *I) const;
+  VPIRMetadata getRecipeMetadata(Instruction *I) const;
 };
 } // end namespace llvm
 

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1194,24 +1194,22 @@ struct VPIRPhi : public VPIRInstruction {
 #endif
 };
 
-using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
-
 /// Helper to manage IR metadata for recipes. It filters out metadata that
 /// cannot be propagated.
 class VPIRMetadata {
   SmallVector<std::pair<unsigned, MDNode *>> Metadata;
 
 protected:
-  VPIRMetadata() {}
   VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
-  VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
 
 public:
+  VPIRMetadata() {}
+  VPIRMetadata(ArrayRef<std::pair<unsigned, MDNode *>> Metadata)
+      : Metadata(Metadata) {}
+  VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
+
   /// Add all metadata to \p I.
   void applyMetadata(Instruction &I) const;
-
-  /// Return the IR metadata.
-  MDArrayRef getMetadata() const { return Metadata; }
 };
 
 /// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2482,7 +2480,8 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
 
 public:
   VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
-                    bool IsUniform, VPValue *Mask = nullptr, ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
+                    bool IsUniform, VPValue *Mask = nullptr,
+                    VPIRMetadata Metadata = {})
       : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
         VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
     if (Mask)
@@ -2492,9 +2491,9 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
   ~VPReplicateRecipe() override = default;
 
   VPReplicateRecipe *clone() override {
-    auto *Copy = new VPReplicateRecipe(
-        getUnderlyingInstr(), operands(), IsUniform,
-        isPredicated() ? getMask() : nullptr, getMetadata());
+    auto *Copy =
+        new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
+                              isPredicated() ? getMask() : nullptr, *this);
     Copy->transferFlags(*this);
     return Copy;
   }
@@ -2654,8 +2653,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
 
   VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
                       std::initializer_list<VPValue *> Operands,
-                      bool Consecutive, bool Reverse, MDArrayRef Metadata,
-                      DebugLoc DL)
+                      bool Consecutive, bool Reverse,
+                      const VPIRMetadata &Metadata, DebugLoc DL)
       : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
         Consecutive(Consecutive), Reverse(Reverse) {
     assert((Consecutive || !Reverse) && "Reverse implies consecutive");
@@ -2714,8 +2713,8 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
 /// optional mask.
 struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
   VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
-                    bool Consecutive, bool Reverse, MDArrayRef Metadata,
-                    DebugLoc DL)
+                    bool Consecutive, bool Reverse,
+                    const VPIRMetadata &Metadata, DebugLoc DL)
       : VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
                             Reverse, Metadata, DL),
         VPValue(this, &Load) {
@@ -2724,7 +2723,7 @@ struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
 
   VPWidenLoadRecipe *clone() override {
     return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
-                                 getMask(), Consecutive, Reverse, getMetadata(),
+                                 getMask(), Consecutive, Reverse, *this,
                                  getDebugLoc());
   }
 
@@ -2756,7 +2755,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
   VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
       : VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
                             {L.getAddr(), &EVL}, L.isConsecutive(),
-                            L.isReverse(), L.getMetadata(), L.getDebugLoc()),
+                            L.isReverse(), L, L.getDebugLoc()),
         VPValue(this, &getIngredient()) {
     setMask(Mask);
   }
@@ -2794,7 +2793,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
 struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
   VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
                      VPValue *Mask, bool Consecutive, bool Reverse,
-                     MDArrayRef Metadata, DebugLoc DL)
+                     const VPIRMetadata &Metadata, DebugLoc DL)
       : VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
                             Consecutive, Reverse, Metadata, DL) {
     setMask(Mask);
@@ -2803,7 +2802,7 @@ struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
   VPWidenStoreRecipe *clone() override {
     return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
                                   getStoredValue(), getMask(), Consecutive,
-                                  Reverse, getMetadata(), getDebugLoc());
+                                  Reverse, *this, getDebugLoc());
   }
 
   VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2837,7 +2836,7 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
   VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
       : VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
                             {S.getAddr(), S.getStoredValue(), &EVL},
-                            S.isConsecutive(), S.isReverse(), S.getMetadata(),
+                            S.isConsecutive(), S.isReverse(), S,
                             S.getDebugLoc()) {
     setMask(Mask);
   }

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -2751,7 +2751,6 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
   } else {
     NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");
   }
-  // Add metadata to the load, but setVectorValue to the reverse shuffle.
   applyMetadata(*cast<Instruction>(NewLI));
   if (Reverse)
     NewLI = Builder.CreateVectorReverse(NewLI, "reverse");

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -82,15 +82,15 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
           ::getMetadataToPropagate(Inst, Metadata);
           NewRecipe = new VPWidenLoadRecipe(
               *Load, Ingredient.getOperand(0), nullptr /*Mask*/,
-              false /*Consecutive*/, false /*Reverse*/, Metadata,
+              false /*Consecutive*/, false /*Reverse*/, {Metadata},
               Ingredient.getDebugLoc());
         } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
           SmallVector<std::pair<unsigned, MDNode *>> Metadata;
           ::getMetadataToPropagate(Inst, Metadata);
           NewRecipe = new VPWidenStoreRecipe(
               *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
               nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/,
-              Metadata, Ingredient.getDebugLoc());
+              {Metadata}, Ingredient.getDebugLoc());
         } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
           NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
         } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
@@ -187,9 +187,9 @@ static bool sinkScalarOperands(VPlan &Plan) {
         // TODO: Handle converting to uniform recipes as separate transform,
         // then cloning should be sufficient here.
         Instruction *I = SinkCandidate->getUnderlyingInstr();
-        Clone = new VPReplicateRecipe(
-            I, SinkCandidate->operands(), true, /*Mask*/ nullptr,
-            cast<VPReplicateRecipe>(SinkCandidate)->getMetadata());
+        Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
+                                      /*Mask*/ nullptr,
+                                      *cast<VPReplicateRecipe>(SinkCandidate));
         // TODO: add ".cloned" suffix to name of Clone's VPValue.
       } else {
         Clone = SinkCandidate->clone();
@@ -351,7 +351,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
   auto *RecipeWithoutMask = new VPReplicateRecipe(
       PredRecipe->getUnderlyingInstr(),
       make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())),
-      PredRecipe->isUniform(), /*Mask*/ nullptr, PredRecipe->getMetadata());
+      PredRecipe->isUniform(), /*Mask*/ nullptr, *PredRecipe);
   auto *Pred =
       Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
 
@@ -2810,7 +2810,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
     // process one original iteration.
     auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(),
                                     WideLoad->operands(), /*IsUniform*/ true,
-                                    /*Mask*/ nullptr, WideLoad->getMetadata());
+                                    /*Mask*/ nullptr, *WideLoad);
     N->insertBefore(WideLoad);
     return N;
   };