Fixups

Author: MacDue

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -1468,6 +1468,15 @@ class TargetTransformInfo {
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index = -1) const;
 
+  /// \return The expected of aggregate inserts and extracts. Use an empty
+  /// ArrayRef to indicate that there is no information on the indices. This is
+  /// used when the instruction is not available; a typical use case is to
+  /// provision the cost of vectorization/scalarization in vectorizer passes.
+  InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+                                            TTI::TargetCostKind CostKind,
+                                            ArrayRef<unsigned> Indices,
+                                            Value *AggDef = nullptr) const;
+
   /// \return The cost of replication shuffle of \p VF elements typed \p EltTy
   /// \p ReplicationFactor times.
   ///
@@ -2200,6 +2209,11 @@ class TargetTransformInfo::Concept {
                             const APInt &DemandedDstElts,
                             TTI::TargetCostKind CostKind) = 0;
 
+  virtual InstructionCost
+  getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+                            TTI::TargetCostKind CostKind,
+                            ArrayRef<unsigned> Indices, Value *AggDef) = 0;
+
   virtual InstructionCost
   getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                   unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -2921,6 +2935,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
                                           DemandedDstElts, CostKind);
   }
+  InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+                                            TTI::TargetCostKind CostKind,
+                                            ArrayRef<unsigned> Indices,
+                                            Value *AggDef) override {
+    return Impl.getInsertExtractValueCost(Opcode, AggType, CostKind, Indices,
+                                          AggDef);
+  }
   InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                   unsigned AddressSpace,
                                   TTI::TargetCostKind CostKind,

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

@@ -745,6 +745,13 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
+  InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+                                            TTI::TargetCostKind CostKind,
+                                            ArrayRef<unsigned> Indices,
+                                            Value *AggDef) const {
+    return 0;
+  }
+
   InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                                   unsigned AddressSpace,
                                   TTI::TargetCostKind CostKind,

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -1113,6 +1113,18 @@ TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
   return Cost;
 }
 
+InstructionCost TargetTransformInfo::getInsertExtractValueCost(
+    unsigned Opcode, Type *AggType, TTI::TargetCostKind CostKind,
+    ArrayRef<unsigned> Indices, Value *AggDef) const {
+  assert((Opcode == Instruction::InsertValue ||
+          Opcode == Instruction::ExtractValue) &&
+         "Expecting Opcode to be insertvalue/extractvalue.");
+  InstructionCost Cost = TTIImpl->getInsertExtractValueCost(
+      Opcode, AggType, CostKind, Indices, AggDef);
+  assert(Cost >= 0 && "TTI should not produce negative costs!");
+  return Cost;
+}
+
 InstructionCost TargetTransformInfo::getReplicationShuffleCost(
     Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
     TTI::TargetCostKind CostKind) const {

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -3605,13 +3605,13 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
         }
       }
 
-      // ExtractValue instructions must be uniform, because the operands are
-      // known to be loop-invariant.
       if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
         if (IsOutOfScope(EVI->getAggregateOperand())) {
           AddToWorklistIfAllowed(EVI);
           continue;
         }
+        // Only ExtractValue instructions where the aggregate value comes from a
+        // call are allowed to be non-uniform.
         assert(isa<CallInst>(EVI->getAggregateOperand()) &&
                "Expected aggregate value to be call return value");
       }

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -1514,8 +1514,14 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
     return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
                                           Ctx.CostKind);
   }
-  case Instruction::ExtractValue:
-    return 0;
+  case Instruction::ExtractValue: {
+    assert(getNumOperands() == 2 && "expected single level extractvalue");
+    Type *VectorizedTy = toVectorizedTy(Ctx.Types.inferScalarType(this), VF);
+    auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
+    return Ctx.TTI.getInsertExtractValueCost(
+        Instruction::ExtractValue, VectorizedTy, Ctx.CostKind,
+        static_cast<unsigned>(CI->getZExtValue()));
+  }
   case Instruction::ICmp:
   case Instruction::FCmp: {
     Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());