[RISCV] Lower non-power-of-2 vector to nearest power-of-2 vector leng…

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

@@ -25,6 +25,7 @@
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TypeSize.h"
 #include "llvm/Support/raw_ostream.h"
@@ -5686,6 +5687,17 @@ SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
   unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
   unsigned InNumElts = InVT.getVectorMinNumElements();
   unsigned VTNumElts = VT.getVectorMinNumElements();
+
+  if (InVT.isScalableVector())
+  {
+     unsigned EltSize = InVT.getScalarType ().getFixedSizeInBits ();
+
+     unsigned MinVScale = getVScaleRange(&DAG.getMachineFunction ().getFunction(), 64)
+                                 .getUnsignedMin().getZExtValue ();
+     InNumElts = InNumElts * MinVScale;
+  }
+
+
   assert(IdxVal % VTNumElts == 0 &&
          "Expected Idx to be a multiple of subvector minimum vector length");
   if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)

llvm/lib/Target/RISCV/CMakeLists.txt

@@ -57,6 +57,7 @@ add_llvm_target(RISCVCodeGen
   RISCVTargetObjectFile.cpp
   RISCVTargetTransformInfo.cpp
   RISCVVectorPeephole.cpp
+  RISCVLegalizeNonPowerOf2Vector.cpp
   GISel/RISCVCallLowering.cpp
   GISel/RISCVInstructionSelector.cpp
   GISel/RISCVLegalizerInfo.cpp

llvm/lib/Target/RISCV/RISCV.h

@@ -99,6 +99,9 @@ void initializeRISCVO0PreLegalizerCombinerPass(PassRegistry &);
 
 FunctionPass *createRISCVPreLegalizerCombiner();
 void initializeRISCVPreLegalizerCombinerPass(PassRegistry &);
+
+FunctionPass *createRISCVLegalizeNonPowerOf2Vector();
+void initializeRISCVLegalizeNonPowerOf2VectorPass(PassRegistry &);
 } // namespace llvm
 
 #endif

llvm/lib/Target/RISCV/RISCVLegalizeNonPowerOf2Vector.cpp

@@ -0,0 +1,199 @@
+#include "RISCV.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/VectorBuilder.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <vector>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-legalize-non-power-of-2-vector"
+#define PASS_NAME "Legalize non-power-of-2 vector type"
+
+namespace {
+class RISCVLegalizeNonPowerOf2Vector : public FunctionPass {
+  const RISCVSubtarget *ST;
+  unsigned MinVScale;
+
+public:
+  static char ID;
+  RISCVLegalizeNonPowerOf2Vector() : FunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<TargetPassConfig>();
+  }
+
+  bool runOnFunction(Function &F) override;
+  StringRef getPassName() const override { return PASS_NAME; }
+
+private:
+  FixedVectorType *extracUsedFixedVectorType(const Instruction &I) const;
+
+  bool isTargetType(FixedVectorType *VecTy) const;
+
+  ScalableVectorType *
+  getContainerForFixedLengthVector(FixedVectorType *FixedVecTy);
+};
+} // namespace
+
+FixedVectorType *RISCVLegalizeNonPowerOf2Vector::extracUsedFixedVectorType(
+    const Instruction &I) const {
+  if (isa<FixedVectorType>(I.getType())) {
+    return cast<FixedVectorType>(I.getType());
+  } else if (isa<StoreInst>(I) &&
+             isa<FixedVectorType>(
+                 cast<StoreInst>(&I)->getValueOperand()->getType())) {
+    return cast<FixedVectorType>(
+        cast<StoreInst>(&I)->getValueOperand()->getType());
+  }
+  return nullptr;
+}
+
+ScalableVectorType *
+RISCVLegalizeNonPowerOf2Vector::getContainerForFixedLengthVector(
+    FixedVectorType *FixedVecTy) {
+  // TODO: Consider vscale_range to pick a better/smaller type.
+  //
+  uint64_t 	 NumElts =
+      std::max<uint64_t>((NextPowerOf2 (FixedVecTy->getNumElements()) / MinVScale), 1);
+
+  Type *ElementType = FixedVecTy->getElementType();
+
+  if (ElementType->isIntegerTy(1))
+      NumElts = std::max(NumElts, 8UL);
+
+  return ScalableVectorType::get(ElementType, NumElts);
+}
+
+bool RISCVLegalizeNonPowerOf2Vector::isTargetType(
+    FixedVectorType *VecTy) const {
+  if (isPowerOf2_32(VecTy->getNumElements()))
+    return false;
+
+  Type *EltTy = VecTy->getElementType();
+
+  if (EltTy->isIntegerTy(1))
+    return false;
+
+  if (EltTy->isIntegerTy(64))
+    return ST->hasVInstructionsI64();
+  else if (EltTy->isFloatTy())
+    return ST->hasVInstructionsF32();
+  else if (EltTy->isDoubleTy())
+    return ST->hasVInstructionsF64();
+  else if (EltTy->isHalfTy())
+    return ST->hasVInstructionsF16Minimal();
+  else if (EltTy->isBFloatTy())
+    return ST->hasVInstructionsBF16Minimal();
+
+  return (EltTy->isIntegerTy(1) || EltTy->isIntegerTy(8) ||
+          EltTy->isIntegerTy(16) || EltTy->isIntegerTy(32));
+}
+
+bool RISCVLegalizeNonPowerOf2Vector::runOnFunction(Function &F) {
+
+  if (skipFunction(F))
+    return false;
+
+  auto &TPC = getAnalysis<TargetPassConfig>();
+  auto &TM = TPC.getTM<RISCVTargetMachine>();
+  ST = &TM.getSubtarget<RISCVSubtarget>(F);
+
+  if (!ST->hasVInstructions())
+    return false;
+
+  auto Attr = F.getFnAttribute(Attribute::VScaleRange);
+  if (Attr.isValid()) {
+      MinVScale = Attr.getVScaleRangeMin ();
+  } else {
+    unsigned MinVLen = ST->getRealMinVLen();
+    if (MinVLen < RISCV::RVVBitsPerBlock)
+      return false;
+    MinVScale = MinVLen / RISCV::RVVBitsPerBlock;
+    AttrBuilder AB(F.getContext());
+    AB.addVScaleRangeAttr(MinVScale,
+                         std::optional<unsigned>());
+
+    F.addFnAttr (AB.getAttribute(Attribute::VScaleRange));
+  }
+
+  bool Modified = false;
+  std::vector<Instruction *> ToBeRemoved;
+  for (auto &BB : F) {
+    for (auto &I : make_range(BB.rbegin(), BB.rend())) {
+      if (auto VecTy = extracUsedFixedVectorType(I)) {
+        if (!isTargetType(VecTy)) {
+          continue;
+        }
+
+        Value *I64Zero = ConstantInt::get(Type::getInt64Ty(F.getContext()), 0);
+
+        // Replace fixed length vector with scalable vector
+        IRBuilder<> Builder(&I);
+        VectorBuilder VecBuilder(Builder);
+        VecBuilder.setStaticVL(VecTy->getNumElements());
+        VectorType *NewVecTy = getContainerForFixedLengthVector(VecTy);
+        VecBuilder.setMask(Builder.CreateVectorSplat(
+            NewVecTy->getElementCount(), Builder.getTrue()));
+
+        if (auto *BinOp = dyn_cast<BinaryOperator>(&I)) {
+          Value *Op1 = BinOp->getOperand(0);
+          Value *Op2 = BinOp->getOperand(1);
+          Value *NewOp1 = Builder.CreateInsertVector(
+              NewVecTy, PoisonValue::get(NewVecTy), Op1, I64Zero);
+          Value *NewOp2 = Builder.CreateInsertVector(
+              NewVecTy, PoisonValue::get(NewVecTy), Op2, I64Zero);
+          Value *NewBinOp = VecBuilder.createVectorInstruction(
+              BinOp->getOpcode(), NewVecTy, {NewOp1, NewOp2});
+          Value *FinalResult =
+              Builder.CreateExtractVector(VecTy, NewBinOp, I64Zero);
+          BinOp->replaceAllUsesWith(FinalResult);
+          ToBeRemoved.push_back(BinOp);
+          Modified = true;
+        } else if (auto *StoreOp = dyn_cast<StoreInst>(&I)) {
+          Value *Val = StoreOp->getOperand(0);
+          Value *Addr = StoreOp->getOperand(1);
+          Value *NewVal = Builder.CreateInsertVector(
+              NewVecTy, PoisonValue::get(NewVecTy), Val, I64Zero);
+          Value *NewStoreOp = VecBuilder.createVectorInstruction(
+              StoreOp->getOpcode(), NewVecTy, {NewVal, Addr});
+          StoreOp->replaceAllUsesWith(NewStoreOp);
+          ToBeRemoved.push_back(StoreOp);
+        } else if (auto *LoadOp = dyn_cast<LoadInst>(&I)) {
+          Value *Addr = LoadOp->getOperand(0);
+          Value *NewLoadOp = VecBuilder.createVectorInstruction(
+              LoadOp->getOpcode(), NewVecTy, {Addr});
+          Value *FinalResult =
+              Builder.CreateExtractVector(VecTy, NewLoadOp, I64Zero);
+          LoadOp->replaceAllUsesWith(FinalResult);
+          ToBeRemoved.push_back(LoadOp);
+        }
+      }
+    }
+  }
+  for_each(ToBeRemoved.begin(), ToBeRemoved.end(),
+           [](Instruction *I) { I->eraseFromParent(); });
+  return Modified;
+}
+
+char RISCVLegalizeNonPowerOf2Vector::ID = 0;
+
+INITIALIZE_PASS_BEGIN(RISCVLegalizeNonPowerOf2Vector, DEBUG_TYPE, PASS_NAME,
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(RISCVLegalizeNonPowerOf2Vector, DEBUG_TYPE, PASS_NAME,
+                    false, false)
+
+FunctionPass *llvm::createRISCVLegalizeNonPowerOf2Vector() {
+  return new RISCVLegalizeNonPowerOf2Vector();
+}

llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

@@ -128,6 +128,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVDAGToDAGISelLegacyPass(*PR);
   initializeRISCVMoveMergePass(*PR);
   initializeRISCVPushPopOptPass(*PR);
+  initializeRISCVLegalizeNonPowerOf2VectorPass(*PR);
 }
 
 static StringRef computeDataLayout(const Triple &TT,
@@ -452,6 +453,7 @@ bool RISCVPassConfig::addPreISel() {
 void RISCVPassConfig::addCodeGenPrepare() {
   if (getOptLevel() != CodeGenOptLevel::None)
     addPass(createTypePromotionLegacyPass());
+  addPass(createRISCVLegalizeNonPowerOf2Vector());
   TargetPassConfig::addCodeGenPrepare();
 }
 

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abs.ll

@@ -39,7 +39,25 @@ define void @abs_v6i16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vslidedown.vi v9, v8, 1
+; CHECK-NEXT:    vmv.x.s a1, v9
+; CHECK-NEXT:    vmv.x.s a2, v8
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v9, a2
+; CHECK-NEXT:    vslide1down.vx v9, v9, a1
+; CHECK-NEXT:    vslidedown.vi v10, v8, 2
+; CHECK-NEXT:    vmv.x.s a1, v10
+; CHECK-NEXT:    vslide1down.vx v9, v9, a1
+; CHECK-NEXT:    vslidedown.vi v10, v8, 3
+; CHECK-NEXT:    vmv.x.s a1, v10
+; CHECK-NEXT:    vslide1down.vx v9, v9, a1
+; CHECK-NEXT:    vslidedown.vi v10, v8, 4
+; CHECK-NEXT:    vmv.x.s a1, v10
+; CHECK-NEXT:    vslide1down.vx v9, v9, a1
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vmv.x.s a1, v8
+; CHECK-NEXT:    vslide1down.vx v8, v9, a1
+; CHECK-NEXT:    vslidedown.vi v8, v8, 2
 ; CHECK-NEXT:    vrsub.vi v9, v8, 0
 ; CHECK-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
 ; CHECK-NEXT:    vmax.vv v8, v8, v9

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll

@@ -220,7 +220,18 @@ define i64 @extractelt_v3i64(ptr %x) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 3, e64, m2, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vmv.x.s a0, v8
+; RV32-NEXT:    vmv.s.x v10, a0
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vsrl.vx v8, v8, a0
+; RV32-NEXT:    vmv.x.s a0, v8
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; RV32-NEXT:    vslideup.vi v8, v10, 4
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v10, v8, 4
 ; RV32-NEXT:    vmv.x.s a0, v10
 ; RV32-NEXT:    vslidedown.vi v8, v8, 5
@@ -567,10 +578,37 @@ define i64 @extractelt_v3i64_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 3, e64, m2, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-NEXT:    vadd.vv v8, v8, v8
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vsrl.vx v10, v8, a0
+; RV32-NEXT:    vmv.x.s a2, v10
+; RV32-NEXT:    vmv.x.s a3, v8
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v10, a3
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v12, v8, 1
+; RV32-NEXT:    vmv.x.s a2, v12
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vsrl.vx v12, v12, a0
+; RV32-NEXT:    vmv.x.s a2, v12
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vmv.x.s a2, v8
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vsrl.vx v8, v8, a0
+; RV32-NEXT:    vmv.x.s a0, v8
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v10, a0
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-NEXT:    add a1, a1, a1
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vx v10, v8, a1
 ; RV32-NEXT:    vmv.x.s a0, v10
 ; RV32-NEXT:    addi a1, a1, 1
@@ -582,8 +620,18 @@ define i64 @extractelt_v3i64_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 3, e64, m2, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV64-NEXT:    vadd.vv v8, v8, v8
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-NEXT:    vmv.x.s a0, v10
+; RV64-NEXT:    vmv.x.s a2, v8
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a2
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    vmv.x.s a0, v8
+; RV64-NEXT:    vslide1down.vx v8, v10, a0
+; RV64-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-NEXT:    vslidedown.vx v8, v8, a1
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret