refactor GenXLowering for mul64

Change-Id: I158ac50a72edc170eee9727319740f374af350b1

Author: aparshin-intel

IGC/VectorCompiler/lib/GenXCodeGen/GenXLowering.cpp

@@ -138,7 +138,6 @@ using namespace genx;
 static cl::opt<bool>
     EnableGenXByteWidening("enable-genx-byte-widening", cl::init(true),
                            cl::Hidden, cl::desc("Enable GenX byte widening."));
-
 namespace {
 
 // GenXLowering : legalize execution widths and GRF crossing
@@ -2391,64 +2390,35 @@ bool GenXLowering::lowerFCmpInst(FCmpInst *Inst) {
 
 // Lower cmp instructions that GenX cannot deal with.
 bool GenXLowering::lowerMul64(Instruction *Inst) {
+
+  LoHiSplitter SplitBuilder(*Inst);
+  if (!SplitBuilder.IsI64Operation())
+    return false;
+
   IRBuilder<> Builder(Inst);
   Builder.SetCurrentDebugLocation(Inst->getDebugLoc());
-  auto Src0 = Inst->getOperand(0);
-  auto Src1 = Inst->getOperand(1);
-  auto ETy = Src0->getType();
-  auto Len = 1;
-  if (ETy->isVectorTy()) {
-    Len = ETy->getVectorNumElements();
-    ETy = ETy->getVectorElementType();
-  }
-  if (!ETy->isIntegerTy() || ETy->getPrimitiveSizeInBits() != 64)
-    return false;
-  auto VTy = VectorType::get(ETy->getInt32Ty(Inst->getContext()), Len * 2);
-  // create src0 bitcast, then the low and high part
-  auto Src0V = Builder.CreateBitCast(Src0, VTy);
-  Region R(Inst);
-  R.Offset = 0;
-  R.Width = Len;
-  R.NumElements = Len;
-  R.Stride = 2;
-  R.VStride = 0;
-  auto Src0L = R.createRdRegion(Src0V, "", Inst, Inst->getDebugLoc());
-  R.Offset = 4;
-  auto Src0H = R.createRdRegion(Src0V, "", Inst, Inst->getDebugLoc());
-  // create src1 bitcast, then the low and high part
-  auto Src1V = Builder.CreateBitCast(Src1, VTy);
-  R.Offset = 0;
-  auto Src1L = R.createRdRegion(Src1V, "", Inst, Inst->getDebugLoc());
-  R.Offset = 4;
-  auto Src1H = R.createRdRegion(Src1V, "", Inst, Inst->getDebugLoc());
+
+  auto Src0 = SplitBuilder.splitOperand(0);
+  auto Src1 = SplitBuilder.splitOperand(1);
+
   // create muls and adds
-  auto ResL = Builder.CreateMul(Src0L, Src1L);
+  auto *ResL = Builder.CreateMul(Src0.Lo, Src1.Lo);
   // create the mulh intrinsic to the get the carry-part
-  Type *tys[2];
-  SmallVector<llvm::Value *, 2> args;
-  // build type-list
-  tys[0] = ResL->getType();
-  tys[1] = Src0L->getType();
+  Type *tys[2] = {ResL->getType(), Src0.Lo->getType()};
   // build argument list
-  args.push_back(Src0L);
-  args.push_back(Src1L);
-  auto M = Inst->getParent()->getParent()->getParent();
+  SmallVector<llvm::Value *, 2> args{Src0.Lo, Src1.Lo};
+  auto *M = Inst->getModule();
   Function *IntrinFunc =
       GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys);
-  Instruction *Cari = CallInst::Create(IntrinFunc, args, "", Inst);
-  Cari->setDebugLoc(Inst->getDebugLoc());
-  auto Temp0 = Builder.CreateMul(Src0L, Src1H);
-  auto Temp1 = Builder.CreateAdd(Cari, Temp0);
-  auto Temp2 = Builder.CreateMul(Src0H, Src1L);
-  auto ResH = Builder.CreateAdd(Temp2, Temp1);
-  // create the write-regions
-  auto UndefV = UndefValue::get(VTy);
-  R.Offset = 0;
-  auto WrL = R.createWrRegion(UndefV, ResL, "WrLow", Inst, Inst->getDebugLoc());
-  R.Offset = 4;
-  auto WrH = R.createWrRegion(WrL, ResH, "WrHigh", Inst, Inst->getDebugLoc());
+
+  auto *Cari = Builder.CreateCall(IntrinFunc, args, ".cari");
+  auto *Temp0 = Builder.CreateMul(Src0.Lo, Src1.Hi);
+  auto *Temp1 = Builder.CreateAdd(Cari, Temp0);
+  auto *Temp2 = Builder.CreateMul(Src0.Hi, Src1.Lo);
+  auto *ResH = Builder.CreateAdd(Temp2, Temp1);
+
   // create the bitcast to the destination-type
-  auto Replace = Builder.CreateBitCast(WrH, Inst->getType(), "mul64");
+  auto *Replace = SplitBuilder.combineSplit(*ResL, *ResH, "mul64");
   Inst->replaceAllUsesWith(Replace);
   ToErase.push_back(Inst);
   return true;

IGC/VectorCompiler/lib/GenXCodeGen/GenXUtil.cpp

@@ -760,6 +760,71 @@ unsigned ShuffleVectorAnalyzer::getSerializeCost(unsigned i) {
   return Cost;
 }
 
+LoHiSplitter::LoHiSplitter(Instruction &Inst, unsigned BaseOpIdx) : Inst(Inst) {
+
+  auto *Operand = Inst.getOperand(BaseOpIdx);
+  ETy = Operand->getType();
+  Len = 1;
+  if (ETy->isVectorTy()) {
+    Len = ETy->getVectorNumElements();
+    ETy = ETy->getVectorElementType();
+  }
+  VI32Ty = VectorType::get(ETy->getInt32Ty(Inst.getContext()), Len * 2);
+}
+
+Region LoHiSplitter::createSplitRegion(Type *Ty, LoHiSplitter::RegionType RT) {
+  Region R(Ty);
+  R.Width = Len;
+  R.NumElements = Len;
+  R.VStride = 0;
+  // take every second element;
+  R.Stride = 2;
+  // offset is encoded in bytes
+  R.Offset = (RT == RegionType::LoRegion) ? 0 : 4;
+  return R;
+}
+
+LoHiSplitter::Split LoHiSplitter::splitOperand(unsigned SourceIdx) {
+
+  const auto &DL = Inst.getDebugLoc();
+  auto Name = Inst.getName();
+
+  assert(Inst.getNumOperands() > SourceIdx);
+  auto *Src = Inst.getOperand(SourceIdx);
+  assert(Src->getType()->getScalarType()->isIntegerTy(64));
+
+  auto *ShreddedSrc = new BitCastInst(Src, VI32Ty, Name + ".iv32cast", &Inst);
+  ShreddedSrc->setDebugLoc(DL);
+
+  auto LoRegion = createSplitRegion(VI32Ty, RegionType::LoRegion);
+  auto *L = LoRegion.createRdRegion(ShreddedSrc, Name + ".lsplit", &Inst, DL);
+
+  auto HiRegion = createSplitRegion(VI32Ty, RegionType::HiRegion);
+  auto *H = HiRegion.createRdRegion(ShreddedSrc, Name + ".rsplit", &Inst, DL);
+
+  return {L, H};
+}
+Value *LoHiSplitter::combineSplit(Value &L, Value &H, const Twine &Name) {
+
+  const auto &DL = Inst.getDebugLoc();
+
+  assert(L.getType() == H.getType() && L.getType()->isVectorTy() &&
+         L.getType()->getVectorElementType()->isIntegerTy(32));
+
+  // create the write-regions
+  auto LoRegion = createSplitRegion(VI32Ty, RegionType::LoRegion);
+  auto *UndefV = UndefValue::get(VI32Ty);
+  auto *WrL = LoRegion.createWrRegion(UndefV, &L, "WrLow", &Inst, DL);
+
+  auto HiRegion = createSplitRegion(VI32Ty, RegionType::HiRegion);
+  auto *WrH = HiRegion.createWrRegion(WrL, &H, "WrHigh", &Inst, DL);
+
+  auto *V64Ty = VectorType::get(ETy->getInt64Ty(Inst.getContext()), Len);
+  auto *Result = new BitCastInst(WrH, V64Ty, Name, &Inst);
+  Result->setDebugLoc(DL);
+  return Result;
+}
+
 /***********************************************************************
  * adjustPhiNodesForBlockRemoval : adjust phi nodes when removing a block
  *

IGC/VectorCompiler/lib/GenXCodeGen/GenXUtil.h

@@ -257,6 +257,40 @@ class ShuffleVectorAnalyzer {
   OperandRegionInfo getMaskRegionPrefix(int StartIdx);
 };
 
+// class for splitting i64 (both vector and scalar) to subregions of i32 vectors
+// Used in GenxLowering and emulation routines
+class LoHiSplitter {
+  Instruction &Inst;
+
+  Type *ETy = nullptr;
+  Type *VI32Ty = nullptr;
+  size_t Len = 0;
+
+  enum class RegionType { LoRegion, HiRegion };
+  Region createSplitRegion(Type *Ty, RegionType RT);
+
+public:
+  struct Split {
+    Value *Lo;
+    Value *Hi;
+  };
+
+  // Instruction is used as an insertion point, debug location source and
+  // as a source of operands to split.
+  // If BaseOpIdx indexes a scalar/vector operand of i64 type, then
+  // IsI64Operation shall return true
+  LoHiSplitter(Instruction &Inst, unsigned BaseOpIdx = 0);
+
+  // Splitted Operand is expected to be a scalar/vector of i64 type
+  Split splitOperand(unsigned SourceIdx);
+
+  // Combined values are expected to be a vector of i32 of the same size
+  Value *combineSplit(Value &L, Value &H, const Twine &Name);
+
+  // convinence method for quick sanity checking
+  bool IsI64Operation() { return ETy->isIntegerTy(64); }
+};
+
 // adjustPhiNodesForBlockRemoval : adjust phi nodes when removing a block
 void adjustPhiNodesForBlockRemoval(BasicBlock *Succ, BasicBlock *BB);