Skip to content

[IRBuilder] Add versions of createInsertVector/createExtractVector that take a uint64_t index. #138324

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 2, 2025
Merged

[IRBuilder] Add versions of createInsertVector/createExtractVector that take a uint64_t index. #138324

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions clang/lib/CodeGen/CGCall.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1372,9 +1372,8 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
auto *Load = CGF.Builder.CreateLoad(Src);
auto *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);
auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
llvm::Value *Result = CGF.Builder.CreateInsertVector(
ScalableDstTy, PoisonVec, Load, Zero, "cast.scalable");
ScalableDstTy, PoisonVec, Load, uint64_t(0), "cast.scalable");
if (ScalableDstTy != Ty)
Result = CGF.Builder.CreateBitCast(Result, Ty);
return Result;
Expand Down Expand Up @@ -1482,10 +1481,8 @@ CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
V = CGF.Builder.CreateBitCast(V, FromTy);
}
if (FromTy->getElementType() == ToTy->getElementType()) {
llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);

V->setName(Name + ".coerce");
V = CGF.Builder.CreateExtractVector(ToTy, V, Zero, "cast.fixed");
V = CGF.Builder.CreateExtractVector(ToTy, V, uint64_t(0), "cast.fixed");
return {V, true};
}
return {V, false};
Expand Down Expand Up @@ -6110,8 +6107,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
dyn_cast<llvm::ScalableVectorType>(V->getType())) {
if (FixedDstTy->getElementType() ==
ScalableSrcTy->getElementType()) {
llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
V = Builder.CreateExtractVector(FixedDstTy, V, Zero,
V = Builder.CreateExtractVector(FixedDstTy, V, uint64_t(0),
"cast.fixed");
return RValue::get(V);
}
Expand Down
10 changes: 4 additions & 6 deletions clang/lib/CodeGen/CGExprScalar.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2487,9 +2487,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
}
if (FixedSrcTy->getElementType() == ScalableDstTy->getElementType()) {
llvm::Value *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);
llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
llvm::Value *Result = Builder.CreateInsertVector(
ScalableDstTy, PoisonVec, Src, Zero, "cast.scalable");
ScalableDstTy, PoisonVec, Src, uint64_t(0), "cast.scalable");
if (Result->getType() != DstTy)
Result = Builder.CreateBitCast(Result, DstTy);
return Result;
Expand All @@ -2512,10 +2511,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
ScalableSrcTy->getElementCount().getKnownMinValue() / 8);
Src = Builder.CreateBitCast(Src, ScalableSrcTy);
}
if (ScalableSrcTy->getElementType() == FixedDstTy->getElementType()) {
llvm::Value *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
return Builder.CreateExtractVector(DstTy, Src, Zero, "cast.fixed");
}
if (ScalableSrcTy->getElementType() == FixedDstTy->getElementType())
return Builder.CreateExtractVector(DstTy, Src, uint64_t(0),
"cast.fixed");
}
}

Expand Down
16 changes: 8 additions & 8 deletions clang/lib/CodeGen/TargetBuiltins/ARM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ llvm::Value *CodeGenFunction::EmitFP8NeonFDOTCall(
if (ExtendLaneArg) {
auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
Builder.getInt64(0));
uint64_t(0));
}
return EmitFP8NeonCall(IID, Tys, Ops, E, name);
}
Expand All @@ -478,7 +478,7 @@ llvm::Value *CodeGenFunction::EmitFP8NeonFMLACall(
if (ExtendLaneArg) {
auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
Builder.getInt64(0));
uint64_t(0));
}
const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
RetTy->getPrimitiveSizeInBits();
Expand All @@ -502,7 +502,7 @@ Value *CodeGenFunction::EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
// Op[0] is mfloat8x16_t, but the intrinsic converts only the lower part of
// the vector.
Tys[1] = llvm::FixedVectorType::get(Int8Ty, 8);
Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], Builder.getInt64(0));
Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], uint64_t(0));
}
return EmitFP8NeonCall(IID, Tys, Ops, E, name);
}
Expand Down Expand Up @@ -4727,7 +4727,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,

llvm::Type *OverloadedTy = getSVEVectorForElementType(EltTy);
Value *InsertSubVec = Builder.CreateInsertVector(
OverloadedTy, PoisonValue::get(OverloadedTy), Vec, Builder.getInt64(0));
OverloadedTy, PoisonValue::get(OverloadedTy), Vec, uint64_t(0));

Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
Expand Down Expand Up @@ -4810,7 +4810,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
case SVE::BI__builtin_sve_svset_neonq_f32:
case SVE::BI__builtin_sve_svset_neonq_f64:
case SVE::BI__builtin_sve_svset_neonq_bf16: {
return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], uint64_t(0));
}

case SVE::BI__builtin_sve_svget_neonq_s8:
Expand All @@ -4825,7 +4825,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
case SVE::BI__builtin_sve_svget_neonq_f32:
case SVE::BI__builtin_sve_svget_neonq_f64:
case SVE::BI__builtin_sve_svget_neonq_bf16: {
return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
return Builder.CreateExtractVector(Ty, Ops[0], uint64_t(0));
}

case SVE::BI__builtin_sve_svdup_neonq_s8:
Expand All @@ -4841,7 +4841,7 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
case SVE::BI__builtin_sve_svdup_neonq_f64:
case SVE::BI__builtin_sve_svdup_neonq_bf16: {
Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
Builder.getInt64(0));
uint64_t(0));
return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
{Insert, Builder.getInt64(0)});
}
Expand Down Expand Up @@ -7767,7 +7767,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
llvm::Type *Ty = llvm::FixedVectorType::get(Int8Ty, 16);
Ops[0] = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
Builder.getInt64(0));
uint64_t(0));
return EmitFP8NeonCvtCall(Intrinsic::aarch64_neon_fp8_fcvtn2, Ty,
Ops[1]->getType(), false, Ops, E, "vfcvtn2");
}
Expand Down
12 changes: 12 additions & 0 deletions llvm/include/llvm/IR/IRBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1101,6 +1101,12 @@ class IRBuilderBase {
Name);
}

/// Create a call to the vector.extract intrinsic.
CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, uint64_t Idx,
const Twine &Name = "") {
return CreateExtractVector(DstType, SrcVec, getInt64(Idx), Name);
}

/// Create a call to the vector.insert intrinsic.
CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
Value *Idx, const Twine &Name = "") {
Expand All @@ -1109,6 +1115,12 @@ class IRBuilderBase {
nullptr, Name);
}

/// Create a call to the vector.extract intrinsic.
CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
uint64_t Idx, const Twine &Name = "") {
return CreateInsertVector(DstType, SrcVec, SubVec, getInt64(Idx), Name);
}

/// Create a call to llvm.stacksave
CallInst *CreateStackSave(const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
Expand Down
6 changes: 2 additions & 4 deletions llvm/lib/IR/AutoUpgrade.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4554,9 +4554,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
Value *NewLdCall = Builder.CreateCall(NewFn, Args);
Value *Ret = llvm::PoisonValue::get(RetTy);
for (unsigned I = 0; I < N; I++) {
Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
}
NewCall = dyn_cast<CallInst>(Ret);
break;
Expand Down Expand Up @@ -4611,9 +4610,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
Value *Ret = llvm::PoisonValue::get(RetTy);
unsigned MinElts = RetTy->getMinNumElements() / N;
for (unsigned I = 0; I < N; I++) {
Value *Idx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
Value *V = CI->getArgOperand(I);
Ret = Builder.CreateInsertVector(RetTy, Ret, V, Idx);
Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
}
NewCall = dyn_cast<CallInst>(Ret);
}
Expand Down
30 changes: 11 additions & 19 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17227,9 +17227,7 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
Value *SubVec = Builder.CreateExtractValue(LdN, Index);

if (UseScalable)
SubVec = Builder.CreateExtractVector(
FVTy, SubVec,
ConstantInt::get(Type::getInt64Ty(VTy->getContext()), 0));
SubVec = Builder.CreateExtractVector(FVTy, SubVec, uint64_t(0));

// Convert the integer vector to pointer vector if the element is pointer.
if (EltTy->isPointerTy())
Expand Down Expand Up @@ -17436,9 +17434,8 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
}

if (UseScalable)
Shuffle = Builder.CreateInsertVector(
STVTy, PoisonValue::get(STVTy), Shuffle,
ConstantInt::get(Type::getInt64Ty(STVTy->getContext()), 0));
Shuffle = Builder.CreateInsertVector(STVTy, PoisonValue::get(STVTy),
Shuffle, uint64_t(0));

Ops.push_back(Shuffle);
}
Expand Down Expand Up @@ -30020,31 +30017,26 @@ Value *AArch64TargetLowering::createComplexDeinterleavingIR(
.getKnownMinValue() /
2;
auto *HalfTy = VectorType::getHalfElementsVectorType(Ty);
auto *LowerSplitA = B.CreateExtractVector(HalfTy, InputA, B.getInt64(0));
auto *LowerSplitB = B.CreateExtractVector(HalfTy, InputB, B.getInt64(0));
auto *UpperSplitA =
B.CreateExtractVector(HalfTy, InputA, B.getInt64(Stride));
auto *UpperSplitB =
B.CreateExtractVector(HalfTy, InputB, B.getInt64(Stride));
auto *LowerSplitA = B.CreateExtractVector(HalfTy, InputA, uint64_t(0));
auto *LowerSplitB = B.CreateExtractVector(HalfTy, InputB, uint64_t(0));
auto *UpperSplitA = B.CreateExtractVector(HalfTy, InputA, Stride);
auto *UpperSplitB = B.CreateExtractVector(HalfTy, InputB, Stride);
Value *LowerSplitAcc = nullptr;
Value *UpperSplitAcc = nullptr;
Type *FullTy = Ty;
FullTy = Accumulator->getType();
auto *HalfAccTy = VectorType::getHalfElementsVectorType(
cast<VectorType>(Accumulator->getType()));
LowerSplitAcc =
B.CreateExtractVector(HalfAccTy, Accumulator, B.getInt64(0));
UpperSplitAcc =
B.CreateExtractVector(HalfAccTy, Accumulator, B.getInt64(AccStride));
LowerSplitAcc = B.CreateExtractVector(HalfAccTy, Accumulator, uint64_t(0));
UpperSplitAcc = B.CreateExtractVector(HalfAccTy, Accumulator, AccStride);
auto *LowerSplitInt = createComplexDeinterleavingIR(
B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
auto *UpperSplitInt = createComplexDeinterleavingIR(
B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);

auto *Result = B.CreateInsertVector(FullTy, PoisonValue::get(FullTy),
LowerSplitInt, B.getInt64(0));
return B.CreateInsertVector(FullTy, Result, UpperSplitInt,
B.getInt64(AccStride));
LowerSplitInt, uint64_t(0));
return B.CreateInsertVector(FullTy, Result, UpperSplitInt, AccStride);
}

if (OperationType == ComplexDeinterleavingOperation::CMulPartial) {
Expand Down
10 changes: 5 additions & 5 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2406,9 +2406,9 @@ static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
if (TyA == B->getType() &&
RetTy == ScalableVectorType::getDoubleElementsVectorType(TyA)) {
auto *SubVec = IC.Builder.CreateInsertVector(
RetTy, PoisonValue::get(RetTy), A, IC.Builder.getInt64(0));
auto *ConcatVec = IC.Builder.CreateInsertVector(
RetTy, SubVec, B, IC.Builder.getInt64(TyA->getMinNumElements()));
RetTy, PoisonValue::get(RetTy), A, uint64_t(0));
auto *ConcatVec = IC.Builder.CreateInsertVector(RetTy, SubVec, B,
TyA->getMinNumElements());
ConcatVec->takeName(&II);
return IC.replaceInstUsesWith(II, ConcatVec);
}
Expand Down Expand Up @@ -2602,9 +2602,9 @@ static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
auto *WideShuffleMaskTy =
ScalableVectorType::get(IC.Builder.getInt32Ty(), PatternElementCount);

auto ZeroIdx = ConstantInt::get(IC.Builder.getInt64Ty(), APInt(64, 0));
auto InsertSubvector = IC.Builder.CreateInsertVector(
II.getType(), PoisonValue::get(II.getType()), InsertEltChain, ZeroIdx);
II.getType(), PoisonValue::get(II.getType()), InsertEltChain,
uint64_t(0));
auto WideBitcast =
IC.Builder.CreateBitOrPointerCast(InsertSubvector, WideScalableTy);
auto WideShuffleMask = ConstantAggregateZero::get(WideShuffleMaskTy);
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2102,8 +2102,7 @@ bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
for (VectorSlice &S : Slices) {
const auto ValName = "largephi.insertslice" + std::to_string(NameSuffix++);
if (S.NumElts > 1)
Vec =
B.CreateInsertVector(FVT, Vec, S.NewPHI, B.getInt64(S.Idx), ValName);
Vec = B.CreateInsertVector(FVT, Vec, S.NewPHI, S.Idx, ValName);
else
Vec = B.CreateInsertElement(Vec, S.NewPHI, S.Idx, ValName);
}
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1440,14 +1440,14 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
if (Src0Ty->getNumElements() > Src0NumElts) {
Src0 = IC.Builder.CreateExtractVector(
FixedVectorType::get(Src0Ty->getElementType(), Src0NumElts), Src0,
IC.Builder.getInt64(0));
uint64_t(0));
MadeChange = true;
}

if (Src1Ty->getNumElements() > Src1NumElts) {
Src1 = IC.Builder.CreateExtractVector(
FixedVectorType::get(Src1Ty->getElementType(), Src1NumElts), Src1,
IC.Builder.getInt64(0));
uint64_t(0));
MadeChange = true;
}

Expand Down
5 changes: 2 additions & 3 deletions llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1322,9 +1322,8 @@ Value *LoopIdiomVectorize::expandFindFirstByte(
Needle0, "needle0");
LoadNeedle = Builder.CreateSelect(PredNeedle, LoadNeedle, Needle0Splat,
"needle_splat");
LoadNeedle =
Builder.CreateExtractVector(FixedVectorType::get(CharTy, VF), LoadNeedle,
ConstantInt::get(I64Ty, 0), "needle_vec");
LoadNeedle = Builder.CreateExtractVector(
FixedVectorType::get(CharTy, VF), LoadNeedle, uint64_t(0), "needle_vec");

// (2.c) Test if there's a match.
Value *MatchPred = Builder.CreateIntrinsic(
Expand Down
5 changes: 2 additions & 3 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5787,8 +5787,7 @@ static Value *createInsertVector(
function_ref<Value *(Value *, Value *, ArrayRef<int>)> Generator = {}) {
const unsigned SubVecVF = getNumElements(V->getType());
if (Index % SubVecVF == 0) {
Vec = Builder.CreateInsertVector(Vec->getType(), Vec, V,
Builder.getInt64(Index));
Vec = Builder.CreateInsertVector(Vec->getType(), Vec, V, Index);
} else {
// Create shuffle, insertvector requires that index is multiple of
// the subvector length.
Expand Down Expand Up @@ -5818,7 +5817,7 @@ static Value *createExtractVector(IRBuilderBase &Builder, Value *Vec,
if (Index % SubVecVF == 0) {
VectorType *SubVecTy =
getWidenedType(Vec->getType()->getScalarType(), SubVecVF);
return Builder.CreateExtractVector(SubVecTy, Vec, Builder.getInt64(Index));
return Builder.CreateExtractVector(SubVecTy, Vec, Index);
}
// Create shuffle, extract_subvector requires that index is multiple of
// the subvector length.
Expand Down