Skip to content

Commit 66e02c5

Browse files
davemgreendavemgreen
committed
[AArch64][GlobalISel] Legalize Insert vector element
This attempts to standardize and extend some of the insert vector element lowering. Most notably: - More types are handled by splitting illegal vectors - The index type for G_INSERT_VECTOR_ELT is canonicalized to TLI.getVectorIdxTy(), similar to extact_vector_element. - Some of the existing patterns now have the index type specified to make sure they can apply to GISel too. - The C++ selection code has been removed, relying on tablegen patterns. - G_INSERT_VECTOR_ELT with small GPR input elements are pre-selected in reg-bank-select to use a i32 type, allowing the existing patterns to apply. - Variable index inserts are lowered in post-legalizer lowering, expanding into a stack store and reload.
1 parent d36d805 commit 66e02c5

18 files changed

+1233
-842
lines changed

‎llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ def : GINodeEquiv<G_CTLZ_ZERO_UNDEF, ctlz_zero_undef>;
141141
def : GINodeEquiv<G_CTTZ_ZERO_UNDEF, cttz_zero_undef>;
142142
def : GINodeEquiv<G_CTPOP, ctpop>;
143143
def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, extractelt>;
144+
def : GINodeEquiv<G_INSERT_VECTOR_ELT, vector_insert>;
144145
def : GINodeEquiv<G_CONCAT_VECTORS, concat_vectors>;
145146
def : GINodeEquiv<G_BUILD_VECTOR, build_vector>;
146147
def : GINodeEquiv<G_FCEIL, fceil>;

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2964,7 +2964,22 @@ bool IRTranslator::translateInsertElement(const User &U,
29642964
Register Res = getOrCreateVReg(U);
29652965
Register Val = getOrCreateVReg(*U.getOperand(0));
29662966
Register Elt = getOrCreateVReg(*U.getOperand(1));
2967-
Register Idx = getOrCreateVReg(*U.getOperand(2));
2967+
const auto &TLI = *MF->getSubtarget().getTargetLowering();
2968+
unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2969+
Register Idx;
2970+
if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(2))) {
2971+
if (CI->getBitWidth() != PreferredVecIdxWidth) {
2972+
APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
2973+
auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2974+
Idx = getOrCreateVReg(*NewIdxCI);
2975+
}
2976+
}
2977+
if (!Idx)
2978+
Idx = getOrCreateVReg(*U.getOperand(2));
2979+
if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
2980+
const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
2981+
Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
2982+
}
29682983
MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
29692984
return true;
29702985
}

‎llvm/lib/Target/AArch64/AArch64Combine.td

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,13 @@ def ext: GICombineRule <
114114
(apply [{ applyEXT(*${root}, ${matchinfo}); }])
115115
>;
116116

117+
def insertelt_nonconst: GICombineRule <
118+
(defs root:$root, shuffle_matchdata:$matchinfo),
119+
(match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
120+
[{ return matchNonConstInsert(*${root}, MRI); }]),
121+
(apply [{ applyNonConstInsert(*${root}, MRI, B); }])
122+
>;
123+
117124
def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">;
118125
def shuf_to_ins: GICombineRule <
119126
(defs root:$root, shuf_to_ins_matchdata:$matchinfo),
@@ -140,8 +147,7 @@ def form_duplane : GICombineRule <
140147
>;
141148

142149
def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn,
143-
form_duplane,
144-
shuf_to_ins]>;
150+
form_duplane, shuf_to_ins]>;
145151

146152
// Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's
147153
def vector_unmerge_lowering : GICombineRule <
@@ -269,7 +275,7 @@ def AArch64PostLegalizerLowering
269275
lower_vector_fcmp, form_truncstore,
270276
vector_sext_inreg_to_shift,
271277
unmerge_ext_to_unmerge, lower_mull,
272-
vector_unmerge_lowering]> {
278+
vector_unmerge_lowering, insertelt_nonconst]> {
273279
}
274280

275281
// Post-legalization combines which are primarily optimizations.

‎llvm/lib/Target/AArch64/AArch64InstrAtomics.td

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -547,10 +547,10 @@ let Predicates = [HasLSE] in {
547547
let Predicates = [HasRCPC3, HasNEON] in {
548548
// LDAP1 loads
549549
def : Pat<(vector_insert (v2i64 VecListOne128:$Rd),
550-
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)), VectorIndexD:$idx),
550+
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)), (i64 VectorIndexD:$idx)),
551551
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
552552
def : Pat<(vector_insert (v2f64 VecListOne128:$Rd),
553-
(f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), VectorIndexD:$idx),
553+
(f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)),
554554
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
555555
def : Pat<(v1i64 (scalar_to_vector
556556
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))),

‎llvm/lib/Target/AArch64/AArch64InstrFormats.td

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7983,7 +7983,7 @@ class SIMDInsFromMain<string size, ValueType vectype,
79837983
"|" # size # "\t$Rd$idx, $Rn}",
79847984
"$Rd = $dst",
79857985
[(set V128:$dst,
7986-
(vector_insert (vectype V128:$Rd), regtype:$Rn, idxtype:$idx))]> {
7986+
(vector_insert (vectype V128:$Rd), regtype:$Rn, (i64 idxtype:$idx)))]> {
79877987
let Inst{14-11} = 0b0011;
79887988
}
79897989

@@ -7997,8 +7997,8 @@ class SIMDInsFromElement<string size, ValueType vectype,
79977997
[(set V128:$dst,
79987998
(vector_insert
79997999
(vectype V128:$Rd),
8000-
(elttype (vector_extract (vectype V128:$Rn), idxtype:$idx2)),
8001-
idxtype:$idx))]>;
8000+
(elttype (vector_extract (vectype V128:$Rn), (i64 idxtype:$idx2))),
8001+
(i64 idxtype:$idx)))]>;
80028002

80038003
class SIMDInsMainMovAlias<string size, Instruction inst,
80048004
RegisterClass regtype, Operand idxtype>

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6581,6 +6581,15 @@ def : Pat<(v8i8 (vector_insert (v8i8 V64:$Rn), (i32 GPR32:$Rm), (i64 VectorIndex
65816581
VectorIndexB:$imm, GPR32:$Rm),
65826582
dsub)>;
65836583

6584+
def : Pat<(v8i8 (vector_insert (v8i8 V64:$Rn), (i8 FPR8:$Rm), (i64 VectorIndexB:$imm))),
6585+
(EXTRACT_SUBREG
6586+
(INSvi8lane (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), V64:$Rn, dsub)),
6587+
VectorIndexB:$imm, (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0)),
6588+
dsub)>;
6589+
def : Pat<(v16i8 (vector_insert (v16i8 V128:$Rn), (i8 FPR8:$Rm), (i64 VectorIndexB:$imm))),
6590+
(INSvi8lane V128:$Rn, VectorIndexB:$imm,
6591+
(v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0))>;
6592+
65846593
// Copy an element at a constant index in one vector into a constant indexed
65856594
// element of another.
65866595
// FIXME refactor to a shared class/dev parameterized on vector type, vector
@@ -6613,26 +6622,26 @@ def : Pat<(v2i64 (int_aarch64_neon_vcopy_lane
66136622
multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64,
66146623
ValueType VTScal, Instruction INS> {
66156624
def : Pat<(VT128 (vector_insert V128:$src,
6616-
(VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
6617-
imm:$Immd)),
6625+
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
6626+
(i64 imm:$Immd))),
66186627
(INS V128:$src, imm:$Immd, V128:$Rn, imm:$Immn)>;
66196628

66206629
def : Pat<(VT128 (vector_insert V128:$src,
6621-
(VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
6622-
imm:$Immd)),
6630+
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
6631+
(i64 imm:$Immd))),
66236632
(INS V128:$src, imm:$Immd,
66246633
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn)>;
66256634

66266635
def : Pat<(VT64 (vector_insert V64:$src,
6627-
(VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
6628-
imm:$Immd)),
6636+
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
6637+
(i64 imm:$Immd))),
66296638
(EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub),
66306639
imm:$Immd, V128:$Rn, imm:$Immn),
66316640
dsub)>;
66326641

66336642
def : Pat<(VT64 (vector_insert V64:$src,
6634-
(VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
6635-
imm:$Immd)),
6643+
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
6644+
(i64 imm:$Immd))),
66366645
(EXTRACT_SUBREG
66376646
(INS (SUBREG_TO_REG (i64 0), V64:$src, dsub), imm:$Immd,
66386647
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn),
@@ -6651,14 +6660,14 @@ defm : Neon_INS_elt_pattern<v2i64, v1i64, i64, INSvi64lane>;
66516660

66526661
// Insert from bitcast
66536662
// vector_insert(bitcast(f32 src), n, lane) -> INSvi32lane(src, lane, INSERT_SUBREG(-, n), 0)
6654-
def : Pat<(v4i32 (vector_insert v4i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), imm:$Immd)),
6663+
def : Pat<(v4i32 (vector_insert v4i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), (i64 imm:$Immd))),
66556664
(INSvi32lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0)>;
6656-
def : Pat<(v2i32 (vector_insert v2i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), imm:$Immd)),
6665+
def : Pat<(v2i32 (vector_insert v2i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), (i64 imm:$Immd))),
66576666
(EXTRACT_SUBREG
66586667
(INSvi32lane (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), V64:$src, dsub)),
66596668
imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0),
66606669
dsub)>;
6661-
def : Pat<(v2i64 (vector_insert v2i64:$src, (i64 (bitconvert (f64 FPR64:$Sn))), imm:$Immd)),
6670+
def : Pat<(v2i64 (vector_insert v2i64:$src, (i64 (bitconvert (f64 FPR64:$Sn))), (i64 imm:$Immd))),
66626671
(INSvi64lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$Sn, dsub), 0)>;
66636672

66646673
// bitcast of an extract
@@ -8080,7 +8089,7 @@ def : Pat<(v8bf16 (AArch64dup (bf16 (load GPR64sp:$Rn)))),
80808089
class Ld1Lane128Pat<SDPatternOperator scalar_load, Operand VecIndex,
80818090
ValueType VTy, ValueType STy, Instruction LD1>
80828091
: Pat<(vector_insert (VTy VecListOne128:$Rd),
8083-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8092+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
80848093
(LD1 VecListOne128:$Rd, VecIndex:$idx, GPR64sp:$Rn)>;
80858094

80868095
def : Ld1Lane128Pat<extloadi8, VectorIndexB, v16i8, i32, LD1i8>;
@@ -8103,14 +8112,14 @@ class Ld1Lane128IdxOpPat<SDPatternOperator scalar_load, Operand
81038112
VecIndex, ValueType VTy, ValueType STy,
81048113
Instruction LD1, SDNodeXForm IdxOp>
81058114
: Pat<(vector_insert (VTy VecListOne128:$Rd),
8106-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8115+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
81078116
(LD1 VecListOne128:$Rd, (IdxOp VecIndex:$idx), GPR64sp:$Rn)>;
81088117

81098118
class Ld1Lane64IdxOpPat<SDPatternOperator scalar_load, Operand VecIndex,
81108119
ValueType VTy, ValueType STy, Instruction LD1,
81118120
SDNodeXForm IdxOp>
81128121
: Pat<(vector_insert (VTy VecListOne64:$Rd),
8113-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8122+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
81148123
(EXTRACT_SUBREG
81158124
(LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
81168125
(IdxOp VecIndex:$idx), GPR64sp:$Rn),
@@ -8150,7 +8159,7 @@ let Predicates = [IsNeonAvailable] in {
81508159
class Ld1Lane64Pat<SDPatternOperator scalar_load, Operand VecIndex,
81518160
ValueType VTy, ValueType STy, Instruction LD1>
81528161
: Pat<(vector_insert (VTy VecListOne64:$Rd),
8153-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8162+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
81548163
(EXTRACT_SUBREG
81558164
(LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
81568165
VecIndex:$idx, GPR64sp:$Rn),

‎llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ class AArch64InstructionSelector : public InstructionSelector {
191191
MachineInstr *tryAdvSIMDModImmFP(Register Dst, unsigned DstSize, APInt Bits,
192192
MachineIRBuilder &MIRBuilder);
193193

194-
bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI);
195194
bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
196195
MachineRegisterInfo &MRI);
197196
/// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
@@ -3496,8 +3495,6 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
34963495
return selectShuffleVector(I, MRI);
34973496
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
34983497
return selectExtractElt(I, MRI);
3499-
case TargetOpcode::G_INSERT_VECTOR_ELT:
3500-
return selectInsertElt(I, MRI);
35013498
case TargetOpcode::G_CONCAT_VECTORS:
35023499
return selectConcatVectors(I, MRI);
35033500
case TargetOpcode::G_JUMP_TABLE:
@@ -5328,65 +5325,6 @@ bool AArch64InstructionSelector::selectUSMovFromExtend(
53285325
return true;
53295326
}
53305327

5331-
bool AArch64InstructionSelector::selectInsertElt(MachineInstr &I,
5332-
MachineRegisterInfo &MRI) {
5333-
assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
5334-
5335-
// Get information on the destination.
5336-
Register DstReg = I.getOperand(0).getReg();
5337-
const LLT DstTy = MRI.getType(DstReg);
5338-
unsigned VecSize = DstTy.getSizeInBits();
5339-
5340-
// Get information on the element we want to insert into the destination.
5341-
Register EltReg = I.getOperand(2).getReg();
5342-
const LLT EltTy = MRI.getType(EltReg);
5343-
unsigned EltSize = EltTy.getSizeInBits();
5344-
if (EltSize < 8 || EltSize > 64)
5345-
return false;
5346-
5347-
// Find the definition of the index. Bail out if it's not defined by a
5348-
// G_CONSTANT.
5349-
Register IdxReg = I.getOperand(3).getReg();
5350-
auto VRegAndVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
5351-
if (!VRegAndVal)
5352-
return false;
5353-
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
5354-
5355-
// Perform the lane insert.
5356-
Register SrcReg = I.getOperand(1).getReg();
5357-
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
5358-
5359-
if (VecSize < 128) {
5360-
// If the vector we're inserting into is smaller than 128 bits, widen it
5361-
// to 128 to do the insert.
5362-
MachineInstr *ScalarToVec =
5363-
emitScalarToVector(VecSize, &AArch64::FPR128RegClass, SrcReg, MIB);
5364-
if (!ScalarToVec)
5365-
return false;
5366-
SrcReg = ScalarToVec->getOperand(0).getReg();
5367-
}
5368-
5369-
// Create an insert into a new FPR128 register.
5370-
// Note that if our vector is already 128 bits, we end up emitting an extra
5371-
// register.
5372-
MachineInstr *InsMI =
5373-
emitLaneInsert(std::nullopt, SrcReg, EltReg, LaneIdx, EltRB, MIB);
5374-
5375-
if (VecSize < 128) {
5376-
// If we had to widen to perform the insert, then we have to demote back to
5377-
// the original size to get the result we want.
5378-
if (!emitNarrowVector(DstReg, InsMI->getOperand(0).getReg(), MIB, MRI))
5379-
return false;
5380-
} else {
5381-
// No widening needed.
5382-
InsMI->getOperand(0).setReg(DstReg);
5383-
constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
5384-
}
5385-
5386-
I.eraseFromParent();
5387-
return true;
5388-
}
5389-
53905328
MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm8(
53915329
Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
53925330
unsigned int Op;

‎llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -883,9 +883,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
883883
.clampMaxNumElements(1, p0, 2);
884884

885885
getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
886-
.legalIf(typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64}))
886+
.legalIf(
887+
typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
887888
.moreElementsToNextPow2(0)
888-
.widenVectorEltsToVectorMinSize(0, 64);
889+
.widenVectorEltsToVectorMinSize(0, 64)
890+
.clampNumElements(0, v8s8, v16s8)
891+
.clampNumElements(0, v4s16, v8s16)
892+
.clampNumElements(0, v2s32, v4s32)
893+
.clampMaxNumElements(0, s64, 2)
894+
.clampMaxNumElements(0, p0, 2);
889895

890896
getActionDefinitionsBuilder(G_BUILD_VECTOR)
891897
.legalFor({{v8s8, s8},

‎llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
3737
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
3838
#include "llvm/CodeGen/GlobalISel/Utils.h"
39+
#include "llvm/CodeGen/MachineFrameInfo.h"
3940
#include "llvm/CodeGen/MachineFunctionPass.h"
4041
#include "llvm/CodeGen/MachineInstrBuilder.h"
4142
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -475,6 +476,54 @@ void applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
475476
MI.eraseFromParent();
476477
}
477478

479+
bool matchNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI) {
480+
assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
481+
482+
auto ValAndVReg =
483+
getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
484+
return !ValAndVReg;
485+
}
486+
487+
void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
488+
MachineIRBuilder &Builder) {
489+
assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
490+
bool InsertVal = true;
491+
Builder.setInstrAndDebugLoc(MI);
492+
493+
Register Offset = MI.getOperand(3).getReg();
494+
LLT VecTy = MRI.getType(MI.getOperand(0).getReg());
495+
LLT EltTy = MRI.getType(MI.getOperand(2).getReg());
496+
LLT IdxTy = MRI.getType(MI.getOperand(3).getReg());
497+
498+
// Create a stack slot and store the vector into it
499+
MachineFunction &MF = Builder.getMF();
500+
int FrameIdx = MF.getFrameInfo().CreateStackObject(VecTy.getSizeInBytes(),
501+
Align(8), false);
502+
LLT FramePtrTy = LLT::pointer(0, 64);
503+
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
504+
auto StackTemp = Builder.buildFrameIndex(FramePtrTy, FrameIdx);
505+
506+
Builder.buildStore(MI.getOperand(1), StackTemp, PtrInfo, Align(8));
507+
508+
// Get the pointer to the element, and be sure not to hit undefined behavior
509+
// if the index is out of bounds.
510+
assert(isPowerOf2_64(VecTy.getNumElements()) &&
511+
"Expected a power-2 vector size");
512+
auto Mask = Builder.buildConstant(IdxTy, VecTy.getNumElements() - 1);
513+
Register And = Builder.buildAnd(IdxTy, Offset, Mask).getReg(0);
514+
auto EltSize = Builder.buildConstant(IdxTy, EltTy.getSizeInBytes());
515+
Register Mul = Builder.buildMul(IdxTy, And, EltSize).getReg(0);
516+
Register EltPtr =
517+
Builder.buildPtrAdd(MRI.getType(StackTemp.getReg(0)), StackTemp, Mul)
518+
.getReg(0);
519+
520+
// Write the inserted element
521+
Builder.buildStore(MI.getOperand(2).getReg(), EltPtr, PtrInfo, Align(1));
522+
// Reload the whole vector.
523+
Builder.buildLoad(MI.getOperand(0).getReg(), StackTemp, PtrInfo, Align(8));
524+
MI.eraseFromParent();
525+
}
526+
478527
/// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
479528
/// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
480529
///

0 commit comments

Comments
 (0)