Skip to content

Commit e75b4a2

Browse files
davemgreendavemgreen
committed
[AArch64][GlobalISel] Legalize Insert vector element
This attempts to standardize and extend some of the insert vector element lowering. Most notably: - More types are handled by splitting illegal vectors - The index type for G_INSERT_VECTOR_ELT is canonicalized to TLI.getVectorIdxTy(), similar to extact_vector_element. - Some of the existing patterns now have the index type specified to make sure they can apply to GISel too. - The C++ selection code has been removed, relying on tablegen patterns. - G_INSERT_VECTOR_ELT with small GPR input elements are pre-selected in reg-bank-select to use a i32 type, allowing the existing patterns to apply. - Variable index inserts are lowered in post-legalizer lowering, expanding into a stack store and reload.
1 parent 8715f25 commit e75b4a2

18 files changed

+1239
-842
lines changed

‎llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ def : GINodeEquiv<G_CTLZ_ZERO_UNDEF, ctlz_zero_undef>;
141141
def : GINodeEquiv<G_CTTZ_ZERO_UNDEF, cttz_zero_undef>;
142142
def : GINodeEquiv<G_CTPOP, ctpop>;
143143
def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, extractelt>;
144+
def : GINodeEquiv<G_INSERT_VECTOR_ELT, vector_insert>;
144145
def : GINodeEquiv<G_CONCAT_VECTORS, concat_vectors>;
145146
def : GINodeEquiv<G_BUILD_VECTOR, build_vector>;
146147
def : GINodeEquiv<G_FCEIL, fceil>;

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2962,7 +2962,22 @@ bool IRTranslator::translateInsertElement(const User &U,
29622962
Register Res = getOrCreateVReg(U);
29632963
Register Val = getOrCreateVReg(*U.getOperand(0));
29642964
Register Elt = getOrCreateVReg(*U.getOperand(1));
2965-
Register Idx = getOrCreateVReg(*U.getOperand(2));
2965+
const auto &TLI = *MF->getSubtarget().getTargetLowering();
2966+
unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2967+
Register Idx;
2968+
if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(2))) {
2969+
if (CI->getBitWidth() != PreferredVecIdxWidth) {
2970+
APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
2971+
auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2972+
Idx = getOrCreateVReg(*NewIdxCI);
2973+
}
2974+
}
2975+
if (!Idx)
2976+
Idx = getOrCreateVReg(*U.getOperand(2));
2977+
if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
2978+
const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
2979+
Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
2980+
}
29662981
MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
29672982
return true;
29682983
}

‎llvm/lib/Target/AArch64/AArch64Combine.td

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,13 @@ def ext: GICombineRule <
114114
(apply [{ applyEXT(*${root}, ${matchinfo}); }])
115115
>;
116116

117+
def insertelt_nonconst: GICombineRule <
118+
(defs root:$root, shuffle_matchdata:$matchinfo),
119+
(match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
120+
[{ return matchNonConstInsert(*${root}, MRI); }]),
121+
(apply [{ applyNonConstInsert(*${root}, MRI, B); }])
122+
>;
123+
117124
def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">;
118125
def shuf_to_ins: GICombineRule <
119126
(defs root:$root, shuf_to_ins_matchdata:$matchinfo),
@@ -140,8 +147,7 @@ def form_duplane : GICombineRule <
140147
>;
141148

142149
def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn,
143-
form_duplane,
144-
shuf_to_ins]>;
150+
form_duplane, shuf_to_ins]>;
145151

146152
// Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's
147153
def vector_unmerge_lowering : GICombineRule <
@@ -269,7 +275,7 @@ def AArch64PostLegalizerLowering
269275
lower_vector_fcmp, form_truncstore,
270276
vector_sext_inreg_to_shift,
271277
unmerge_ext_to_unmerge, lower_mull,
272-
vector_unmerge_lowering]> {
278+
vector_unmerge_lowering, insertelt_nonconst]> {
273279
}
274280

275281
// Post-legalization combines which are primarily optimizations.

‎llvm/lib/Target/AArch64/AArch64InstrAtomics.td

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -547,10 +547,10 @@ let Predicates = [HasLSE] in {
547547
let Predicates = [HasRCPC3, HasNEON] in {
548548
// LDAP1 loads
549549
def : Pat<(vector_insert (v2i64 VecListOne128:$Rd),
550-
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)), VectorIndexD:$idx),
550+
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)), (i64 VectorIndexD:$idx)),
551551
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
552552
def : Pat<(vector_insert (v2f64 VecListOne128:$Rd),
553-
(f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), VectorIndexD:$idx),
553+
(f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)),
554554
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
555555
def : Pat<(v1i64 (scalar_to_vector
556556
(i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))),

‎llvm/lib/Target/AArch64/AArch64InstrFormats.td

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7983,7 +7983,7 @@ class SIMDInsFromMain<string size, ValueType vectype,
79837983
"|" # size # "\t$Rd$idx, $Rn}",
79847984
"$Rd = $dst",
79857985
[(set V128:$dst,
7986-
(vector_insert (vectype V128:$Rd), regtype:$Rn, idxtype:$idx))]> {
7986+
(vector_insert (vectype V128:$Rd), regtype:$Rn, (i64 idxtype:$idx)))]> {
79877987
let Inst{14-11} = 0b0011;
79887988
}
79897989

@@ -7997,8 +7997,8 @@ class SIMDInsFromElement<string size, ValueType vectype,
79977997
[(set V128:$dst,
79987998
(vector_insert
79997999
(vectype V128:$Rd),
8000-
(elttype (vector_extract (vectype V128:$Rn), idxtype:$idx2)),
8001-
idxtype:$idx))]>;
8000+
(elttype (vector_extract (vectype V128:$Rn), (i64 idxtype:$idx2))),
8001+
(i64 idxtype:$idx)))]>;
80028002

80038003
class SIMDInsMainMovAlias<string size, Instruction inst,
80048004
RegisterClass regtype, Operand idxtype>

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6584,6 +6584,15 @@ def : Pat<(v8i8 (vector_insert (v8i8 V64:$Rn), (i32 GPR32:$Rm), (i64 VectorIndex
65846584
VectorIndexB:$imm, GPR32:$Rm),
65856585
dsub)>;
65866586

6587+
def : Pat<(v8i8 (vector_insert (v8i8 V64:$Rn), (i8 FPR8:$Rm), (i64 VectorIndexB:$imm))),
6588+
(EXTRACT_SUBREG
6589+
(INSvi8lane (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), V64:$Rn, dsub)),
6590+
VectorIndexB:$imm, (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0)),
6591+
dsub)>;
6592+
def : Pat<(v16i8 (vector_insert (v16i8 V128:$Rn), (i8 FPR8:$Rm), (i64 VectorIndexB:$imm))),
6593+
(INSvi8lane V128:$Rn, VectorIndexB:$imm,
6594+
(v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0))>;
6595+
65876596
// Copy an element at a constant index in one vector into a constant indexed
65886597
// element of another.
65896598
// FIXME refactor to a shared class/dev parameterized on vector type, vector
@@ -6616,26 +6625,26 @@ def : Pat<(v2i64 (int_aarch64_neon_vcopy_lane
66166625
multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64,
66176626
ValueType VTScal, Instruction INS> {
66186627
def : Pat<(VT128 (vector_insert V128:$src,
6619-
(VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
6620-
imm:$Immd)),
6628+
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
6629+
(i64 imm:$Immd))),
66216630
(INS V128:$src, imm:$Immd, V128:$Rn, imm:$Immn)>;
66226631

66236632
def : Pat<(VT128 (vector_insert V128:$src,
6624-
(VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
6625-
imm:$Immd)),
6633+
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
6634+
(i64 imm:$Immd))),
66266635
(INS V128:$src, imm:$Immd,
66276636
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn)>;
66286637

66296638
def : Pat<(VT64 (vector_insert V64:$src,
6630-
(VTScal (vector_extract (VT128 V128:$Rn), imm:$Immn)),
6631-
imm:$Immd)),
6639+
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
6640+
(i64 imm:$Immd))),
66326641
(EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub),
66336642
imm:$Immd, V128:$Rn, imm:$Immn),
66346643
dsub)>;
66356644

66366645
def : Pat<(VT64 (vector_insert V64:$src,
6637-
(VTScal (vector_extract (VT64 V64:$Rn), imm:$Immn)),
6638-
imm:$Immd)),
6646+
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
6647+
(i64 imm:$Immd))),
66396648
(EXTRACT_SUBREG
66406649
(INS (SUBREG_TO_REG (i64 0), V64:$src, dsub), imm:$Immd,
66416650
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn),
@@ -6654,14 +6663,14 @@ defm : Neon_INS_elt_pattern<v2i64, v1i64, i64, INSvi64lane>;
66546663

66556664
// Insert from bitcast
66566665
// vector_insert(bitcast(f32 src), n, lane) -> INSvi32lane(src, lane, INSERT_SUBREG(-, n), 0)
6657-
def : Pat<(v4i32 (vector_insert v4i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), imm:$Immd)),
6666+
def : Pat<(v4i32 (vector_insert v4i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), (i64 imm:$Immd))),
66586667
(INSvi32lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0)>;
6659-
def : Pat<(v2i32 (vector_insert v2i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), imm:$Immd)),
6668+
def : Pat<(v2i32 (vector_insert v2i32:$src, (i32 (bitconvert (f32 FPR32:$Sn))), (i64 imm:$Immd))),
66606669
(EXTRACT_SUBREG
66616670
(INSvi32lane (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), V64:$src, dsub)),
66626671
imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0),
66636672
dsub)>;
6664-
def : Pat<(v2i64 (vector_insert v2i64:$src, (i64 (bitconvert (f64 FPR64:$Sn))), imm:$Immd)),
6673+
def : Pat<(v2i64 (vector_insert v2i64:$src, (i64 (bitconvert (f64 FPR64:$Sn))), (i64 imm:$Immd))),
66656674
(INSvi64lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$Sn, dsub), 0)>;
66666675

66676676
// bitcast of an extract
@@ -8048,7 +8057,7 @@ def : Pat<(v8bf16 (AArch64dup (bf16 (load GPR64sp:$Rn)))),
80488057
class Ld1Lane128Pat<SDPatternOperator scalar_load, Operand VecIndex,
80498058
ValueType VTy, ValueType STy, Instruction LD1>
80508059
: Pat<(vector_insert (VTy VecListOne128:$Rd),
8051-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8060+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
80528061
(LD1 VecListOne128:$Rd, VecIndex:$idx, GPR64sp:$Rn)>;
80538062

80548063
def : Ld1Lane128Pat<extloadi8, VectorIndexB, v16i8, i32, LD1i8>;
@@ -8071,14 +8080,14 @@ class Ld1Lane128IdxOpPat<SDPatternOperator scalar_load, Operand
80718080
VecIndex, ValueType VTy, ValueType STy,
80728081
Instruction LD1, SDNodeXForm IdxOp>
80738082
: Pat<(vector_insert (VTy VecListOne128:$Rd),
8074-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8083+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
80758084
(LD1 VecListOne128:$Rd, (IdxOp VecIndex:$idx), GPR64sp:$Rn)>;
80768085

80778086
class Ld1Lane64IdxOpPat<SDPatternOperator scalar_load, Operand VecIndex,
80788087
ValueType VTy, ValueType STy, Instruction LD1,
80798088
SDNodeXForm IdxOp>
80808089
: Pat<(vector_insert (VTy VecListOne64:$Rd),
8081-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8090+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
80828091
(EXTRACT_SUBREG
80838092
(LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
80848093
(IdxOp VecIndex:$idx), GPR64sp:$Rn),
@@ -8118,7 +8127,7 @@ let Predicates = [IsNeonAvailable] in {
81188127
class Ld1Lane64Pat<SDPatternOperator scalar_load, Operand VecIndex,
81198128
ValueType VTy, ValueType STy, Instruction LD1>
81208129
: Pat<(vector_insert (VTy VecListOne64:$Rd),
8121-
(STy (scalar_load GPR64sp:$Rn)), VecIndex:$idx),
8130+
(STy (scalar_load GPR64sp:$Rn)), (i64 VecIndex:$idx)),
81228131
(EXTRACT_SUBREG
81238132
(LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
81248133
VecIndex:$idx, GPR64sp:$Rn),

‎llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ class AArch64InstructionSelector : public InstructionSelector {
191191
MachineInstr *tryAdvSIMDModImmFP(Register Dst, unsigned DstSize, APInt Bits,
192192
MachineIRBuilder &MIRBuilder);
193193

194-
bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI);
195194
bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
196195
MachineRegisterInfo &MRI);
197196
/// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
@@ -3487,8 +3486,6 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
34873486
return selectShuffleVector(I, MRI);
34883487
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
34893488
return selectExtractElt(I, MRI);
3490-
case TargetOpcode::G_INSERT_VECTOR_ELT:
3491-
return selectInsertElt(I, MRI);
34923489
case TargetOpcode::G_CONCAT_VECTORS:
34933490
return selectConcatVectors(I, MRI);
34943491
case TargetOpcode::G_JUMP_TABLE:
@@ -5319,65 +5316,6 @@ bool AArch64InstructionSelector::selectUSMovFromExtend(
53195316
return true;
53205317
}
53215318

5322-
bool AArch64InstructionSelector::selectInsertElt(MachineInstr &I,
5323-
MachineRegisterInfo &MRI) {
5324-
assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
5325-
5326-
// Get information on the destination.
5327-
Register DstReg = I.getOperand(0).getReg();
5328-
const LLT DstTy = MRI.getType(DstReg);
5329-
unsigned VecSize = DstTy.getSizeInBits();
5330-
5331-
// Get information on the element we want to insert into the destination.
5332-
Register EltReg = I.getOperand(2).getReg();
5333-
const LLT EltTy = MRI.getType(EltReg);
5334-
unsigned EltSize = EltTy.getSizeInBits();
5335-
if (EltSize < 8 || EltSize > 64)
5336-
return false;
5337-
5338-
// Find the definition of the index. Bail out if it's not defined by a
5339-
// G_CONSTANT.
5340-
Register IdxReg = I.getOperand(3).getReg();
5341-
auto VRegAndVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
5342-
if (!VRegAndVal)
5343-
return false;
5344-
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
5345-
5346-
// Perform the lane insert.
5347-
Register SrcReg = I.getOperand(1).getReg();
5348-
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
5349-
5350-
if (VecSize < 128) {
5351-
// If the vector we're inserting into is smaller than 128 bits, widen it
5352-
// to 128 to do the insert.
5353-
MachineInstr *ScalarToVec =
5354-
emitScalarToVector(VecSize, &AArch64::FPR128RegClass, SrcReg, MIB);
5355-
if (!ScalarToVec)
5356-
return false;
5357-
SrcReg = ScalarToVec->getOperand(0).getReg();
5358-
}
5359-
5360-
// Create an insert into a new FPR128 register.
5361-
// Note that if our vector is already 128 bits, we end up emitting an extra
5362-
// register.
5363-
MachineInstr *InsMI =
5364-
emitLaneInsert(std::nullopt, SrcReg, EltReg, LaneIdx, EltRB, MIB);
5365-
5366-
if (VecSize < 128) {
5367-
// If we had to widen to perform the insert, then we have to demote back to
5368-
// the original size to get the result we want.
5369-
if (!emitNarrowVector(DstReg, InsMI->getOperand(0).getReg(), MIB, MRI))
5370-
return false;
5371-
} else {
5372-
// No widening needed.
5373-
InsMI->getOperand(0).setReg(DstReg);
5374-
constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
5375-
}
5376-
5377-
I.eraseFromParent();
5378-
return true;
5379-
}
5380-
53815319
MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm8(
53825320
Register Dst, unsigned DstSize, APInt Bits, MachineIRBuilder &Builder) {
53835321
unsigned int Op;

‎llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -883,8 +883,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
883883
.clampMaxNumElements(1, p0, 2);
884884

885885
getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
886-
.legalIf(typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64}))
887-
.widenVectorEltsToVectorMinSize(0, 64);
886+
.legalIf(
887+
typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
888+
.widenVectorEltsToVectorMinSize(0, 64)
889+
.moreElementsToNextPow2(0)
890+
.clampNumElements(0, v8s8, v16s8)
891+
.clampNumElements(0, v4s16, v8s16)
892+
.clampNumElements(0, v2s32, v4s32)
893+
.clampMaxNumElements(0, s64, 2)
894+
.clampMaxNumElements(0, p0, 2);
888895

889896
getActionDefinitionsBuilder(G_BUILD_VECTOR)
890897
.legalFor({{v8s8, s8},

‎llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
3737
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
3838
#include "llvm/CodeGen/GlobalISel/Utils.h"
39+
#include "llvm/CodeGen/MachineFrameInfo.h"
3940
#include "llvm/CodeGen/MachineFunctionPass.h"
4041
#include "llvm/CodeGen/MachineInstrBuilder.h"
4142
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -475,6 +476,59 @@ void applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
475476
MI.eraseFromParent();
476477
}
477478

479+
bool matchNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI) {
480+
assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
481+
482+
auto ValAndVReg =
483+
getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
484+
return !ValAndVReg;
485+
}
486+
487+
void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
488+
MachineIRBuilder &Builder) {
489+
assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
490+
bool InsertVal = true;
491+
Builder.setInstrAndDebugLoc(MI);
492+
493+
Register Offset = MI.getOperand(3).getReg();
494+
LLT VecTy = MRI.getType(MI.getOperand(0).getReg());
495+
LLT EltTy = MRI.getType(MI.getOperand(2).getReg());
496+
LLT IdxTy = MRI.getType(MI.getOperand(3).getReg());
497+
498+
// Create a stack slot and store the vector into it
499+
MachineFunction &MF = Builder.getMF();
500+
int FrameIdx = MF.getFrameInfo().CreateStackObject(VecTy.getSizeInBytes(),
501+
Align(8), false);
502+
LLT FramePtrTy = LLT::pointer(0, 64);
503+
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
504+
auto StackTemp = Builder.buildFrameIndex(FramePtrTy, FrameIdx);
505+
506+
Builder.buildStore(MI.getOperand(1), StackTemp, PtrInfo, Align(8));
507+
508+
// Get the pointer to the element, and be sure not to hit undefined behavior
509+
// if the index is out of bounds.
510+
assert(isPowerOf2_64(VecTy.getNumElements()) &&
511+
"Expected a power-2 vector size");
512+
auto Mask = Builder.buildConstant(IdxTy, VecTy.getNumElements() - 1);
513+
Register And = Builder.buildAnd(IdxTy, Offset, Mask).getReg(0);
514+
auto EltSize = Builder.buildConstant(IdxTy, EltTy.getSizeInBytes());
515+
Register Mul = Builder.buildMul(IdxTy, And, EltSize).getReg(0);
516+
Register EltPtr =
517+
Builder.buildPtrAdd(MRI.getType(StackTemp.getReg(0)), StackTemp, Mul)
518+
.getReg(0);
519+
520+
if (InsertVal) {
521+
// Write the inserted element
522+
Builder.buildStore(MI.getOperand(2).getReg(), EltPtr, PtrInfo, Align(1));
523+
524+
// Reload the whole vector.
525+
Builder.buildLoad(MI.getOperand(0).getReg(), StackTemp, PtrInfo, Align(8));
526+
} else {
527+
Builder.buildLoad(MI.getOperand(0).getReg(), EltPtr, PtrInfo, Align(1));
528+
}
529+
MI.eraseFromParent();
530+
}
531+
478532
/// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
479533
/// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
480534
///

0 commit comments

Comments
 (0)