Skip to content

Commit ccfe7d4

Browse files
author
Thorsten Schütt
authored
[GlobalIsel] Cleanup G_EXTRACT_VECTOR_ELT combines (#109047)
Reduce duplicated build vector patterns by exploiting variadic args. Make index parameter const to improve hit rate. Use `getIConstantFromReg` to retrieve immediate because they are not fallible anymore. Improve extraction from build vector and shuffle vector.
1 parent 8242bd1 commit ccfe7d4

File tree

4 files changed

+56
-176
lines changed

4 files changed

+56
-176
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,8 @@ class CombinerHelper {
845845
bool matchExtractVectorElement(MachineInstr &MI, BuildFnTy &MatchInfo);
846846

847847
/// Combine extract vector element with a build vector on the vector register.
848-
bool matchExtractVectorElementWithBuildVector(const MachineOperand &MO,
848+
bool matchExtractVectorElementWithBuildVector(const MachineInstr &MI,
849+
const MachineInstr &MI2,
849850
BuildFnTy &MatchInfo);
850851

851852
/// Combine extract vector element with a build vector trunc on the vector
@@ -855,7 +856,8 @@ class CombinerHelper {
855856

856857
/// Combine extract vector element with a shuffle vector on the vector
857858
/// register.
858-
bool matchExtractVectorElementWithShuffleVector(const MachineOperand &MO,
859+
bool matchExtractVectorElementWithShuffleVector(const MachineInstr &MI,
860+
const MachineInstr &MI2,
859861
BuildFnTy &MatchInfo);
860862

861863
/// Combine extract vector element with a insert vector element on the vector

‎llvm/include/llvm/Target/GlobalISel/Combine.td

Lines changed: 15 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1373,110 +1373,23 @@ def extract_vector_element_different_indices : GICombineRule<
13731373
[{ return Helper.matchExtractVectorElementWithDifferentIndices(${root}, ${matchinfo}); }]),
13741374
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
13751375

1376-
def extract_vector_element_build_vector2 : GICombineRule<
1376+
def extract_vector_element_build_vector : GICombineRule<
13771377
(defs root:$root, build_fn_matchinfo:$matchinfo),
1378-
(match (G_BUILD_VECTOR $src, $x, $y),
1379-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1380-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1381-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1382-
1383-
def extract_vector_element_build_vector3 : GICombineRule<
1384-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1385-
(match (G_BUILD_VECTOR $src, $x, $y, $z),
1386-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1387-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1388-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1389-
1390-
def extract_vector_element_build_vector4 : GICombineRule<
1391-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1392-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a),
1393-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1394-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1395-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1396-
1397-
def extract_vector_element_build_vector5 : GICombineRule<
1398-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1399-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b),
1400-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1401-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1402-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1403-
1404-
def extract_vector_element_build_vector6 : GICombineRule<
1405-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1406-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c),
1407-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1408-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1409-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1410-
1411-
def extract_vector_element_build_vector7 : GICombineRule<
1412-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1413-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d),
1414-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1415-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1416-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1417-
1418-
def extract_vector_element_build_vector8 : GICombineRule<
1419-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1420-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e),
1421-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1422-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1423-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1424-
1425-
def extract_vector_element_build_vector9 : GICombineRule<
1426-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1427-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f),
1428-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1429-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1430-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1431-
1432-
def extract_vector_element_build_vector10 : GICombineRule<
1433-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1434-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g),
1435-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1436-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1437-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1438-
1439-
def extract_vector_element_build_vector11 : GICombineRule<
1440-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1441-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h),
1442-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1443-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1444-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1445-
1446-
def extract_vector_element_build_vector12 : GICombineRule<
1447-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1448-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i),
1449-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1450-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1451-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1452-
1453-
def extract_vector_element_build_vector13 : GICombineRule<
1454-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1455-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j),
1456-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1457-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1458-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1378+
(match (G_CONSTANT $idx, $imm),
1379+
(G_BUILD_VECTOR $src, GIVariadic<>:$unused):$Build,
1380+
(G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract,
1381+
[{ return Helper.matchExtractVectorElementWithBuildVector(*${Extract}, *${Build},
1382+
${matchinfo}); }]),
1383+
(apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>;
14591384

1460-
def extract_vector_element_build_vector14 : GICombineRule<
1461-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1462-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k),
1463-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1464-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1465-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1466-
1467-
def extract_vector_element_build_vector15 : GICombineRule<
1385+
def extract_vector_element_shuffle_vector : GICombineRule<
14681386
(defs root:$root, build_fn_matchinfo:$matchinfo),
1469-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l),
1470-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1471-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1472-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1473-
1474-
def extract_vector_element_build_vector16 : GICombineRule<
1475-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1476-
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m),
1477-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1478-
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
1479-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1387+
(match (G_CONSTANT $idx, $imm),
1388+
(G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$Shuffle,
1389+
(G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract,
1390+
[{ return Helper.matchExtractVectorElementWithShuffleVector(*${Extract}, *${Shuffle},
1391+
${matchinfo}); }]),
1392+
(apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>;
14801393

14811394
def extract_vector_element_build_vector_trunc2 : GICombineRule<
14821395
(defs root:$root, build_fn_matchinfo:$matchinfo),
@@ -1547,13 +1460,6 @@ def nneg_zext : GICombineRule<
15471460
[{ return Helper.matchNonNegZext(${root}, ${matchinfo}); }]),
15481461
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
15491462

1550-
def extract_vector_element_shuffle_vector : GICombineRule<
1551-
(defs root:$root, build_fn_matchinfo:$matchinfo),
1552-
(match (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask),
1553-
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
1554-
[{ return Helper.matchExtractVectorElementWithShuffleVector(${root}, ${matchinfo}); }]),
1555-
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1556-
15571463
// Combines concat operations
15581464
def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
15591465
def combine_concat_vector : GICombineRule<
@@ -1647,20 +1553,7 @@ match_extract_of_element,
16471553
insert_vector_elt_oob,
16481554
extract_vector_element_not_const,
16491555
extract_vector_element_different_indices,
1650-
extract_vector_element_build_vector2,
1651-
extract_vector_element_build_vector3,
1652-
extract_vector_element_build_vector4,
1653-
extract_vector_element_build_vector5,
1654-
extract_vector_element_build_vector7,
1655-
extract_vector_element_build_vector8,
1656-
extract_vector_element_build_vector9,
1657-
extract_vector_element_build_vector10,
1658-
extract_vector_element_build_vector11,
1659-
extract_vector_element_build_vector12,
1660-
extract_vector_element_build_vector13,
1661-
extract_vector_element_build_vector14,
1662-
extract_vector_element_build_vector15,
1663-
extract_vector_element_build_vector16,
1556+
extract_vector_element_build_vector,
16641557
extract_vector_element_build_vector_trunc2,
16651558
extract_vector_element_build_vector_trunc3,
16661559
extract_vector_element_build_vector_trunc4,

‎llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp

Lines changed: 10 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,9 @@ bool CombinerHelper::matchExtractVectorElementWithDifferentIndices(
146146
}
147147

148148
bool CombinerHelper::matchExtractVectorElementWithBuildVector(
149-
const MachineOperand &MO, BuildFnTy &MatchInfo) {
150-
MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI);
151-
GExtractVectorElement *Extract = cast<GExtractVectorElement>(Root);
149+
const MachineInstr &MI, const MachineInstr &MI2, BuildFnTy &MatchInfo) {
150+
const GExtractVectorElement *Extract = cast<GExtractVectorElement>(&MI);
151+
const GBuildVector *Build = cast<GBuildVector>(&MI2);
152152

153153
//
154154
// %zero:_(s64) = G_CONSTANT i64 0
@@ -160,23 +160,8 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector(
160160
// %extract:_(32) = COPY %arg1(s32)
161161
//
162162
//
163-
//
164-
// %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
165-
// %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
166-
//
167-
// -->
168-
//
169-
// %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32)
170-
// %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64)
171-
//
172163

173164
Register Vector = Extract->getVectorReg();
174-
175-
// We expect a buildVector on the Vector register.
176-
GBuildVector *Build = getOpcodeDef<GBuildVector>(Vector, MRI);
177-
if (!Build)
178-
return false;
179-
180165
LLT VectorTy = MRI.getType(Vector);
181166

182167
// There is a one-use check. There are more combines on build vectors.
@@ -185,22 +170,15 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector(
185170
!getTargetLowering().aggressivelyPreferBuildVectorSources(Ty))
186171
return false;
187172

188-
Register Index = Extract->getIndexReg();
189-
190-
// If the Index is constant, then we can extract the element from the given
191-
// offset.
192-
std::optional<ValueAndVReg> MaybeIndex =
193-
getIConstantVRegValWithLookThrough(Index, MRI);
194-
if (!MaybeIndex)
195-
return false;
173+
APInt Index = getIConstantFromReg(Extract->getIndexReg(), MRI);
196174

197175
// We now know that there is a buildVector def'd on the Vector register and
198176
// the index is const. The combine will succeed.
199177

200178
Register Dst = Extract->getReg(0);
201179

202180
MatchInfo = [=](MachineIRBuilder &B) {
203-
B.buildCopy(Dst, Build->getSourceReg(MaybeIndex->Value.getZExtValue()));
181+
B.buildCopy(Dst, Build->getSourceReg(Index.getZExtValue()));
204182
};
205183

206184
return true;
@@ -274,9 +252,9 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(
274252
}
275253

276254
bool CombinerHelper::matchExtractVectorElementWithShuffleVector(
277-
const MachineOperand &MO, BuildFnTy &MatchInfo) {
278-
GExtractVectorElement *Extract =
279-
cast<GExtractVectorElement>(getDefIgnoringCopies(MO.getReg(), MRI));
255+
const MachineInstr &MI, const MachineInstr &MI2, BuildFnTy &MatchInfo) {
256+
const GExtractVectorElement *Extract = cast<GExtractVectorElement>(&MI);
257+
const GShuffleVector *Shuffle = cast<GShuffleVector>(&MI2);
280258

281259
//
282260
// %zero:_(s64) = G_CONSTANT i64 0
@@ -302,32 +280,12 @@ bool CombinerHelper::matchExtractVectorElementWithShuffleVector(
302280
// %extract:_(s32) = G_IMPLICIT_DEF
303281
//
304282
//
305-
//
306-
//
307-
//
308-
// %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
309-
// shufflemask(0, 0, 0, -1)
310-
// %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64)
311-
//
312-
// -->
313-
//
314-
// %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
315-
// shufflemask(0, 0, 0, -1)
316-
// %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64)
317-
//
318-
319-
// We try to get the value of the Index register.
320-
std::optional<ValueAndVReg> MaybeIndex =
321-
getIConstantVRegValWithLookThrough(Extract->getIndexReg(), MRI);
322-
if (!MaybeIndex)
323-
return false;
324283

325-
GShuffleVector *Shuffle =
326-
cast<GShuffleVector>(getDefIgnoringCopies(Extract->getVectorReg(), MRI));
284+
APInt Index = getIConstantFromReg(Extract->getIndexReg(), MRI);
327285

328286
ArrayRef<int> Mask = Shuffle->getMask();
329287

330-
unsigned Offset = MaybeIndex->Value.getZExtValue();
288+
unsigned Offset = Index.getZExtValue();
331289
int SrcIdx = Mask[Offset];
332290

333291
LLT Src1Type = MRI.getType(Shuffle->getSrc1Reg());

‎llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,3 +634,30 @@ body: |
634634
RET_ReallyLR implicit $x0
635635
...
636636
---
637+
name: extract_from_build_vector_const_huge
638+
alignment: 4
639+
liveins:
640+
- { reg: '$x0' }
641+
- { reg: '$x1' }
642+
frameInfo:
643+
maxAlignment: 1
644+
body: |
645+
bb.1:
646+
liveins: $x0, $x1
647+
; CHECK-LABEL: name: extract_from_build_vector_const_huge
648+
; CHECK: liveins: $x0, $x1
649+
; CHECK-NEXT: {{ $}}
650+
; CHECK-NEXT: %arg1:_(s64) = COPY $x0
651+
; CHECK-NEXT: $x0 = COPY %arg1(s64)
652+
; CHECK-NEXT: RET_ReallyLR implicit $x0
653+
%vec:_(<2 x s64>) = COPY $q0
654+
%idx:_(s64) = G_CONSTANT i64 0
655+
%arg1:_(s64) = COPY $x0
656+
%arg2:_(s64) = COPY $x1
657+
%bv:_(<18 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64)
658+
%extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<18 x s64>), %idx(s64)
659+
$x0 = COPY %extract(s64)
660+
RET_ReallyLR implicit $x0
661+
662+
...
663+
---

0 commit comments

Comments
 (0)