Skip to content

Commit 7ec996d

Browse files
authored
[GlobalISel][TableGen] Support Intrinsics in MIR Patterns (llvm#79278)
1 parent 65066c0 commit 7ec996d

File tree

14 files changed

+313
-22
lines changed

14 files changed

+313
-22
lines changed

llvm/docs/GlobalISel/MIRPatterns.rst

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ MIR patterns use the DAG datatype in TableGen.
3636
3737
(inst operand0, operand1, ...)
3838
39-
``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``)
40-
or ``GICombinePatFrag``.
39+
``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``),
40+
``Intrinsic`` or ``GICombinePatFrag``.
4141

4242
Operands essentially fall into one of two categories:
4343

@@ -227,7 +227,6 @@ Limitations
227227

228228
This a non-exhaustive list of known issues with MIR patterns at this time.
229229

230-
* Matching intrinsics is not yet possible.
231230
* Using ``GICombinePatFrag`` within another ``GICombinePatFrag`` is not
232231
supported.
233232
* ``GICombinePatFrag`` can only have a single root.

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,11 @@ enum {
379379
/// - Flags(2) - Register Flags
380380
GIR_AddRegister,
381381

382+
/// Adds an intrinsic ID to the specified instruction.
383+
/// - InsnID(ULEB128) - Instruction ID to modify
384+
/// - IID(2) - Intrinsic ID
385+
GIR_AddIntrinsicID,
386+
382387
/// Marks the implicit def of a register as dead.
383388
/// - InsnID(ULEB128) - Instruction ID to modify
384389
/// - OpIdx(ULEB128) - The implicit def operand index

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1116,6 +1116,16 @@ bool GIMatchTableExecutor::executeMatchTable(
11161116
<< "], " << RegNum << ", " << RegFlags << ")\n");
11171117
break;
11181118
}
1119+
case GIR_AddIntrinsicID: {
1120+
uint64_t InsnID = readULEB();
1121+
uint16_t Value = readU16();
1122+
assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1123+
OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1124+
DEBUG_WITH_TYPE(TgtExecutor::getName(),
1125+
dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1126+
<< InsnID << "], " << Value << ")\n");
1127+
break;
1128+
}
11191129
case GIR_SetImplicitDefDead: {
11201130
uint64_t InsnID = readULEB();
11211131
uint64_t OpIdx = readULEB();
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Dummy intrinsic definitions for TableGen.
2+
3+
4+
def int_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], []>;
5+
def int_0in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [], []>;
6+
7+
def int_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrHasSideEffects]>;
8+
9+
def int_convergent_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
10+
def int_convergent_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrHasSideEffects]>;

llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def TestPF: GICombinePatFrag<
5353
(outs root:$def),
5454
(ins),
5555
[(pattern (COPY $def, $src))]>;
56-
// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction
56+
// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction or Intrinsic
5757
def eraseroot_notinstmatch: GICombineRule<
5858
(defs root:$mi),
5959
(match (TestPF $dst):$mi),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
2+
// RUN: -combiners=MyCombiner %s | \
3+
// RUN: FileCheck %s
4+
5+
include "llvm/Target/Target.td"
6+
include "llvm/Target/GlobalISel/Combine.td"
7+
8+
include "test-intrinsics.td"
9+
10+
def MyTargetISA : InstrInfo;
11+
def MyTarget : Target { let InstructionSet = MyTargetISA; }
12+
13+
def IntrinTest0 : GICombineRule<
14+
(defs root:$a),
15+
(match (int_1in_1out $a, 0)),
16+
(apply (int_1in_1out $a, $x),
17+
(int_0in_1out i32:$x))>;
18+
19+
def SpecialIntrins : GICombineRule<
20+
(defs root:$a),
21+
(match (int_sideeffects_1in_1out $a, $b)),
22+
(apply (int_convergent_1in_1out i32:$x, $b),
23+
(int_convergent_sideeffects_1in_1out $a, $x))>;
24+
25+
def MyCombiner: GICombiner<"GenMyCombiner", [
26+
IntrinTest0,
27+
SpecialIntrins
28+
]>;
29+
30+
31+
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
32+
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
33+
// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(114), GIMT_Encode2(116), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132),
34+
// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18),
35+
// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73),
36+
// CHECK-NEXT: // Label 0: @18
37+
// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 //
38+
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
39+
// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
40+
// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out),
41+
// CHECK-NEXT: // MIs[0] a
42+
// CHECK-NEXT: // No operand predicates
43+
// CHECK-NEXT: // MIs[0] Operand 2
44+
// CHECK-NEXT: GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 0,
45+
// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
46+
// CHECK-NEXT: // Combiner Rule #0: IntrinTest0
47+
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
48+
// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
49+
// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out),
50+
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
51+
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
52+
// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out),
53+
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
54+
// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
55+
// CHECK-NEXT: GIR_Done,
56+
// CHECK-NEXT: // Label 3: @72
57+
// CHECK-NEXT: GIM_Reject,
58+
// CHECK-NEXT: // Label 1: @73
59+
// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 //
60+
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
61+
// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
62+
// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out),
63+
// CHECK-NEXT: // MIs[0] a
64+
// CHECK-NEXT: // No operand predicates
65+
// CHECK-NEXT: // MIs[0] b
66+
// CHECK-NEXT: // No operand predicates
67+
// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
68+
// CHECK-NEXT: // Combiner Rule #1: SpecialIntrins
69+
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT),
70+
// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
71+
// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out),
72+
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b
73+
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS),
74+
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
75+
// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out),
76+
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
77+
// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0,
78+
// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
79+
// CHECK-NEXT: GIR_Done,
80+
// CHECK-NEXT: // Label 4: @131
81+
// CHECK-NEXT: GIM_Reject,
82+
// CHECK-NEXT: // Label 2: @132
83+
// CHECK-NEXT: GIM_Reject,
84+
// CHECK-NEXT: }; // Size: 133 bytes
85+
// CHECK-NEXT: return MatchTable0;
86+
// CHECK-NEXT: }

llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
// RUN: not llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
1+
// RUN: not llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
22
// RUN: -combiners=MyCombiner %s 2>&1| \
33
// RUN: FileCheck %s -implicit-check-not=error:
44

55
include "llvm/Target/Target.td"
66
include "llvm/Target/GlobalISel/Combine.td"
77

8+
include "test-intrinsics.td"
9+
810
def MyTargetISA : InstrInfo;
911
def MyTarget : Target { let InstructionSet = MyTargetISA; }
1012

@@ -248,6 +250,13 @@ def miflags_in_builtin : GICombineRule<
248250
(match (COPY $x, $y)),
249251
(apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
250252

253+
// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns
254+
// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))'
255+
def miflags_in_intrin : GICombineRule<
256+
(defs root:$x),
257+
(match (int_1in_1out $x, $y)),
258+
(apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
259+
251260
// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions
252261
// CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor
253262
def using_flagref_in_match : GICombineRule<
@@ -300,6 +309,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
300309
not_miflagenum_1,
301310
not_miflagenum_2,
302311
miflags_in_builtin,
312+
miflags_in_intrin,
303313
using_flagref_in_match,
304314
badflagref_in_apply
305315
]>;

llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
1+
// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
22
// RUN: -gicombiner-stop-after-parse -combiners=MyCombiner %s | \
33
// RUN: FileCheck %s
44

55
include "llvm/Target/Target.td"
66
include "llvm/Target/GlobalISel/Combine.td"
77

8+
include "test-intrinsics.td"
9+
810
def MyTargetISA : InstrInfo;
911
def MyTarget : Target { let InstructionSet = MyTargetISA; }
1012

@@ -342,6 +344,48 @@ def MIFlagsTest : GICombineRule<
342344
(match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi),
343345
(apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>;
344346

347+
// CHECK-NEXT: (CombineRule name:IntrinTest0 id:12 root:a
348+
// CHECK-NEXT: (MatchPats
349+
// CHECK-NEXT: <match_root>__IntrinTest0_match_0:(CodeGenInstructionPattern G_INTRINSIC operands:[<def>$a, $b] intrinsic(@llvm.1in.1out))
350+
// CHECK-NEXT: )
351+
// CHECK-NEXT: (ApplyPats
352+
// CHECK-NEXT: <apply_root>__IntrinTest0_apply_0:(CodeGenInstructionPattern G_INTRINSIC_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.sideeffects.1in.1out))
353+
// CHECK-NEXT: )
354+
// CHECK-NEXT: (OperandTable MatchPats
355+
// CHECK-NEXT: a -> __IntrinTest0_match_0
356+
// CHECK-NEXT: b -> <live-in>
357+
// CHECK-NEXT: )
358+
// CHECK-NEXT: (OperandTable ApplyPats
359+
// CHECK-NEXT: a -> __IntrinTest0_apply_0
360+
// CHECK-NEXT: b -> <live-in>
361+
// CHECK-NEXT: )
362+
// CHECK-NEXT: )
363+
def IntrinTest0 : GICombineRule<
364+
(defs root:$a),
365+
(match (int_1in_1out $a, $b)),
366+
(apply (int_sideeffects_1in_1out $a, $b))>;
367+
368+
// CHECK: (CombineRule name:IntrinTest1 id:13 root:a
369+
// CHECK-NEXT: (MatchPats
370+
// CHECK-NEXT: <match_root>__IntrinTest1_match_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT operands:[<def>$a, $b] intrinsic(@llvm.convergent.1in.1out))
371+
// CHECK-NEXT: )
372+
// CHECK-NEXT: (ApplyPats
373+
// CHECK-NEXT: <apply_root>__IntrinTest1_apply_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.convergent.sideeffects.1in.1out))
374+
// CHECK-NEXT: )
375+
// CHECK-NEXT: (OperandTable MatchPats
376+
// CHECK-NEXT: a -> __IntrinTest1_match_0
377+
// CHECK-NEXT: b -> <live-in>
378+
// CHECK-NEXT: )
379+
// CHECK-NEXT: (OperandTable ApplyPats
380+
// CHECK-NEXT: a -> __IntrinTest1_apply_0
381+
// CHECK-NEXT: b -> <live-in>
382+
// CHECK-NEXT: )
383+
// CHECK-NEXT: )
384+
def IntrinTest1 : GICombineRule<
385+
(defs root:$a),
386+
(match (int_convergent_1in_1out $a, $b)),
387+
(apply (int_convergent_sideeffects_1in_1out $a, $b))>;
388+
345389
def MyCombiner: GICombiner<"GenMyCombiner", [
346390
WipOpcodeTest0,
347391
WipOpcodeTest1,
@@ -354,5 +398,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
354398
VariadicsInTest,
355399
VariadicsOutTest,
356400
TypeOfTest,
357-
MIFlagsTest
401+
MIFlagsTest,
402+
IntrinTest0,
403+
IntrinTest1
358404
]>;

llvm/test/TableGen/lit.local.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
config.suffixes = [".td"]
2-
config.excludes = ["Common"]
2+
config.excludes = ["Common", "Inputs"]

llvm/utils/TableGen/GlobalISel/Patterns.cpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
#include "Patterns.h"
1010
#include "../CodeGenInstruction.h"
11+
#include "../CodeGenIntrinsics.h"
1112
#include "CXXPredicates.h"
1213
#include "CodeExpander.h"
1314
#include "CodeExpansions.h"
@@ -331,7 +332,7 @@ bool CodeGenInstructionPattern::is(StringRef OpcodeName) const {
331332
}
332333

333334
bool CodeGenInstructionPattern::isVariadic() const {
334-
return I.Operands.isVariadic;
335+
return !isIntrinsic() && I.Operands.isVariadic;
335336
}
336337

337338
bool CodeGenInstructionPattern::hasVariadicDefs() const {
@@ -352,6 +353,9 @@ bool CodeGenInstructionPattern::hasVariadicDefs() const {
352353
}
353354

354355
unsigned CodeGenInstructionPattern::getNumInstDefs() const {
356+
if (isIntrinsic())
357+
return IntrinInfo->IS.RetTys.size();
358+
355359
if (!isVariadic() || !hasVariadicDefs())
356360
return I.Operands.NumDefs;
357361
unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs;
@@ -360,6 +364,9 @@ unsigned CodeGenInstructionPattern::getNumInstDefs() const {
360364
}
361365

362366
unsigned CodeGenInstructionPattern::getNumInstOperands() const {
367+
if (isIntrinsic())
368+
return IntrinInfo->IS.RetTys.size() + IntrinInfo->IS.ParamTys.size();
369+
363370
unsigned NumCGIOps = I.Operands.size();
364371
return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size())
365372
: NumCGIOps;
@@ -376,6 +383,9 @@ StringRef CodeGenInstructionPattern::getInstName() const {
376383
}
377384

378385
void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const {
386+
if (isIntrinsic())
387+
OS << " intrinsic(@" << IntrinInfo->Name << ")";
388+
379389
if (!FI)
380390
return;
381391

llvm/utils/TableGen/GlobalISel/Patterns.h

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class SMLoc;
3434
class StringInit;
3535
class CodeExpansions;
3636
class CodeGenInstruction;
37+
struct CodeGenIntrinsic;
3738

3839
namespace gi {
3940

@@ -396,7 +397,7 @@ class OperandTable {
396397
StringMap<InstructionPattern *> Table;
397398
};
398399

399-
//===- CodeGenInstructionPattern ------------------------------------------===//
400+
//===- MIFlagsInfo --------------------------------------------------------===//
400401

401402
/// Helper class to contain data associated with a MIFlags operand.
402403
class MIFlagsInfo {
@@ -413,7 +414,17 @@ class MIFlagsInfo {
413414
SetVector<StringRef> SetF, UnsetF, CopyF;
414415
};
415416

416-
/// Matches an instruction, e.g. `G_ADD $x, $y, $z`.
417+
//===- CodeGenInstructionPattern ------------------------------------------===//
418+
419+
/// Matches an instruction or intrinsic:
420+
/// e.g. `G_ADD $x, $y, $z` or `int_amdgcn_cos $a`
421+
///
422+
/// Intrinsics are just normal instructions with a special operand for intrinsic
423+
/// ID. Despite G_INTRINSIC opcodes being variadic, we consider that the
424+
/// Intrinsic's info takes priority. This means we return:
425+
/// - false for isVariadic() and other variadic-related queries.
426+
/// - getNumInstDefs and getNumInstOperands use the intrinsic's in/out
427+
/// operands.
417428
class CodeGenInstructionPattern : public InstructionPattern {
418429
public:
419430
CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name)
@@ -425,6 +436,10 @@ class CodeGenInstructionPattern : public InstructionPattern {
425436

426437
bool is(StringRef OpcodeName) const;
427438

439+
void setIntrinsic(const CodeGenIntrinsic *I) { IntrinInfo = I; }
440+
const CodeGenIntrinsic *getIntrinsic() const { return IntrinInfo; }
441+
bool isIntrinsic() const { return IntrinInfo; }
442+
428443
bool hasVariadicDefs() const;
429444
bool isVariadic() const override;
430445
unsigned getNumInstDefs() const override;
@@ -440,6 +455,7 @@ class CodeGenInstructionPattern : public InstructionPattern {
440455
void printExtras(raw_ostream &OS) const override;
441456

442457
const CodeGenInstruction &I;
458+
const CodeGenIntrinsic *IntrinInfo = nullptr;
443459
std::unique_ptr<MIFlagsInfo> FI;
444460
};
445461

0 commit comments

Comments
 (0)