[GlobalISel][TableGen] Support Intrinsics in MIR Patterns

llvm/docs/GlobalISel/MIRPatterns.rst

@@ -36,8 +36,8 @@ MIR patterns use the DAG datatype in TableGen.
 
   (inst operand0, operand1, ...)
 
-``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``)
-or ``GICombinePatFrag``.
+``inst`` must be a def which inherits from ``Instruction`` (e.g. ``G_FADD``),
+``Intrinsic`` or ``GICombinePatFrag``.
 
 Operands essentially fall into one of two categories:
 
@@ -227,7 +227,6 @@ Limitations
 
 This a non-exhaustive list of known issues with MIR patterns at this time.
 
-* Matching intrinsics is not yet possible.
 * Using ``GICombinePatFrag`` within another ``GICombinePatFrag`` is not
   supported.
 * ``GICombinePatFrag`` can only have a single root.

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h

@@ -379,6 +379,11 @@ enum {
   /// - Flags(2) - Register Flags
   GIR_AddRegister,
 
+  /// Adds an intrinsic ID to the specified instruction.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - IID(2) - Intrinsic ID
+  GIR_AddIntrinsicID,
+
   /// Marks the implicit def of a register as dead.
   /// - InsnID(ULEB128) - Instruction ID to modify
   /// - OpIdx(ULEB128) - The implicit def operand index

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h

@@ -1116,6 +1116,16 @@ bool GIMatchTableExecutor::executeMatchTable(
                           << "], " << RegNum << ", " << RegFlags << ")\n");
       break;
     }
+    case GIR_AddIntrinsicID: {
+      uint64_t InsnID = readULEB();
+      uint16_t Value = readU16();
+      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
+      OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
+                             << InsnID << "], " << Value << ")\n");
+      break;
+    }
     case GIR_SetImplicitDefDead: {
       uint64_t InsnID = readULEB();
       uint64_t OpIdx = readULEB();

llvm/test/TableGen/GlobalISelCombinerEmitter/Inputs/test-intrinsics.td

@@ -0,0 +1,10 @@
+// Dummy intrinsic definitions for TableGen.
+
+
+def int_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], []>;
+def int_0in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [], []>;
+
+def int_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrHasSideEffects]>;
+
+def int_convergent_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
+def int_convergent_sideeffects_1in_1out : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrHasSideEffects]>;

llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/builtin-pattern-errors.td

@@ -53,7 +53,7 @@ def TestPF: GICombinePatFrag<
     (outs root:$def),
     (ins),
     [(pattern (COPY $def, $src))]>;
-// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: GIEraseRoot can only be used if the root is a CodeGenInstruction or Intrinsic
 def eraseroot_notinstmatch: GICombineRule<
   (defs root:$mi),
   (match (TestPF $dst):$mi),

llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td

@@ -0,0 +1,86 @@
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
+// RUN:     -combiners=MyCombiner %s | \
+// RUN: FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+include "test-intrinsics.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def IntrinTest0 : GICombineRule<
+  (defs root:$a),
+  (match (int_1in_1out $a, 0)),
+  (apply (int_1in_1out $a, $x),
+         (int_0in_1out i32:$x))>;
+
+def SpecialIntrins : GICombineRule<
+  (defs root:$a),
+  (match (int_sideeffects_1in_1out $a, $b)),
+  (apply (int_convergent_1in_1out i32:$x, $b),
+         (int_convergent_sideeffects_1in_1out $a, $x))>;
+
+def MyCombiner: GICombiner<"GenMyCombiner", [
+  IntrinTest0,
+  SpecialIntrins
+]>;
+
+
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(114), GIMT_Encode2(116), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132),
+// CHECK-NEXT:     /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18),
+// CHECK-NEXT:     /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73),
+// CHECK-NEXT:     // Label 0: @18
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] Operand 2
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 0,
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT:       // Combiner Rule #0: IntrinTest0
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out),
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 3: @72
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 1: @73
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT:       // Combiner Rule #1: SpecialIntrins
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT),
+// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
+// CHECK-NEXT:       GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 4: @131
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 2: @132
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     }; // Size: 133 bytes
+// CHECK-NEXT:   return MatchTable0;
+// CHECK-NEXT: }

llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td

@@ -1,10 +1,12 @@
-// RUN: not llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: not llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
 // RUN:     -combiners=MyCombiner %s 2>&1| \
 // RUN: FileCheck %s -implicit-check-not=error:
 
 include "llvm/Target/Target.td"
 include "llvm/Target/GlobalISel/Combine.td"
 
+include "test-intrinsics.td"
+
 def MyTargetISA : InstrInfo;
 def MyTarget : Target { let InstructionSet = MyTargetISA; }
 
@@ -248,6 +250,13 @@ def miflags_in_builtin : GICombineRule<
   (match (COPY $x, $y)),
   (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
 
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))'
+def miflags_in_intrin : GICombineRule<
+  (defs root:$x),
+  (match (int_1in_1out $x, $y)),
+  (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>;
+
 // CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor
 def using_flagref_in_match : GICombineRule<
@@ -300,6 +309,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   not_miflagenum_1,
   not_miflagenum_2,
   miflags_in_builtin,
+  miflags_in_intrin,
   using_flagref_in_match,
   badflagref_in_apply
 ]>;

llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td

@@ -1,10 +1,12 @@
-// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN: llvm-tblgen -I %S/Inputs -I %p/../../../include -gen-global-isel-combiner \
 // RUN:     -gicombiner-stop-after-parse -combiners=MyCombiner %s | \
 // RUN: FileCheck %s
 
 include "llvm/Target/Target.td"
 include "llvm/Target/GlobalISel/Combine.td"
 
+include "test-intrinsics.td"
+
 def MyTargetISA : InstrInfo;
 def MyTarget : Target { let InstructionSet = MyTargetISA; }
 
@@ -342,6 +344,48 @@ def MIFlagsTest : GICombineRule<
   (match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi),
   (apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>;
 
+// CHECK-NEXT: (CombineRule name:IntrinTest0 id:12 root:a
+// CHECK-NEXT:   (MatchPats
+// CHECK-NEXT:     <match_root>__IntrinTest0_match_0:(CodeGenInstructionPattern G_INTRINSIC operands:[<def>$a, $b] intrinsic(@llvm.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (ApplyPats
+// CHECK-NEXT:     <apply_root>__IntrinTest0_apply_0:(CodeGenInstructionPattern G_INTRINSIC_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.sideeffects.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable MatchPats
+// CHECK-NEXT:     a -> __IntrinTest0_match_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable ApplyPats
+// CHECK-NEXT:     a -> __IntrinTest0_apply_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT: )
+def IntrinTest0 : GICombineRule<
+  (defs root:$a),
+  (match (int_1in_1out $a, $b)),
+  (apply (int_sideeffects_1in_1out $a, $b))>;
+
+// CHECK:      (CombineRule name:IntrinTest1 id:13 root:a
+// CHECK-NEXT:   (MatchPats
+// CHECK-NEXT:     <match_root>__IntrinTest1_match_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT operands:[<def>$a, $b] intrinsic(@llvm.convergent.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (ApplyPats
+// CHECK-NEXT:     <apply_root>__IntrinTest1_apply_0:(CodeGenInstructionPattern G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS operands:[<def>$a, $b] intrinsic(@llvm.convergent.sideeffects.1in.1out))
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable MatchPats
+// CHECK-NEXT:     a -> __IntrinTest1_match_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable ApplyPats
+// CHECK-NEXT:     a -> __IntrinTest1_apply_0
+// CHECK-NEXT:     b -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT: )
+def IntrinTest1 : GICombineRule<
+  (defs root:$a),
+  (match (int_convergent_1in_1out $a, $b)),
+  (apply (int_convergent_sideeffects_1in_1out $a, $b))>;
+
 def MyCombiner: GICombiner<"GenMyCombiner", [
   WipOpcodeTest0,
   WipOpcodeTest1,
@@ -354,5 +398,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   VariadicsInTest,
   VariadicsOutTest,
   TypeOfTest,
-  MIFlagsTest
+  MIFlagsTest,
+  IntrinTest0,
+  IntrinTest1
 ]>;

llvm/test/TableGen/lit.local.cfg

@@ -1,2 +1,2 @@
 config.suffixes = [".td"]
-config.excludes = ["Common"]
+config.excludes = ["Common", "Inputs"]

llvm/utils/TableGen/GlobalISel/Patterns.cpp

@@ -8,6 +8,7 @@
 
 #include "Patterns.h"
 #include "../CodeGenInstruction.h"
+#include "../CodeGenIntrinsics.h"
 #include "CXXPredicates.h"
 #include "CodeExpander.h"
 #include "CodeExpansions.h"
@@ -331,7 +332,7 @@ bool CodeGenInstructionPattern::is(StringRef OpcodeName) const {
 }
 
 bool CodeGenInstructionPattern::isVariadic() const {
-  return I.Operands.isVariadic;
+  return !isIntrinsic() && I.Operands.isVariadic;
 }
 
 bool CodeGenInstructionPattern::hasVariadicDefs() const {
@@ -352,6 +353,9 @@ bool CodeGenInstructionPattern::hasVariadicDefs() const {
 }
 
 unsigned CodeGenInstructionPattern::getNumInstDefs() const {
+  if (isIntrinsic())
+    return IntrinInfo->IS.RetTys.size();
+
   if (!isVariadic() || !hasVariadicDefs())
     return I.Operands.NumDefs;
   unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs;
@@ -360,6 +364,9 @@ unsigned CodeGenInstructionPattern::getNumInstDefs() const {
 }
 
 unsigned CodeGenInstructionPattern::getNumInstOperands() const {
+  if (isIntrinsic())
+    return IntrinInfo->IS.RetTys.size() + IntrinInfo->IS.ParamTys.size();
+
   unsigned NumCGIOps = I.Operands.size();
   return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size())
                       : NumCGIOps;
@@ -376,6 +383,9 @@ StringRef CodeGenInstructionPattern::getInstName() const {
 }
 
 void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const {
+  if (isIntrinsic())
+    OS << " intrinsic(@" << IntrinInfo->Name << ")";
+
   if (!FI)
     return;
 

llvm/utils/TableGen/GlobalISel/Patterns.h

@@ -34,6 +34,7 @@ class SMLoc;
 class StringInit;
 class CodeExpansions;
 class CodeGenInstruction;
+struct CodeGenIntrinsic;
 
 namespace gi {
 
@@ -396,7 +397,7 @@ class OperandTable {
   StringMap<InstructionPattern *> Table;
 };
 
-//===- CodeGenInstructionPattern ------------------------------------------===//
+//===- MIFlagsInfo --------------------------------------------------------===//
 
 /// Helper class to contain data associated with a MIFlags operand.
 class MIFlagsInfo {
@@ -413,7 +414,17 @@ class MIFlagsInfo {
   SetVector<StringRef> SetF, UnsetF, CopyF;
 };
 
-/// Matches an instruction, e.g. `G_ADD $x, $y, $z`.
+//===- CodeGenInstructionPattern ------------------------------------------===//
+
+/// Matches an instruction or intrinsic:
+///    e.g. `G_ADD $x, $y, $z` or `int_amdgcn_cos $a`
+///
+/// Intrinsics are just normal instructions with a special operand for intrinsic
+/// ID. Despite G_INTRINSIC opcodes being variadic, we consider that the
+/// Intrinsic's info takes priority. This means we return:
+///   - false for isVariadic() and other variadic-related queries.
+///   - getNumInstDefs and getNumInstOperands use the intrinsic's in/out
+///   operands.
 class CodeGenInstructionPattern : public InstructionPattern {
 public:
   CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name)
@@ -425,6 +436,10 @@ class CodeGenInstructionPattern : public InstructionPattern {
 
   bool is(StringRef OpcodeName) const;
 
+  void setIntrinsic(const CodeGenIntrinsic *I) { IntrinInfo = I; }
+  const CodeGenIntrinsic *getIntrinsic() const { return IntrinInfo; }
+  bool isIntrinsic() const { return IntrinInfo; }
+
   bool hasVariadicDefs() const;
   bool isVariadic() const override;
   unsigned getNumInstDefs() const override;
@@ -440,6 +455,7 @@ class CodeGenInstructionPattern : public InstructionPattern {
   void printExtras(raw_ostream &OS) const override;
 
   const CodeGenInstruction &I;
+  const CodeGenIntrinsic *IntrinInfo = nullptr;
   std::unique_ptr<MIFlagsInfo> FI;
 };