[GlobalISel][TableGen] MIR Pattern Variadics (#100563)

Allow for matching & rewriting a variable number of arguments in an
instructions.

Solves #87459

Author: Pierre-vh

llvm/docs/GlobalISel/MIRPatterns.rst

@@ -115,7 +115,7 @@ GITypeOf
 ``GITypeOf<"$x">`` is a ``GISpecialType`` that allows for the creation of a
 register or immediate with the same type as another (register) operand.
 
-Operand:
+Type Parameters:
 
 * An operand name as a string, prefixed by ``$``.
 
@@ -143,6 +143,59 @@ Semantics:
     (apply (G_FSUB $dst, $src, $tmp),
            (G_FNEG GITypeOf<"$dst">:$tmp, $src))>;
 
+GIVariadic
+~~~~~~~~~~
+
+``GIVariadic<>`` is a ``GISpecialType`` that allows for matching 1 or
+more operands remaining on an instruction.
+
+Type Parameters:
+
+* The minimum number of additional operands to match. Must be greater than zero.
+
+  * Default is 1.
+
+* The maximum number of additional operands to match. Must be strictly greater
+  than the minimum.
+
+  * 0 can be used to indicate there is no upper limit.
+  * Default is 0.
+
+Semantics:
+
+* ``GIVariadic<>`` operands can only appear on variadic instructions.
+* ``GIVariadic<>`` operands cannot be defs.
+* ``GIVariadic<>`` operands can only appear as the last operand in a 'match' pattern.
+* Each instance within a 'match' pattern must be uniquely named.
+* Re-using a ``GIVariadic<>`` operand in an 'apply' pattern will result in all
+  the matched operands being copied from the original instruction.
+* The min/max operands will result in the matcher checking that the number of operands
+  falls within that range.
+* ``GIVariadic<>`` operands can be used in C++ code within a rule, which will
+  result in the operand name being expanded to a value of type ``ArrayRef<MachineOperand>``.
+
+.. code-block:: text
+
+  // bool checkBuildVectorToUnmerge(ArrayRef<MachineOperand>);
+
+  def build_vector_to_unmerge: GICombineRule <
+    (defs root:$root),
+    (match (G_BUILD_VECTOR $root, GIVariadic<>:$args),
+           [{ return checkBuildVectorToUnmerge(${args}); }]),
+    (apply (G_UNMERGE_VALUES $root, $args))
+  >;
+
+.. code-block:: text
+
+  // Will additionally check the number of operands is >= 3 and <= 5.
+  // ($root is one operand, then 2 to 4 variadic operands).
+  def build_vector_to_unmerge: GICombineRule <
+    (defs root:$root),
+    (match (G_BUILD_VECTOR $root, GIVariadic<2, 4>:$two_to_four),
+           [{ return checkBuildVectorToUnmerge(${two_to_four}); }]),
+    (apply (G_UNMERGE_VALUES $root, $two_to_four))
+  >;
+
 Builtin Operations
 ------------------
 
@@ -240,6 +293,8 @@ This a non-exhaustive list of known issues with MIR patterns at this time.
   match. e.g. if a pattern needs to work on both i32 and i64, you either
   need to leave it untyped and check the type in C++, or duplicate the
   pattern.
+* ``GISpecialType`` operands are not allowed within a ``GICombinePatFrag``.
+* ``GIVariadic<>`` matched operands must each have a unique name.
 
 GICombineRule
 -------------

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h

@@ -15,11 +15,13 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTOR_H
 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTOR_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Bitset.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGenTypes/LowLevelType.h"
 #include "llvm/IR/Function.h"
 #include <bitset>
@@ -133,6 +135,12 @@ enum {
   /// - Ops(ULEB128) - Expected number of operands
   GIM_CheckNumOperands,
 
+  /// Check the instruction has a number of operands <= or >= than given number.
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Ops(ULEB128) - Number of operands
+  GIM_CheckNumOperandsLE,
+  GIM_CheckNumOperandsGE,
+
   /// Check an immediate predicate on the specified instruction
   /// - InsnID(ULEB128) - Instruction ID
   /// - Pred(2) - The predicate to test
@@ -294,12 +302,15 @@ enum {
   /// Check the specified operands are identical.
   /// The IgnoreCopies variant looks through COPY instructions before
   /// comparing the operands.
+  /// The "All" variants check all operands starting from the index.
   /// - InsnID(ULEB128) - Instruction ID
   /// - OpIdx(ULEB128) - Operand index
   /// - OtherInsnID(ULEB128) - Other instruction ID
   /// - OtherOpIdx(ULEB128) - Other operand index
   GIM_CheckIsSameOperand,
   GIM_CheckIsSameOperandIgnoreCopies,
+  GIM_CheckAllSameOperand,
+  GIM_CheckAllSameOperandIgnoreCopies,
 
   /// Check we can replace all uses of a register with another.
   /// - OldInsnID(ULEB128)
@@ -362,6 +373,13 @@ enum {
   /// GIR_Copy but with both New/OldInsnIDs omitted and defaulting to zero.
   GIR_RootToRootCopy,
 
+  /// Copies all operand starting from OpIdx in OldInsnID into the new
+  /// instruction NewInsnID.
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
+  /// - OpIdx(ULEB128) - The first operand to copy
+  GIR_CopyRemaining,
+
   /// Copy an operand to the specified instruction or add a zero register if the
   /// operand is a zero immediate.
   /// - NewInsnID(ULEB128) - Instruction ID to modify
@@ -713,6 +731,12 @@ class GIMatchTableExecutor {
     return Ret;
   }
 
+  static ArrayRef<MachineOperand> getRemainingOperands(const MachineInstr &MI,
+                                                       unsigned FirstVarOp) {
+    auto Operands = drop_begin(MI.operands(), FirstVarOp);
+    return {Operands.begin(), Operands.end()};
+  }
+
 public:
   // Faster ULEB128 decoder tailored for the Match Table Executor.
   //

llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h

@@ -309,6 +309,23 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
+    case GIM_CheckNumOperandsGE:
+    case GIM_CheckNumOperandsLE: {
+      uint64_t InsnID = readULEB();
+      uint64_t Expected = readULEB();
+      const bool IsLE = (MatcherOpcode == GIM_CheckNumOperandsLE);
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands"
+                             << (IsLE ? "LE" : "GE") << "(MIs[" << InsnID
+                             << "], Expected=" << Expected << ")\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      const unsigned NumOps = State.MIs[InsnID]->getNumOperands();
+      if (IsLE ? (NumOps <= Expected) : (NumOps >= Expected)) {
+        if (handleReject() == RejectAndGiveUp)
+          return false;
+      }
+      break;
+    }
     case GIM_CheckNumOperands: {
       uint64_t InsnID = readULEB();
       uint64_t Expected = readULEB();
@@ -1081,6 +1098,22 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
+    case GIR_CopyRemaining: {
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
+      MachineInstr &OldMI = *State.MIs[OldInsnID];
+      MachineInstrBuilder &NewMI = OutMIs[NewInsnID];
+      for (const auto &Op : drop_begin(OldMI.operands(), OpIdx))
+        NewMI.add(Op);
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx << ": GIR_CopyRemaining(OutMIs["
+                             << NewInsnID << "], MIs[" << OldInsnID
+                             << "], /*start=*/" << OpIdx << ")\n");
+      break;
+    }
+
     case GIR_CopyOrAddZeroReg: {
       uint64_t NewInsnID = readULEB();
       uint64_t OldInsnID = readULEB();

llvm/include/llvm/Target/GlobalISel/Combine.td

@@ -128,6 +128,16 @@ class GITypeOf<string opName> : GISpecialType {
   string OpName = opName;
 }
 
+// The type of an operand that can match a variable amount of operands.
+// This type contains a minimum and maximum number of operands to match.
+// The minimum must be 1 or more, as we cannot have an operand representing
+// zero operands, and the max can be zero (which means "unlimited") or a value
+// greater than the minimum.
+class GIVariadic<int min = 1, int max = 0> : GISpecialType {
+  int MinArgs = min;
+  int MaxArgs = max;
+}
+
 //===----------------------------------------------------------------------===//
 // Pattern Builtins
 //===----------------------------------------------------------------------===//

llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td

@@ -28,19 +28,48 @@ def InstTest3 : GICombineRule<
   (match (G_UNMERGE_VALUES $a, $b, $c, $d)),
   (apply [{ APPLY }])>;
 
+def VariadicTypeTestCxx : GICombineRule<
+  (defs root:$a),
+  (match (G_BUILD_VECTOR $a, GIVariadic<2, 4>:$b)),
+  (apply [{ ${b} }])>;
+
+def VariadicTypeTestReuse : GICombineRule<
+  (defs root:$a),
+  (match (G_BUILD_VECTOR $a, $c, GIVariadic<2, 4>:$b)),
+  (apply (G_MERGE_VALUES $a, $b, $c))>;
+
 def MyCombiner: GICombiner<"GenMyCombiner", [
   InstTest0,
   InstTest1,
   InstTest2,
-  InstTest3
+  InstTest3,
+  VariadicTypeTestCxx,
+  VariadicTypeTestReuse
 ]>;
 
+// CHECK:     bool GenMyCombiner::runCustomAction(unsigned ApplyID, const MatcherState &State, NewMIVector &OutMIs) const {
+// CHECK-NEXT:   Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
+// CHECK-NEXT:   switch(ApplyID) {
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner0:{
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner1:{
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     getRemainingOperands(*State.MIs[0], 1)
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   llvm_unreachable("Unknown Apply Action");
+// CHECK-NEXT: }
+
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2([[#LOWER:]]), GIMT_Encode2([[#UPPER:]]), /*)*//*default:*//*Label 2*/ GIMT_Encode4([[#DEFAULT:]]),
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(70), GIMT_Encode2(74), /*)*//*default:*//*Label 2*/ GIMT_Encode4(127),
 // CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0),
 // CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(55),
-// CHECK-NEXT:     // Label 0: @[[#%u, mul(UPPER-LOWER, 4) + 10]]
+// CHECK-NEXT:     // Label 0: @26
 // CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(40), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
@@ -77,7 +106,35 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // Combiner Rule #1: InstTest1
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
 // CHECK-NEXT:     // Label 5: @69
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(83), // Rule ID 0 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(86), // Rule ID 4 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperandsGE, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckNumOperandsLE, /*MI*/0, /*Expected*/5,
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // Combiner Rule #4: VariadicTypeTestCxx
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
+// CHECK-NEXT:     // Label 6: @86
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(112), // Rule ID 5 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
+// CHECK-NEXT:       GIM_CheckNumOperandsGE, /*MI*/0, /*Expected*/4,
+// CHECK-NEXT:       GIM_CheckNumOperandsLE, /*MI*/0, /*Expected*/6,
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] c
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // Combiner Rule #5: VariadicTypeTestReuse
+// CHECK-NEXT:       GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_MERGE_VALUES),
+// CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/0, // a
+// CHECK-NEXT:       GIR_CopyRemaining, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // b
+// CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/1, // c
+// CHECK-NEXT:       GIR_EraseRootFromParent_Done,
+// CHECK-NEXT:     // Label 7: @112
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(126), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
 // CHECK-NEXT:       // MIs[0] a
@@ -90,10 +147,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: InstTest0
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 6: @83
+// CHECK-NEXT:     // Label 8: @126
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @[[#%u, DEFAULT]]
+// CHECK-NEXT:     // Label 2: @127
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: [[#%u, DEFAULT + 1]] bytes
+// CHECK-NEXT:     }; // Size: 128 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }

llvm/test/TableGen/GlobalISelCombinerEmitter/operand-types.td

@@ -104,8 +104,32 @@ def TypeOfProp : GICombineRule<
   (apply (G_ANYEXT $x, GITypeOf<"$y">:$tmp),
          (G_ANYEXT $tmp, $y))>;
 
+// CHECK:      (CombineRule name:VariadicTypeTest id:3 root:a
+// CHECK-NEXT:   (MatchPats
+// CHECK-NEXT:     <match_root>__VariadicTypeTest_match_0:(CodeGenInstructionPattern G_UNMERGE_VALUES operands:[<def>$a, <def>$b, GIVariadic<1,0>:$z])
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (ApplyPats
+// CHECK-NEXT:     <apply_root>__VariadicTypeTest_apply_0:(CodeGenInstructionPattern G_UNMERGE_VALUES operands:[<def>$a, <def>$b, GIVariadic<1,0>:$z])
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable MatchPats
+// CHECK-NEXT:     a -> __VariadicTypeTest_match_0
+// CHECK-NEXT:     b -> __VariadicTypeTest_match_0
+// CHECK-NEXT:     z -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT:   (OperandTable ApplyPats
+// CHECK-NEXT:     a -> __VariadicTypeTest_apply_0
+// CHECK-NEXT:     b -> __VariadicTypeTest_apply_0
+// CHECK-NEXT:     z -> <live-in>
+// CHECK-NEXT:   )
+// CHECK-NEXT: )
+def VariadicTypeTest: GICombineRule<
+  (defs root:$a),
+  (match (G_UNMERGE_VALUES $a, $b, GIVariadic<>:$z)),
+  (apply (G_UNMERGE_VALUES $a, $b, $z))>;
+
 def MyCombiner: GICombiner<"GenMyCombiner", [
   InstTest0,
   PatFragTest0,
-  TypeOfProp
+  TypeOfProp,
+  VariadicTypeTest,
 ]>;

llvm/test/TableGen/GlobalISelCombinerEmitter/typeof-errors.td

@@ -21,7 +21,7 @@ def UnknownOperand : GICombineRule<
   (match (G_ZEXT $dst, $src)),
   (apply (G_ANYEXT $dst, (GITypeOf<"$unknown"> 0)))>;
 
-// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GISpecialType is not supported in 'match' patterns
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GITypeOf is not supported in 'match' patterns
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: operand 1 of '__UseInMatch_match_0' has type 'GITypeOf<$dst>'
 def UseInMatch : GICombineRule<
   (defs root:$dst),
@@ -41,7 +41,7 @@ def UseInPF: GICombineRule<
   (match (PFWithTypeOF $dst)),
   (apply (G_ANYEXT $dst, (i32 0)))>;
 
-// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GISpecialType is not supported in 'match' patterns
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GITypeOf is not supported in 'match' patterns
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: operand 1 of '__InferredUseInMatch_match_0' has type 'GITypeOf<$dst>'
 def InferredUseInMatch : GICombineRule<
   (defs root:$dst),
@@ -63,6 +63,13 @@ def TypeOfApplyTmp : GICombineRule<
   (apply (G_ANYEXT $dst, i32:$tmp),
          (G_ANYEXT $tmp, (GITypeOf<"$tmp"> 0)))>;
 
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: type 'GITypeOf<$src>' is ill-formed: 'src' is a variadic pack operand
+def TypeOfVariadic : GICombineRule<
+  (defs root:$dst),
+  (match (G_BUILD_VECTOR $dst,  $x, GIVariadic<>:$src)),
+  (apply (G_ANYEXT GITypeOf<"$src">:$tmp, $x),
+         (G_ANYEXT $dst, $tmp))>;
+
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse one or more rules
 def MyCombiner: GICombiner<"GenMyCombiner", [
   NoDollarSign,
@@ -71,5 +78,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   UseInPF,
   InferredUseInMatch,
   InferenceConflict,
-  TypeOfApplyTmp
+  TypeOfApplyTmp,
+  TypeOfVariadic
 ]>;