Skip to content

Commit 09854f2

Browse files
mgehre-amdmgehre-amd
committed
[SelectionDAG] Emit calls to __divei4 and friends for division/remainder of large integers
Emit calls to __divei4 and friends for divison/remainder of large integers. This fixes llvm#44994. The overall RFC is in https://discourse.llvm.org/t/rfc-add-support-for-division-of-large-bitint-builtins-selectiondag-globalisel-clang/60329 The compiler-rt part is in https://reviews.llvm.org/D120327 Differential Revision: https://reviews.llvm.org/D120329
1 parent f3cbe60 commit 09854f2

File tree

5 files changed

+1529
-34
lines changed

5 files changed

+1529
-34
lines changed

‎llvm/include/llvm/IR/RuntimeLibcalls.def

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ HANDLE_LIBCALL(MUL_I16, "__mulhi3")
4747
HANDLE_LIBCALL(MUL_I32, "__mulsi3")
4848
HANDLE_LIBCALL(MUL_I64, "__muldi3")
4949
HANDLE_LIBCALL(MUL_I128, "__multi3")
50+
HANDLE_LIBCALL(MUL_IEXT, nullptr)
51+
5052
HANDLE_LIBCALL(MULO_I32, "__mulosi4")
5153
HANDLE_LIBCALL(MULO_I64, "__mulodi4")
5254
HANDLE_LIBCALL(MULO_I128, "__muloti4")
@@ -55,31 +57,43 @@ HANDLE_LIBCALL(SDIV_I16, "__divhi3")
5557
HANDLE_LIBCALL(SDIV_I32, "__divsi3")
5658
HANDLE_LIBCALL(SDIV_I64, "__divdi3")
5759
HANDLE_LIBCALL(SDIV_I128, "__divti3")
60+
HANDLE_LIBCALL(SDIV_IEXT, "__divei4")
61+
5862
HANDLE_LIBCALL(UDIV_I8, "__udivqi3")
5963
HANDLE_LIBCALL(UDIV_I16, "__udivhi3")
6064
HANDLE_LIBCALL(UDIV_I32, "__udivsi3")
6165
HANDLE_LIBCALL(UDIV_I64, "__udivdi3")
6266
HANDLE_LIBCALL(UDIV_I128, "__udivti3")
67+
HANDLE_LIBCALL(UDIV_IEXT, "__udivei4")
68+
6369
HANDLE_LIBCALL(SREM_I8, "__modqi3")
6470
HANDLE_LIBCALL(SREM_I16, "__modhi3")
6571
HANDLE_LIBCALL(SREM_I32, "__modsi3")
6672
HANDLE_LIBCALL(SREM_I64, "__moddi3")
6773
HANDLE_LIBCALL(SREM_I128, "__modti3")
74+
HANDLE_LIBCALL(SREM_IEXT, "__modei4")
75+
6876
HANDLE_LIBCALL(UREM_I8, "__umodqi3")
6977
HANDLE_LIBCALL(UREM_I16, "__umodhi3")
7078
HANDLE_LIBCALL(UREM_I32, "__umodsi3")
7179
HANDLE_LIBCALL(UREM_I64, "__umoddi3")
7280
HANDLE_LIBCALL(UREM_I128, "__umodti3")
81+
HANDLE_LIBCALL(UREM_IEXT, "__umodei4")
82+
7383
HANDLE_LIBCALL(SDIVREM_I8, nullptr)
7484
HANDLE_LIBCALL(SDIVREM_I16, nullptr)
7585
HANDLE_LIBCALL(SDIVREM_I32, nullptr)
7686
HANDLE_LIBCALL(SDIVREM_I64, nullptr)
7787
HANDLE_LIBCALL(SDIVREM_I128, nullptr)
88+
HANDLE_LIBCALL(SDIVREM_IEXT, nullptr)
89+
7890
HANDLE_LIBCALL(UDIVREM_I8, nullptr)
7991
HANDLE_LIBCALL(UDIVREM_I16, nullptr)
8092
HANDLE_LIBCALL(UDIVREM_I32, nullptr)
8193
HANDLE_LIBCALL(UDIVREM_I64, nullptr)
8294
HANDLE_LIBCALL(UDIVREM_I128, nullptr)
95+
HANDLE_LIBCALL(UDIVREM_IEXT, nullptr)
96+
8397
HANDLE_LIBCALL(NEG_I32, "__negsi2")
8498
HANDLE_LIBCALL(NEG_I64, "__negdi2")
8599
HANDLE_LIBCALL(CTLZ_I32, "__clzsi2")

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 33 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -141,12 +141,10 @@ class SelectionDAGLegalize {
141141
RTLIB::Libcall Call_F128,
142142
RTLIB::Libcall Call_PPCF128,
143143
SmallVectorImpl<SDValue> &Results);
144-
SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
145-
RTLIB::Libcall Call_I8,
146-
RTLIB::Libcall Call_I16,
147-
RTLIB::Libcall Call_I32,
148-
RTLIB::Libcall Call_I64,
149-
RTLIB::Libcall Call_I128);
144+
SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
145+
RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
146+
RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128,
147+
RTLIB::Libcall Call_IEXT);
150148
void ExpandArgFPLibCall(SDNode *Node,
151149
RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
152150
RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
@@ -2105,15 +2103,17 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
21052103
ExpandFPLibCall(Node, LC, Results);
21062104
}
21072105

2108-
SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2109-
RTLIB::Libcall Call_I8,
2110-
RTLIB::Libcall Call_I16,
2111-
RTLIB::Libcall Call_I32,
2112-
RTLIB::Libcall Call_I64,
2113-
RTLIB::Libcall Call_I128) {
2106+
SDValue SelectionDAGLegalize::ExpandIntLibCall(
2107+
SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
2108+
RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32, RTLIB::Libcall Call_I64,
2109+
RTLIB::Libcall Call_I128, RTLIB::Libcall Call_IEXT) {
21142110
RTLIB::Libcall LC;
21152111
switch (Node->getSimpleValueType(0).SimpleTy) {
2116-
default: llvm_unreachable("Unexpected request for libcall!");
2112+
2113+
default:
2114+
LC = Call_IEXT;
2115+
break;
2116+
21172117
case MVT::i8: LC = Call_I8; break;
21182118
case MVT::i16: LC = Call_I16; break;
21192119
case MVT::i32: LC = Call_I32; break;
@@ -2148,7 +2148,11 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
21482148

21492149
RTLIB::Libcall LC;
21502150
switch (Node->getSimpleValueType(0).SimpleTy) {
2151-
default: llvm_unreachable("Unexpected request for libcall!");
2151+
2152+
default:
2153+
LC = isSigned ? RTLIB::SDIVREM_IEXT : RTLIB::UDIVREM_IEXT;
2154+
break;
2155+
21522156
case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
21532157
case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
21542158
case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
@@ -4319,39 +4323,34 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
43194323
RTLIB::SUB_PPCF128, Results);
43204324
break;
43214325
case ISD::SREM:
4322-
Results.push_back(ExpandIntLibCall(Node, true,
4323-
RTLIB::SREM_I8,
4324-
RTLIB::SREM_I16, RTLIB::SREM_I32,
4325-
RTLIB::SREM_I64, RTLIB::SREM_I128));
4326+
Results.push_back(ExpandIntLibCall(
4327+
Node, true, RTLIB::SREM_I8, RTLIB::SREM_I16, RTLIB::SREM_I32,
4328+
RTLIB::SREM_I64, RTLIB::SREM_I128, RTLIB::SREM_IEXT));
43264329
break;
43274330
case ISD::UREM:
4328-
Results.push_back(ExpandIntLibCall(Node, false,
4329-
RTLIB::UREM_I8,
4330-
RTLIB::UREM_I16, RTLIB::UREM_I32,
4331-
RTLIB::UREM_I64, RTLIB::UREM_I128));
4331+
Results.push_back(ExpandIntLibCall(
4332+
Node, false, RTLIB::UREM_I8, RTLIB::UREM_I16, RTLIB::UREM_I32,
4333+
RTLIB::UREM_I64, RTLIB::UREM_I128, RTLIB::UREM_IEXT));
43324334
break;
43334335
case ISD::SDIV:
4334-
Results.push_back(ExpandIntLibCall(Node, true,
4335-
RTLIB::SDIV_I8,
4336-
RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4337-
RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4336+
Results.push_back(ExpandIntLibCall(
4337+
Node, true, RTLIB::SDIV_I8, RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4338+
RTLIB::SDIV_I64, RTLIB::SDIV_I128, RTLIB::SDIV_IEXT));
43384339
break;
43394340
case ISD::UDIV:
4340-
Results.push_back(ExpandIntLibCall(Node, false,
4341-
RTLIB::UDIV_I8,
4342-
RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4343-
RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4341+
Results.push_back(ExpandIntLibCall(
4342+
Node, false, RTLIB::UDIV_I8, RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4343+
RTLIB::UDIV_I64, RTLIB::UDIV_I128, RTLIB::UDIV_IEXT));
43444344
break;
43454345
case ISD::SDIVREM:
43464346
case ISD::UDIVREM:
43474347
// Expand into divrem libcall
43484348
ExpandDivRemLibCall(Node, Results);
43494349
break;
43504350
case ISD::MUL:
4351-
Results.push_back(ExpandIntLibCall(Node, false,
4352-
RTLIB::MUL_I8,
4353-
RTLIB::MUL_I16, RTLIB::MUL_I32,
4354-
RTLIB::MUL_I64, RTLIB::MUL_I128));
4351+
Results.push_back(ExpandIntLibCall(
4352+
Node, false, RTLIB::MUL_I8, RTLIB::MUL_I16, RTLIB::MUL_I32,
4353+
RTLIB::MUL_I64, RTLIB::MUL_I128, RTLIB::MUL_IEXT));
43554354
break;
43564355
case ISD::CTLZ_ZERO_UNDEF:
43574356
switch (Node->getSimpleValueType(0).SimpleTy) {

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3914,6 +3914,70 @@ void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
39143914
ReplaceValueWith(SDValue(Node, 1), Ovf);
39153915
}
39163916

3917+
// Emit a call to __udivei4 and friends which require
3918+
// the arguments be based on the stack
3919+
// and extra argument that contains the number of bits of the operands.
3920+
// Returns the result of the call operation.
3921+
static SDValue ExpandExtIntRes_DIVREM(const TargetLowering &TLI,
3922+
const RTLIB::Libcall &LC,
3923+
SelectionDAG &DAG, SDNode *N,
3924+
const SDLoc &DL, const EVT &VT) {
3925+
3926+
SDValue InChain = DAG.getEntryNode();
3927+
3928+
TargetLowering::ArgListTy Args;
3929+
TargetLowering::ArgListEntry Entry;
3930+
3931+
// The signature of __udivei4 is
3932+
// void __udivei4(unsigned int *quo, unsigned int *a, unsigned int *b,
3933+
// unsigned int bits)
3934+
EVT ArgVT = N->op_begin()->getValueType();
3935+
assert(ArgVT.isInteger() && ArgVT.getSizeInBits() > 128 &&
3936+
"Unexpected argument type for lowering");
3937+
Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3938+
3939+
SDValue Output = DAG.CreateStackTemporary(ArgVT);
3940+
Entry.Node = Output;
3941+
Entry.Ty = ArgTy->getPointerTo();
3942+
Entry.IsSExt = false;
3943+
Entry.IsZExt = false;
3944+
Args.push_back(Entry);
3945+
3946+
for (const llvm::SDUse &Op : N->ops()) {
3947+
SDValue StackPtr = DAG.CreateStackTemporary(ArgVT);
3948+
InChain = DAG.getStore(InChain, DL, Op, StackPtr, MachinePointerInfo());
3949+
Entry.Node = StackPtr;
3950+
Entry.Ty = ArgTy->getPointerTo();
3951+
Entry.IsSExt = false;
3952+
Entry.IsZExt = false;
3953+
Args.push_back(Entry);
3954+
}
3955+
3956+
int Bits = N->getOperand(0)
3957+
.getValueType()
3958+
.getTypeForEVT(*DAG.getContext())
3959+
->getIntegerBitWidth();
3960+
Entry.Node = DAG.getConstant(Bits, DL, TLI.getPointerTy(DAG.getDataLayout()));
3961+
Entry.Ty = Type::getInt32Ty(*DAG.getContext());
3962+
Entry.IsSExt = false;
3963+
Entry.IsZExt = true;
3964+
Args.push_back(Entry);
3965+
3966+
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
3967+
TLI.getPointerTy(DAG.getDataLayout()));
3968+
3969+
TargetLowering::CallLoweringInfo CLI(DAG);
3970+
CLI.setDebugLoc(DL)
3971+
.setChain(InChain)
3972+
.setLibCallee(TLI.getLibcallCallingConv(LC),
3973+
Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args))
3974+
.setDiscardResult();
3975+
3976+
SDValue Chain = TLI.LowerCallTo(CLI).second;
3977+
3978+
return DAG.getLoad(ArgVT, DL, Chain, Output, MachinePointerInfo());
3979+
}
3980+
39173981
void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
39183982
SDValue &Lo, SDValue &Hi) {
39193983
EVT VT = N->getValueType(0);
@@ -3935,6 +3999,14 @@ void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
39353999
LC = RTLIB::SDIV_I64;
39364000
else if (VT == MVT::i128)
39374001
LC = RTLIB::SDIV_I128;
4002+
4003+
else {
4004+
SDValue Result =
4005+
ExpandExtIntRes_DIVREM(TLI, RTLIB::SDIV_IEXT, DAG, N, dl, VT);
4006+
SplitInteger(Result, Lo, Hi);
4007+
return;
4008+
}
4009+
39384010
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
39394011

39404012
TargetLowering::MakeLibCallOptions CallOptions;
@@ -4126,6 +4198,14 @@ void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
41264198
LC = RTLIB::SREM_I64;
41274199
else if (VT == MVT::i128)
41284200
LC = RTLIB::SREM_I128;
4201+
4202+
else {
4203+
SDValue Result =
4204+
ExpandExtIntRes_DIVREM(TLI, RTLIB::SREM_IEXT, DAG, N, dl, VT);
4205+
SplitInteger(Result, Lo, Hi);
4206+
return;
4207+
}
4208+
41294209
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
41304210

41314211
TargetLowering::MakeLibCallOptions CallOptions;
@@ -4301,6 +4381,14 @@ void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
43014381
LC = RTLIB::UDIV_I64;
43024382
else if (VT == MVT::i128)
43034383
LC = RTLIB::UDIV_I128;
4384+
4385+
else {
4386+
SDValue Result =
4387+
ExpandExtIntRes_DIVREM(TLI, RTLIB::UDIV_IEXT, DAG, N, dl, VT);
4388+
SplitInteger(Result, Lo, Hi);
4389+
return;
4390+
}
4391+
43044392
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
43054393

43064394
TargetLowering::MakeLibCallOptions CallOptions;
@@ -4328,6 +4416,14 @@ void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
43284416
LC = RTLIB::UREM_I64;
43294417
else if (VT == MVT::i128)
43304418
LC = RTLIB::UREM_I128;
4419+
4420+
else {
4421+
SDValue Result =
4422+
ExpandExtIntRes_DIVREM(TLI, RTLIB::UREM_IEXT, DAG, N, dl, VT);
4423+
SplitInteger(Result, Lo, Hi);
4424+
return;
4425+
}
4426+
43314427
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
43324428

43334429
TargetLowering::MakeLibCallOptions CallOptions;

0 commit comments

Comments
 (0)