Skip to content

Commit d659bd1

Browse files
davemgreendavemgreen
authored
[GlobalISel][AArch64] Tail call libcalls. (llvm#74929)
This tries to allow libcalls to be tail called, using a similar method to DAG where the type is checked to make sure they match, and if so the backend, through lowerCall checks that the tailcall is valid for all arguments.
1 parent 5b5614c commit d659bd1

23 files changed

+205
-313
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -288,11 +288,14 @@ class LegalizerHelper {
288288

289289
// Implements floating-point environment read/write via library function call.
290290
LegalizeResult createGetStateLibcall(MachineIRBuilder &MIRBuilder,
291-
MachineInstr &MI);
291+
MachineInstr &MI,
292+
LostDebugLocObserver &LocObserver);
292293
LegalizeResult createSetStateLibcall(MachineIRBuilder &MIRBuilder,
293-
MachineInstr &MI);
294+
MachineInstr &MI,
295+
LostDebugLocObserver &LocObserver);
294296
LegalizeResult createResetStateLibcall(MachineIRBuilder &MIRBuilder,
295-
MachineInstr &MI);
297+
MachineInstr &MI,
298+
LostDebugLocObserver &LocObserver);
296299

297300
public:
298301
/// Return the alignment to use for a stack temporary object with the given
@@ -440,13 +443,15 @@ class LegalizerHelper {
440443
LegalizerHelper::LegalizeResult
441444
createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
442445
const CallLowering::ArgInfo &Result,
443-
ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);
446+
ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC,
447+
LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
444448

445449
/// Helper function that creates the given libcall.
446450
LegalizerHelper::LegalizeResult
447451
createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
448452
const CallLowering::ArgInfo &Result,
449-
ArrayRef<CallLowering::ArgInfo> Args);
453+
ArrayRef<CallLowering::ArgInfo> Args,
454+
LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr);
450455

451456
/// Create a libcall to memcpy et al.
452457
LegalizerHelper::LegalizeResult

‎llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ extern cl::opt<bool> DisableGISelLegalityCheck;
3535
class MachineFunction;
3636
class raw_ostream;
3737
class LegalizerHelper;
38+
class LostDebugLocObserver;
3839
class MachineInstr;
3940
class MachineRegisterInfo;
4041
class MCInstrInfo;
@@ -1288,8 +1289,8 @@ class LegalizerInfo {
12881289
const MachineRegisterInfo &MRI) const;
12891290

12901291
/// Called for instructions with the Custom LegalizationAction.
1291-
virtual bool legalizeCustom(LegalizerHelper &Helper,
1292-
MachineInstr &MI) const {
1292+
virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
1293+
LostDebugLocObserver &LocObserver) const {
12931294
llvm_unreachable("must implement this if custom action is used");
12941295
}
12951296

‎llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

Lines changed: 76 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
149149
return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
150150
case Custom:
151151
LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
152-
return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
152+
return LI.legalizeCustom(*this, MI, LocObserver) ? Legalized
153+
: UnableToLegalize;
153154
default:
154155
LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
155156
return UnableToLegalize;
@@ -567,7 +568,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
567568

568569
/// True if an instruction is in tail position in its caller. Intended for
569570
/// legalizing libcalls as tail calls when possible.
570-
static bool isLibCallInTailPosition(MachineInstr &MI,
571+
static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result,
572+
MachineInstr &MI,
571573
const TargetInstrInfo &TII,
572574
MachineRegisterInfo &MRI) {
573575
MachineBasicBlock &MBB = *MI.getParent();
@@ -596,17 +598,12 @@ static bool isLibCallInTailPosition(MachineInstr &MI,
596598
// RET_ReallyLR implicit $x0
597599
auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
598600
if (Next != MBB.instr_end() && Next->isCopy()) {
599-
switch (MI.getOpcode()) {
600-
default:
601-
llvm_unreachable("unsupported opcode");
602-
case TargetOpcode::G_BZERO:
601+
if (MI.getOpcode() == TargetOpcode::G_BZERO)
603602
return false;
604-
case TargetOpcode::G_MEMCPY:
605-
case TargetOpcode::G_MEMMOVE:
606-
case TargetOpcode::G_MEMSET:
607-
break;
608-
}
609603

604+
// For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the
605+
// mempy/etc routines return the same parameter. For other it will be the
606+
// returned value.
610607
Register VReg = MI.getOperand(0).getReg();
611608
if (!VReg.isVirtual() || VReg != Next->getOperand(1).getReg())
612609
return false;
@@ -622,7 +619,7 @@ static bool isLibCallInTailPosition(MachineInstr &MI,
622619
if (Ret->getNumImplicitOperands() != 1)
623620
return false;
624621

625-
if (PReg != Ret->getOperand(0).getReg())
622+
if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg())
626623
return false;
627624

628625
// Skip over the COPY that we just validated.
@@ -639,42 +636,73 @@ LegalizerHelper::LegalizeResult
639636
llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
640637
const CallLowering::ArgInfo &Result,
641638
ArrayRef<CallLowering::ArgInfo> Args,
642-
const CallingConv::ID CC) {
639+
const CallingConv::ID CC, LostDebugLocObserver &LocObserver,
640+
MachineInstr *MI) {
643641
auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
644642

645643
CallLowering::CallLoweringInfo Info;
646644
Info.CallConv = CC;
647645
Info.Callee = MachineOperand::CreateES(Name);
648646
Info.OrigRet = Result;
647+
if (MI)
648+
Info.IsTailCall =
649+
(Result.Ty->isVoidTy() ||
650+
Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) &&
651+
isLibCallInTailPosition(Result, *MI, MIRBuilder.getTII(),
652+
*MIRBuilder.getMRI());
653+
649654
std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
650655
if (!CLI.lowerCall(MIRBuilder, Info))
651656
return LegalizerHelper::UnableToLegalize;
652657

658+
if (MI && Info.LoweredTailCall) {
659+
assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
660+
661+
// Check debug locations before removing the return.
662+
LocObserver.checkpoint(true);
663+
664+
// We must have a return following the call (or debug insts) to get past
665+
// isLibCallInTailPosition.
666+
do {
667+
MachineInstr *Next = MI->getNextNode();
668+
assert(Next &&
669+
(Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
670+
"Expected instr following MI to be return or debug inst?");
671+
// We lowered a tail call, so the call is now the return from the block.
672+
// Delete the old return.
673+
Next->eraseFromParent();
674+
} while (MI->getNextNode());
675+
676+
// We expect to lose the debug location from the return.
677+
LocObserver.checkpoint(false);
678+
}
653679
return LegalizerHelper::Legalized;
654680
}
655681

656682
LegalizerHelper::LegalizeResult
657683
llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
658684
const CallLowering::ArgInfo &Result,
659-
ArrayRef<CallLowering::ArgInfo> Args) {
685+
ArrayRef<CallLowering::ArgInfo> Args,
686+
LostDebugLocObserver &LocObserver, MachineInstr *MI) {
660687
auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
661688
const char *Name = TLI.getLibcallName(Libcall);
662689
const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
663-
return createLibcall(MIRBuilder, Name, Result, Args, CC);
690+
return createLibcall(MIRBuilder, Name, Result, Args, CC, LocObserver, MI);
664691
}
665692

666693
// Useful for libcalls where all operands have the same type.
667694
static LegalizerHelper::LegalizeResult
668695
simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
669-
Type *OpType) {
696+
Type *OpType, LostDebugLocObserver &LocObserver) {
670697
auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
671698

672699
// FIXME: What does the original arg index mean here?
673700
SmallVector<CallLowering::ArgInfo, 3> Args;
674701
for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
675702
Args.push_back({MO.getReg(), OpType, 0});
676703
return createLibcall(MIRBuilder, Libcall,
677-
{MI.getOperand(0).getReg(), OpType, 0}, Args);
704+
{MI.getOperand(0).getReg(), OpType, 0}, Args,
705+
LocObserver, &MI);
678706
}
679707

680708
LegalizerHelper::LegalizeResult
@@ -733,8 +761,9 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
733761
Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
734762
Info.Callee = MachineOperand::CreateES(Name);
735763
Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0);
736-
Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
737-
isLibCallInTailPosition(MI, MIRBuilder.getTII(), MRI);
764+
Info.IsTailCall =
765+
MI.getOperand(MI.getNumOperands() - 1).getImm() &&
766+
isLibCallInTailPosition(Info.OrigRet, MI, MIRBuilder.getTII(), MRI);
738767

739768
std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
740769
if (!CLI.lowerCall(MIRBuilder, Info))
@@ -789,11 +818,11 @@ static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
789818

790819
static LegalizerHelper::LegalizeResult
791820
conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
792-
Type *FromType) {
821+
Type *FromType, LostDebugLocObserver &LocObserver) {
793822
RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
794-
return createLibcall(MIRBuilder, Libcall,
795-
{MI.getOperand(0).getReg(), ToType, 0},
796-
{{MI.getOperand(1).getReg(), FromType, 0}});
823+
return createLibcall(
824+
MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType, 0},
825+
{{MI.getOperand(1).getReg(), FromType, 0}}, LocObserver, &MI);
797826
}
798827

799828
static RTLIB::Libcall
@@ -829,7 +858,8 @@ getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
829858
//
830859
LegalizerHelper::LegalizeResult
831860
LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
832-
MachineInstr &MI) {
861+
MachineInstr &MI,
862+
LostDebugLocObserver &LocObserver) {
833863
const DataLayout &DL = MIRBuilder.getDataLayout();
834864
auto &MF = MIRBuilder.getMF();
835865
auto &MRI = *MIRBuilder.getMRI();
@@ -850,7 +880,8 @@ LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
850880
auto Res =
851881
createLibcall(MIRBuilder, RTLibcall,
852882
CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
853-
CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
883+
CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}),
884+
LocObserver, nullptr);
854885
if (Res != LegalizerHelper::Legalized)
855886
return Res;
856887

@@ -867,7 +898,8 @@ LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
867898
// content of memory region.
868899
LegalizerHelper::LegalizeResult
869900
LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
870-
MachineInstr &MI) {
901+
MachineInstr &MI,
902+
LostDebugLocObserver &LocObserver) {
871903
const DataLayout &DL = MIRBuilder.getDataLayout();
872904
auto &MF = MIRBuilder.getMF();
873905
auto &MRI = *MIRBuilder.getMRI();
@@ -892,7 +924,8 @@ LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
892924
RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
893925
return createLibcall(MIRBuilder, RTLibcall,
894926
CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
895-
CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
927+
CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}),
928+
LocObserver, nullptr);
896929
}
897930

898931
// The function is used to legalize operations that set default environment
@@ -902,7 +935,8 @@ LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
902935
// it is not true, the target must provide custom lowering.
903936
LegalizerHelper::LegalizeResult
904937
LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder,
905-
MachineInstr &MI) {
938+
MachineInstr &MI,
939+
LostDebugLocObserver &LocObserver) {
906940
const DataLayout &DL = MIRBuilder.getDataLayout();
907941
auto &MF = MIRBuilder.getMF();
908942
auto &Ctx = MF.getFunction().getContext();
@@ -919,7 +953,8 @@ LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder,
919953
RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
920954
return createLibcall(MIRBuilder, RTLibcall,
921955
CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
922-
CallLowering::ArgInfo({ Dest.getReg(), StatePtrTy, 0}));
956+
CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}),
957+
LocObserver, &MI);
923958
}
924959

925960
LegalizerHelper::LegalizeResult
@@ -938,7 +973,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
938973
LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
939974
unsigned Size = LLTy.getSizeInBits();
940975
Type *HLTy = IntegerType::get(Ctx, Size);
941-
auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
976+
auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver);
942977
if (Status != Legalized)
943978
return Status;
944979
break;
@@ -974,7 +1009,7 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
9741009
LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
9751010
return UnableToLegalize;
9761011
}
977-
auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
1012+
auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver);
9781013
if (Status != Legalized)
9791014
return Status;
9801015
break;
@@ -985,7 +1020,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
9851020
Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
9861021
if (!FromTy || !ToTy)
9871022
return UnableToLegalize;
988-
LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy );
1023+
LegalizeResult Status =
1024+
conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver);
9891025
if (Status != Legalized)
9901026
return Status;
9911027
break;
@@ -1000,7 +1036,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
10001036
LegalizeResult Status = conversionLibcall(
10011037
MI, MIRBuilder,
10021038
ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
1003-
FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
1039+
FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
1040+
LocObserver);
10041041
if (Status != Legalized)
10051042
return Status;
10061043
break;
@@ -1015,7 +1052,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
10151052
LegalizeResult Status = conversionLibcall(
10161053
MI, MIRBuilder,
10171054
ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
1018-
FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
1055+
FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
1056+
LocObserver);
10191057
if (Status != Legalized)
10201058
return Status;
10211059
break;
@@ -1032,19 +1070,20 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
10321070
return Result;
10331071
}
10341072
case TargetOpcode::G_GET_FPMODE: {
1035-
LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI);
1073+
LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI, LocObserver);
10361074
if (Result != Legalized)
10371075
return Result;
10381076
break;
10391077
}
10401078
case TargetOpcode::G_SET_FPMODE: {
1041-
LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI);
1079+
LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI, LocObserver);
10421080
if (Result != Legalized)
10431081
return Result;
10441082
break;
10451083
}
10461084
case TargetOpcode::G_RESET_FPMODE: {
1047-
LegalizeResult Result = createResetStateLibcall(MIRBuilder, MI);
1085+
LegalizeResult Result =
1086+
createResetStateLibcall(MIRBuilder, MI, LocObserver);
10481087
if (Result != Legalized)
10491088
return Result;
10501089
break;

‎llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,8 +1150,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
11501150
verify(*ST.getInstrInfo());
11511151
}
11521152

1153-
bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
1154-
MachineInstr &MI) const {
1153+
bool AArch64LegalizerInfo::legalizeCustom(
1154+
LegalizerHelper &Helper, MachineInstr &MI,
1155+
LostDebugLocObserver &LocObserver) const {
11551156
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
11561157
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
11571158
GISelChangeObserver &Observer = Helper.Observer;

‎llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ class AArch64LegalizerInfo : public LegalizerInfo {
2828
public:
2929
AArch64LegalizerInfo(const AArch64Subtarget &ST);
3030

31-
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override;
31+
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
32+
LostDebugLocObserver &LocObserver) const override;
3233

3334
bool legalizeIntrinsic(LegalizerHelper &Helper,
3435
MachineInstr &MI) const override;

‎llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1996,8 +1996,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
19961996
verify(*ST.getInstrInfo());
19971997
}
19981998

1999-
bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
2000-
MachineInstr &MI) const {
1999+
bool AMDGPULegalizerInfo::legalizeCustom(
2000+
LegalizerHelper &Helper, MachineInstr &MI,
2001+
LostDebugLocObserver &LocObserver) const {
20012002
MachineIRBuilder &B = Helper.MIRBuilder;
20022003
MachineRegisterInfo &MRI = *B.getMRI();
20032004

‎llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
3535
AMDGPULegalizerInfo(const GCNSubtarget &ST,
3636
const GCNTargetMachine &TM);
3737

38-
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override;
38+
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
39+
LostDebugLocObserver &LocObserver) const override;
3940

4041
Register getSegmentAperture(unsigned AddrSpace,
4142
MachineRegisterInfo &MRI,

0 commit comments

Comments
 (0)