Skip to content

Commit 509a494

Browse files
aemersonaemerson
committed
Add an operand to memory intrinsics to denote the "tail" marker.
We need to propagate this information from the IR in order to be able to safely do tail call optimizations on the intrinsics during legalization. Assuming it's safe to do tail call opt without checking for the marker isn't safe because the mem libcall may use allocas from the caller. This adds an extra immediate operand to the end of the intrinsics and fixes the legalizer to handle it. Differential Revision: https://reviews.llvm.org/D68151 llvm-svn: 373140
1 parent 76f44f6 commit 509a494

File tree

14 files changed

+168
-70
lines changed

14 files changed

+168
-70
lines changed

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,6 +1143,11 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
11431143
DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
11441144
}
11451145

1146+
// We need to propagate the tail call flag from the IR inst as an argument.
1147+
// Otherwise, we have to pessimize and assume later that we cannot tail call
1148+
// any memory intrinsics.
1149+
ICall.addImm(CI.isTailCall() ? 1 : 0);
1150+
11461151
// Create mem operands to store the alignment and volatile info.
11471152
auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
11481153
ICall.addMemOperand(MF->getMachineMemOperand(

‎llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,8 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
395395
auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
396396

397397
SmallVector<CallLowering::ArgInfo, 3> Args;
398-
for (unsigned i = 1; i < MI.getNumOperands(); i++) {
398+
// Add all the args, except for the last which is an imm denoting 'tail'.
399+
for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
399400
Register Reg = MI.getOperand(i).getReg();
400401

401402
// Need derive an IR type for call lowering.
@@ -433,7 +434,8 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
433434
Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
434435
Info.Callee = MachineOperand::CreateES(Name);
435436
Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
436-
Info.IsTailCall = isLibCallInTailPosition(MI);
437+
Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
438+
isLibCallInTailPosition(MI);
437439

438440
std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
439441
if (!CLI.lowerCall(MIRBuilder, Info))

‎llvm/lib/CodeGen/MachineVerifier.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,6 +1368,20 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
13681368
break;
13691369
}
13701370
}
1371+
switch (IntrID) {
1372+
case Intrinsic::memcpy:
1373+
if (MI->getNumOperands() != 5)
1374+
report("Expected memcpy intrinsic to have 5 operands", MI);
1375+
break;
1376+
case Intrinsic::memmove:
1377+
if (MI->getNumOperands() != 5)
1378+
report("Expected memmove intrinsic to have 5 operands", MI);
1379+
break;
1380+
case Intrinsic::memset:
1381+
if (MI->getNumOperands() != 5)
1382+
report("Expected memset intrinsic to have 5 operands", MI);
1383+
break;
1384+
}
13711385
break;
13721386
}
13731387
case TargetOpcode::G_SEXT_INREG: {

‎llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,18 +1134,28 @@ define void @test_memcpy(i8* %dst, i8* %src, i64 %size) {
11341134
; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
11351135
; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
11361136
; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
1137-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) :: (store 1 into %ir.dst), (load 1 from %ir.src)
1137+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
11381138
call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
11391139
ret void
11401140
}
11411141

1142+
define void @test_memcpy_tail(i8* %dst, i8* %src, i64 %size) {
1143+
; CHECK-LABEL: name: test_memcpy_tail
1144+
; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
1145+
; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
1146+
; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
1147+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 1 :: (store 1 into %ir.dst), (load 1 from %ir.src)
1148+
tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
1149+
ret void
1150+
}
1151+
11421152
declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)*, i8 addrspace(1)*, i64, i1)
11431153
define void @test_memcpy_nonzero_as(i8 addrspace(1)* %dst, i8 addrspace(1) * %src, i64 %size) {
11441154
; CHECK-LABEL: name: test_memcpy_nonzero_as
11451155
; CHECK: [[DST:%[0-9]+]]:_(p1) = COPY $x0
11461156
; CHECK: [[SRC:%[0-9]+]]:_(p1) = COPY $x1
11471157
; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
1148-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p1), [[SRC]](p1), [[SIZE]](s64) :: (store 1 into %ir.dst, addrspace 1), (load 1 from %ir.src, addrspace 1)
1158+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p1), [[SRC]](p1), [[SIZE]](s64), 0 :: (store 1 into %ir.dst, addrspace 1), (load 1 from %ir.src, addrspace 1)
11491159
call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %size, i1 0)
11501160
ret void
11511161
}
@@ -1156,7 +1166,7 @@ define void @test_memmove(i8* %dst, i8* %src, i64 %size) {
11561166
; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
11571167
; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
11581168
; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
1159-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) :: (store 1 into %ir.dst), (load 1 from %ir.src)
1169+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
11601170
call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
11611171
ret void
11621172
}
@@ -1168,7 +1178,7 @@ define void @test_memset(i8* %dst, i8 %val, i64 %size) {
11681178
; CHECK: [[SRC_C:%[0-9]+]]:_(s32) = COPY $w1
11691179
; CHECK: [[SRC:%[0-9]+]]:_(s8) = G_TRUNC [[SRC_C]]
11701180
; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
1171-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64) :: (store 1 into %ir.dst)
1181+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64), 0 :: (store 1 into %ir.dst)
11721182
call void @llvm.memset.p0i8.i64(i8* %dst, i8 %val, i64 %size, i1 0)
11731183
ret void
11741184
}

‎llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,12 @@ body: |
5252
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
5353
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
5454
; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
55-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
55+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
5656
; CHECK: RET_ReallyLR
5757
%0:_(p0) = COPY $x0
5858
%1:_(p0) = COPY $x1
5959
%2:_(s64) = COPY $x2
60-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
60+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
6161
RET_ReallyLR
6262
6363
...
@@ -104,7 +104,7 @@ body: |
104104
%0:_(p0) = COPY $x0
105105
%1:_(p0) = COPY $x1
106106
%2:_(s64) = G_CONSTANT i64 72
107-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
107+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
108108
RET_ReallyLR
109109
110110
...
@@ -171,7 +171,7 @@ body: |
171171
%0:_(p0) = COPY $x0
172172
%1:_(p0) = COPY $x1
173173
%2:_(s64) = G_CONSTANT i64 143
174-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
174+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
175175
RET_ReallyLR
176176
177177
...

‎llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ body: |
5555
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
5656
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
5757
; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
58-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
58+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
5959
; CHECK: RET_ReallyLR
6060
%0:_(p0) = COPY $x0
6161
%1:_(p0) = COPY $x1
6262
%2:_(s64) = COPY $x2
63-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
63+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
6464
RET_ReallyLR
6565
6666
...
@@ -94,7 +94,7 @@ body: |
9494
%0:_(p0) = COPY $x0
9595
%1:_(p0) = COPY $x1
9696
%2:_(s64) = G_CONSTANT i64 48
97-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
97+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
9898
RET_ReallyLR
9999
100100
...
@@ -111,12 +111,12 @@ body: |
111111
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
112112
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
113113
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 96
114-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
114+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
115115
; CHECK: RET_ReallyLR
116116
%0:_(p0) = COPY $x0
117117
%1:_(p0) = COPY $x1
118118
%2:_(s64) = G_CONSTANT i64 96
119-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
119+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
120120
RET_ReallyLR
121121
122122
...
@@ -156,7 +156,7 @@ body: |
156156
%0:_(p0) = COPY $x0
157157
%1:_(p0) = COPY $x1
158158
%2:_(s64) = G_CONSTANT i64 52
159-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
159+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
160160
RET_ReallyLR
161161
162162
...

‎llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ body: |
5454
; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
5555
; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
5656
; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
57-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[COPY]](p0), [[TRUNC]](s8), [[ZEXT]](s64) :: (store 1 into %ir.dst)
57+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[COPY]](p0), [[TRUNC]](s8), [[ZEXT]](s64), 1 :: (store 1 into %ir.dst)
5858
; CHECK: RET_ReallyLR
5959
%0:_(p0) = COPY $x0
6060
%1:_(s32) = COPY $w1
6161
%2:_(s32) = COPY $w2
6262
%3:_(s8) = G_TRUNC %1(s32)
6363
%4:_(s64) = G_ZEXT %2(s32)
64-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64) :: (store 1 into %ir.dst)
64+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), 1 :: (store 1 into %ir.dst)
6565
RET_ReallyLR
6666
6767
...
@@ -90,7 +90,7 @@ body: |
9090
%1:_(s32) = COPY $w1
9191
%3:_(s64) = G_CONSTANT i64 16
9292
%2:_(s8) = G_TRUNC %1(s32)
93-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %2(s8), %3(s64) :: (store 1 into %ir.dst)
93+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %2(s8), %3(s64), 1 :: (store 1 into %ir.dst)
9494
RET_ReallyLR
9595
9696
...
@@ -114,7 +114,7 @@ body: |
114114
%0:_(p0) = COPY $x0
115115
%1:_(s8) = G_CONSTANT i8 64
116116
%2:_(s64) = G_CONSTANT i64 16
117-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64) :: (store 1 into %ir.dst)
117+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64), 1 :: (store 1 into %ir.dst)
118118
RET_ReallyLR
119119
120120
...
@@ -142,7 +142,7 @@ body: |
142142
%0:_(p0) = COPY $x0
143143
%1:_(s8) = G_CONSTANT i8 64
144144
%2:_(s64) = G_CONSTANT i64 18
145-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64) :: (store 1 into %ir.dst)
145+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64), 1 :: (store 1 into %ir.dst)
146146
RET_ReallyLR
147147
148148
...

‎llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memcpy.mir

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ body: |
5353
%0:_(p0) = COPY $x0
5454
%1:_(p0) = COPY $x1
5555
%2:_(s64) = G_CONSTANT i64 32
56-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
56+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
5757
RET_ReallyLR
5858
5959
...
@@ -75,12 +75,12 @@ body: |
7575
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
7676
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
7777
; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
78-
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
78+
; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
7979
; CHECK: RET_ReallyLR
8080
%0:_(p0) = COPY $x0
8181
%1:_(p0) = COPY $x1
8282
%2:_(s64) = G_CONSTANT i64 36
83-
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
83+
G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
8484
RET_ReallyLR
8585
8686
...

0 commit comments

Comments
 (0)