Don't sibcall between SysV and Win64 convention functions

The shadow stack space expectations won't match.

Fixes PR22709.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@230667 91177308-0d34-0410-b5e6-96231b3b80d8

Author: rnk

lib/Target/X86/X86ISelLowering.cpp

@@ -3381,6 +3381,12 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
   bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC);
   bool IsCallerWin64 = Subtarget->isCallingConvWin64(CallerCC);
 
+  // Win64 functions have extra shadow space for argument homing. Don't do the
+  // sibcall if the caller and callee have mismatched expectations for this
+  // space.
+  if (IsCalleeWin64 != IsCallerWin64)
+    return false;
+
   if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
     if (IsTailCallConvention(CalleeCC) && CCMatch)
       return true;

test/CodeGen/X86/sibcall-win64.ll

@@ -0,0 +1,42 @@
+; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s
+
+declare x86_64_win64cc void @win64_callee(i32)
+declare void @sysv_callee(i32)
+
+define void @sysv_caller(i32 %p1) {
+entry:
+  tail call x86_64_win64cc void @win64_callee(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: sysv_caller:
+; CHECK: subq $40, %rsp
+; CHECK: callq win64_callee
+; CHECK: addq $40, %rsp
+; CHECK: retq
+
+define x86_64_win64cc void @win64_caller(i32 %p1) {
+entry:
+  tail call void @sysv_callee(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: win64_caller:
+; CHECK: callq sysv_callee
+; CHECK: retq
+
+define void @sysv_matched(i32 %p1) {
+  tail call void @sysv_callee(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: sysv_matched:
+; CHECK: jmp sysv_callee # TAILCALL
+
+define x86_64_win64cc void @win64_matched(i32 %p1) {
+  tail call x86_64_win64cc void @win64_callee(i32 %p1)
+  ret void
+}
+
+; CHECK-LABEL: win64_matched:
+; CHECK: jmp win64_callee # TAILCALL