[llvm][CallBrPrepare] add llvm.callbr.landingpad intrinsic

Insert a new intrinsic call after splitting critical edges, and verify
it. Later commits will update the SSA values to use this new value along
indirect branches rather than the callbr's value, and have SelectionDAG
consume this new value.

Part 2b of
https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453/8.

Reviewed By: efriedma, jyknight

Differential Revision: https://reviews.llvm.org/D139883

Author: nickdesaulniers

llvm/include/llvm/IR/Intrinsics.td

@@ -620,6 +620,11 @@ def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i
 def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>;
 def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>;
 
+// This intrinsic is intentionally undocumented and users shouldn't call it;
+// it's produced then quickly consumed during codegen.
+def int_callbr_landingpad : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>],
+                                      [IntrNoMerge]>;
+
 //===------------------- Standard C Library Intrinsics --------------------===//
 //
 

llvm/lib/CodeGen/CallBrPrepare.cpp

@@ -32,13 +32,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -104,6 +108,23 @@ bool CallBrPrepare::SplitCriticalEdges(ArrayRef<CallBrInst *> CBRs,
   return Changed;
 }
 
+static bool InsertIntrinsicCalls(ArrayRef<CallBrInst *> CBRs) {
+  bool Changed = false;
+  SmallPtrSet<const BasicBlock *, 4> Visited;
+  IRBuilder<> Builder(CBRs[0]->getContext());
+  for (CallBrInst *CBR : CBRs) {
+    for (BasicBlock *IndDest : CBR->getIndirectDests()) {
+      if (!Visited.insert(IndDest).second)
+        continue;
+      Builder.SetInsertPoint(&*IndDest->begin());
+      Builder.CreateIntrinsic(CBR->getType(), Intrinsic::callbr_landingpad,
+                              {CBR});
+      Changed = true;
+    }
+  }
+  return Changed;
+}
+
 bool CallBrPrepare::runOnFunction(Function &Fn) {
   bool Changed = false;
   SmallVector<CallBrInst *, 2> CBRs = FindCallBrs(Fn);
@@ -130,5 +151,8 @@ bool CallBrPrepare::runOnFunction(Function &Fn) {
   if (SplitCriticalEdges(CBRs, *DT))
     Changed = true;
 
+  if (InsertIntrinsicCalls(CBRs))
+    Changed = true;
+
   return Changed;
 }

llvm/lib/IR/Verifier.cpp

@@ -5824,6 +5824,35 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
           "isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call);
     break;
   }
+  case Intrinsic::callbr_landingpad: {
+    const auto *CBR = dyn_cast<CallBrInst>(Call.getOperand(0));
+    Check(CBR, "intrinstic requires callbr operand", &Call);
+    if (!CBR)
+      break;
+
+    const BasicBlock *LandingPadBB = Call.getParent();
+    const BasicBlock *PredBB = LandingPadBB->getUniquePredecessor();
+    if (!PredBB) {
+      CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call);
+      break;
+    }
+    if (!isa<CallBrInst>(PredBB->getTerminator())) {
+      CheckFailed("Intrinsic must have corresponding callbr in predecessor",
+                  &Call);
+      break;
+    }
+    Check(llvm::any_of(CBR->getIndirectDests(),
+                       [LandingPadBB](const BasicBlock *IndDest) {
+                         return IndDest == LandingPadBB;
+                       }),
+          "Intrinsic's corresponding callbr must have intrinsic's parent basic "
+          "block in indirect destination list",
+          &Call);
+    const Instruction &First = *LandingPadBB->begin();
+    Check(&First == &Call, "No other instructions may proceed intrinsic",
+          &Call);
+    break;
+  }
   };
 
   // Verify that there aren't any unmediated control transfers between funclets.

llvm/test/CodeGen/AArch64/callbr-prepare.ll

@@ -7,11 +7,13 @@ define i32 @test0() {
 ; CHECK-NEXT:    [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"()
 ; CHECK-NEXT:    to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge]
 ; CHECK:       entry.indirect_crit_edge:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]])
 ; CHECK-NEXT:    br label [[INDIRECT:%.*]]
 ; CHECK:       direct:
 ; CHECK-NEXT:    [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"()
 ; CHECK-NEXT:    to label [[DIRECT2:%.*]] [label %direct.indirect_crit_edge]
 ; CHECK:       direct.indirect_crit_edge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]])
 ; CHECK-NEXT:    br label [[INDIRECT]]
 ; CHECK:       direct2:
 ; CHECK-NEXT:    ret i32 0
@@ -67,6 +69,7 @@ define i32 @dont_split1() {
 ; CHECK:       x:
 ; CHECK-NEXT:    ret i32 42
 ; CHECK:       y:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
 ; CHECK-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
@@ -138,12 +141,13 @@ define i32 @split_me0() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = callbr i32 asm "", "=r,!i"()
 ; CHECK-NEXT:    to label [[X:%.*]] [label %entry.y_crit_edge]
 ; CHECK:       entry.y_crit_edge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
 ; CHECK-NEXT:    br label [[Y:%.*]]
 ; CHECK:       x:
 ; CHECK-NEXT:    br label [[Y]]
 ; CHECK:       y:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ]
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[ENTRY_Y_CRIT_EDGE:%.*]] ], [ 42, [[X]] ]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   %0 = callbr i32 asm "", "=r,!i"()
@@ -168,12 +172,13 @@ define i32 @split_me1(i1 %z) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"()
 ; CHECK-NEXT:    to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge]
 ; CHECK:       w.v_crit_edge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
 ; CHECK-NEXT:    br label [[V]]
 ; CHECK:       x:
 ; CHECK-NEXT:    ret i32 42
 ; CHECK:       v:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   br i1 %z, label %w, label %v
@@ -200,12 +205,13 @@ define i32 @split_me2(i1 %z) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i"()
 ; CHECK-NEXT:    to label [[X:%.*]] [label [[W_V_CRIT_EDGE:%.*]], label %w.v_crit_edge]
 ; CHECK:       w.v_crit_edge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[TMP0]])
 ; CHECK-NEXT:    br label [[V]]
 ; CHECK:       x:
 ; CHECK-NEXT:    ret i32 42
 ; CHECK:       v:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[W_V_CRIT_EDGE]] ], [ 42, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   br i1 %z, label %w, label %v

llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll

@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
+; RUN: opt < %s -S -passes="simplifycfg<sink-common-insts>" \
+; RUN:   -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
 
 define void @callbr_duplicate_dest() {
 ; CHECK-LABEL: @callbr_duplicate_dest(
@@ -57,3 +58,49 @@ bb2:
 bb3:
   ret void
 }
+
+; Validate that callbr landingpad intrinsics do not get merged (via the
+; IntrNoMerge attribute).
+define i32 @callbr_landingpad_nomerge() {
+; CHECK-LABEL: @callbr_landingpad_nomerge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"()
+; CHECK-NEXT:    to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge]
+; CHECK:       entry.indirect_crit_edge:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]])
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       direct:
+; CHECK-NEXT:    [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"()
+; CHECK-NEXT:    to label [[COMMON_RET]] [label %direct.indirect_crit_edge]
+; CHECK:       direct.indirect_crit_edge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]])
+; CHECK-NEXT:    br label [[COMMON_RET]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[DIRECT]] ], [ [[TMP0]], [[ENTRY_INDIRECT_CRIT_EDGE:%.*]] ], [ [[TMP1]], [[DIRECT_INDIRECT_CRIT_EDGE:%.*]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+;
+entry:
+  %out = callbr i32 asm "# $0", "=r,!i"()
+  to label %direct [label %entry.indirect_crit_edge]
+
+entry.indirect_crit_edge:
+  %0 = call i32 @llvm.callbr.landingpad.i32(i32 %out)
+  br label %indirect
+
+direct:
+  %out2 = callbr i32 asm "# $0", "=r,!i"()
+  to label %direct2 [label %direct.indirect_crit_edge]
+
+direct.indirect_crit_edge:
+  %1 = call i32 @llvm.callbr.landingpad.i32(i32 %out2)
+  br label %indirect
+
+direct2:
+  ret i32 0
+
+indirect:
+  %out3 = phi i32 [ %0, %entry.indirect_crit_edge ], [ %1, %direct.indirect_crit_edge ]
+  ret i32 %out3
+}
+
+declare i32 @llvm.callbr.landingpad.i32(i32)

llvm/test/Verifier/callbr.ll

@@ -68,3 +68,55 @@ normal:
 abnormal:
   ret i32 %ret
 }
+
+;; Tests of the callbr.landingpad intrinsic function.
+declare i32 @llvm.callbr.landingpad.i64(i64)
+define void @callbrpad_bad_type() {
+entry:
+; CHECK: Intrinsic has incorrect argument type!
+; CHECK-NEXT: ptr @llvm.callbr.landingpad.i64
+  %foo = call i32 @llvm.callbr.landingpad.i64(i64 42)
+  ret void
+}
+
+declare i32 @llvm.callbr.landingpad.i32(i32)
+define i32 @callbrpad_multi_preds() {
+entry:
+  %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
+direct:
+  br label %indirect
+indirect:
+; CHECK-NEXT: Intrinsic in block must have 1 unique predecessor
+; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+  %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+  ret i32 %out
+}
+
+define void @callbrpad_wrong_callbr() {
+entry:
+  %foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
+direct:
+; CHECK-NEXT: Intrinsic's corresponding callbr must have intrinsic's parent basic block in indirect destination list
+; CHECK-NEXT: %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+  %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
+  ret void
+indirect:
+  ret void
+}
+
+declare i32 @foo(i32)
+define i32 @test_callbr_landingpad_not_first_inst() {
+entry:
+  %0 = callbr i32 asm "", "=r,!i"()
+          to label %asm.fallthrough [label %landingpad]
+
+asm.fallthrough:
+  ret i32 42
+
+landingpad:
+  %foo = call i32 @foo(i32 42)
+; CHECK-NEXT: No other instructions may proceed intrinsic
+; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
+  %out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
+  ret i32 %out
+}