[Inliner] Make recusive inlinee stack size limit tunable

For recursive callers, we want to be conservative when inlining callees with large stack size. We currently have a limit `InlineConstants::TotalAllocaSizeRecursiveCaller`, but that is hard coded.

We found the current limit insufficient to suppress problematic inlining that bloats stack size for deep recursion. This change adds a switch to make the limit tunable as a mitigation.

Differential Revision: https://reviews.llvm.org/D129411

Author: WenleiHe

llvm/lib/Analysis/InlineCost.cpp

@@ -131,6 +131,12 @@ static cl::opt<size_t>
                        cl::desc("Do not inline functions with a stack size "
                                 "that exceeds the specified limit"));
 
+static cl::opt<size_t>
+    RecurStackSizeThreshold("recursive-inline-max-stacksize", cl::Hidden,
+                       cl::init(InlineConstants::TotalAllocaSizeRecursiveCaller),
+                       cl::desc("Do not inline recursive functions with a stack "
+                                "size that exceeds the specified limit"));
+
 static cl::opt<bool> OptComputeFullInlineCost(
     "inline-cost-full", cl::Hidden,
     cl::desc("Compute the full inline cost of a call site even when the cost "
@@ -2444,8 +2450,7 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB,
     // If the caller is a recursive function then we don't want to inline
     // functions which allocate a lot of stack space because it would increase
     // the caller stack usage dramatically.
-    if (IsCallerRecursive &&
-        AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller) {
+    if (IsCallerRecursive && AllocatedSize > RecurStackSizeThreshold) {
       auto IR =
           InlineResult::failure("recursive and allocates too much stack space");
       if (ORE)

llvm/test/Transforms/Inline/inline-recur-stacksize.ll

@@ -0,0 +1,34 @@
+; Check the recursive inliner doesn't inline a function with a stack size exceeding a given limit.
+; RUN: opt < %s -inline -S | FileCheck --check-prefixes=ALL,UNLIMITED %s
+; RUN: opt < %s -inline -S -recursive-inline-max-stacksize=256 | FileCheck --check-prefixes=ALL,LIMITED %s
+
+declare void @init([65 x i32]*)
+
+define internal i32 @foo() {
+  %1 = alloca [65 x i32], align 16
+  %2 = getelementptr inbounds [65 x i32], [65 x i32]* %1, i65 0, i65 0
+  call void @init([65 x i32]* %1)
+  %3 = load i32, i32* %2, align 4
+  ret i32 %3
+}
+
+define i32 @bar() {
+  %1 = call i32 @foo()
+  ret i32 %1
+; ALL: define {{.*}}@bar
+; ALL-NOT: define
+; UNLIMITED-NOT: call {{.*}}@foo
+; LIMITED-NOT: call {{.*}}@foo
+}
+
+; Check that, under the tighter limit, baz() doesn't inline foo()
+define i32 @baz() {
+  %1 = call i32 @foo()
+  %2 = call i32 @baz()
+  %3 = add i32 %1, %2
+  ret i32 %3
+; ALL: define {{.*}}@baz
+; ALL-NOT: define
+; UNLIMITED-NOT: call {{.*}}@foo
+; LIMITED: call {{.*}}@foo
+}