[InlineCost] Cache collectEphemeralValues() to save compile time

`CallAnalyzer::analyze()` can take several hours to run if the function contains
thousands of @llvm.assume() calls and there are thousands of callsites.
The time is spent in `collectEphemeralvalues()`.
This patch adds a caching and will only collect the ephemeral values once
per function.

Author: vporpo

llvm/include/llvm/Analysis/InlineAdvisor.h

@@ -236,6 +236,7 @@ class DefaultInlineAdvisor : public InlineAdvisor {
   std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override;
 
   InlineParams Params;
+  EphValuesCacheT EphValuesCache;
 };
 
 /// Used for dynamically registering InlineAdvisors as plugins

llvm/include/llvm/Analysis/InlineCost.h

@@ -14,6 +14,7 @@
 #define LLVM_ANALYSIS_INLINECOST_H
 
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Analysis/InlineModelFeatureMaps.h"
 #include "llvm/IR/PassManager.h"
@@ -31,6 +32,9 @@ class Function;
 class ProfileSummaryInfo;
 class TargetTransformInfo;
 class TargetLibraryInfo;
+class Value;
+
+using EphValuesCacheT = MapVector<Function *, SmallPtrSet<const Value *, 32>>;
 
 namespace InlineConstants {
 // Various thresholds used by inline cost analysis.
@@ -273,14 +277,13 @@ int getCallsiteCost(const TargetTransformInfo &TTI, const CallBase &Call,
 ///
 /// Also note that calling this function *dynamically* computes the cost of
 /// inlining the callsite. It is an expensive, heavyweight call.
-InlineCost
-getInlineCost(CallBase &Call, const InlineParams &Params,
-              TargetTransformInfo &CalleeTTI,
-              function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
-              function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
-              function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
-              ProfileSummaryInfo *PSI = nullptr,
-              OptimizationRemarkEmitter *ORE = nullptr);
+InlineCost getInlineCost(
+    CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI,
+    function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
+    function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
+    function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
+    ProfileSummaryInfo *PSI = nullptr, OptimizationRemarkEmitter *ORE = nullptr,
+    EphValuesCacheT *EphValuesCache = nullptr);
 
 /// Get an InlineCost with the callee explicitly specified.
 /// This allows you to calculate the cost of inlining a function via a
@@ -294,7 +297,8 @@ getInlineCost(CallBase &Call, Function *Callee, const InlineParams &Params,
               function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
               function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
               ProfileSummaryInfo *PSI = nullptr,
-              OptimizationRemarkEmitter *ORE = nullptr);
+              OptimizationRemarkEmitter *ORE = nullptr,
+              EphValuesCacheT *EphValuesCache = nullptr);
 
 /// Returns InlineResult::success() if the call site should be always inlined
 /// because of user directives, and the inlining is viable. Returns

llvm/lib/Analysis/InlineAdvisor.cpp

@@ -133,7 +133,8 @@ void DefaultInlineAdvice::recordInliningImpl() {
 }
 
 std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
-    CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
+    CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params,
+    EphValuesCacheT *EphValuesCache = nullptr) {
   Function &Caller = *CB.getCaller();
   ProfileSummaryInfo *PSI =
       FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
@@ -158,7 +159,8 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
         Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
             DEBUG_TYPE);
     return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
-                         GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
+                         GetBFI, PSI, RemarksEnabled ? &ORE : nullptr,
+                         EphValuesCache);
   };
   return llvm::shouldInline(
       CB, CalleeTTI, GetInlineCost, ORE,
@@ -167,7 +169,7 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
 
 std::unique_ptr<InlineAdvice>
 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
-  auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
+  auto OIC = getDefaultInlineAdvice(CB, FAM, Params, &EphValuesCache);
   return std::make_unique<DefaultInlineAdvice>(
       this, CB, OIC,
       FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));

llvm/lib/Analysis/InlineCost.cpp

@@ -173,6 +173,11 @@ static cl::opt<bool> DisableGEPConstOperand(
     "disable-gep-const-evaluation", cl::Hidden, cl::init(false),
     cl::desc("Disables evaluation of GetElementPtr with constant operands"));
 
+static cl::opt<unsigned> EphValuesCacheSizeLimit(
+    "inline-ephvalues-cache-size-limit", cl::Hidden, cl::init(8),
+    cl::desc(
+        "Clear the cache of ephemeral values if it grows larger than this"));
+
 namespace llvm {
 std::optional<int> getStringFnAttrAsInt(const Attribute &Attr) {
   if (Attr.isValid()) {
@@ -269,6 +274,10 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   /// easily cacheable. Instead, use the cover function paramHasAttr.
   CallBase &CandidateCall;
 
+  /// Collecting the ephemeral values over and over again can be expensive, so
+  /// cache them.
+  EphValuesCacheT *EphValuesCache;
+
   /// Extension points for handling callsite features.
   // Called before a basic block was analyzed.
   virtual void onBlockStart(const BasicBlock *BB) {}
@@ -510,10 +519,11 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
       function_ref<BlockFrequencyInfo &(Function &)> GetBFI = nullptr,
       function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
       ProfileSummaryInfo *PSI = nullptr,
-      OptimizationRemarkEmitter *ORE = nullptr)
+      OptimizationRemarkEmitter *ORE = nullptr,
+      EphValuesCacheT *EphValuesCache = nullptr)
       : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI),
         GetTLI(GetTLI), PSI(PSI), F(Callee), DL(F.getDataLayout()), ORE(ORE),
-        CandidateCall(Call) {}
+        CandidateCall(Call), EphValuesCache(EphValuesCache) {}
 
   InlineResult analyze();
 
@@ -1126,9 +1136,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
       function_ref<const TargetLibraryInfo &(Function &)> GetTLI = nullptr,
       ProfileSummaryInfo *PSI = nullptr,
       OptimizationRemarkEmitter *ORE = nullptr, bool BoostIndirect = true,
-      bool IgnoreThreshold = false)
+      bool IgnoreThreshold = false, EphValuesCacheT *EphValuesCache = nullptr)
       : CallAnalyzer(Callee, Call, TTI, GetAssumptionCache, GetBFI, GetTLI, PSI,
-                     ORE),
+                     ORE, EphValuesCache),
         ComputeFullInlineCost(OptComputeFullInlineCost ||
                               Params.ComputeFullInlineCost || ORE ||
                               isCostBenefitAnalysisEnabled()),
@@ -2781,11 +2791,27 @@ InlineResult CallAnalyzer::analyze() {
   NumConstantOffsetPtrArgs = ConstantOffsetPtrs.size();
   NumAllocaArgs = SROAArgValues.size();
 
-  // FIXME: If a caller has multiple calls to a callee, we end up recomputing
-  // the ephemeral values multiple times (and they're completely determined by
-  // the callee, so this is purely duplicate work).
-  SmallPtrSet<const Value *, 32> EphValues;
-  CodeMetrics::collectEphemeralValues(&F, &GetAssumptionCache(F), EphValues);
+  // Collecting the ephemeral values of `F` can be expensive, so collect them
+  // once per function and cache them for future reuse.
+  SmallPtrSet<const Value *, 32> EphValuesSet;
+  SmallPtrSet<const Value *, 32> *EphValues;
+  auto &AC = GetAssumptionCache(F);
+  if (EphValuesCache == nullptr) {
+    // No cache is being used, so collect the values every time.
+    CodeMetrics::collectEphemeralValues(&F, &AC, EphValuesSet);
+    EphValues = &EphValuesSet;
+  } else {
+    // If the ephemeral values for `F` are in the cache, then reuse them. Else
+    // collect them from scratch.
+    if (EphValuesCache->size() >= EphValuesCacheSizeLimit)
+      // If the cache grows larger than a limit remove the first element.
+      // NOTE: This is a linear-time operation so keep the cache size small!
+      EphValuesCache->erase(EphValuesCache->begin()->first);
+    auto Pair = EphValuesCache->insert({&F, {}});
+    if (Pair.second)
+      CodeMetrics::collectEphemeralValues(&F, &AC, Pair.first->second);
+    EphValues = &Pair.first->second;
+  }
 
   // The worklist of live basic blocks in the callee *after* inlining. We avoid
   // adding basic blocks of the callee which can be proven to be dead for this
@@ -2824,7 +2850,7 @@ InlineResult CallAnalyzer::analyze() {
 
     // Analyze the cost of this block. If we blow through the threshold, this
     // returns false, and we can bail on out.
-    InlineResult IR = analyzeBlock(BB, EphValues);
+    InlineResult IR = analyzeBlock(BB, *EphValues);
     if (!IR.isSuccess())
       return IR;
 
@@ -2967,9 +2993,11 @@ InlineCost llvm::getInlineCost(
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
-    ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) {
+    ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE,
+    EphValuesCacheT *EphValuesCache) {
   return getInlineCost(Call, Call.getCalledFunction(), Params, CalleeTTI,
-                       GetAssumptionCache, GetTLI, GetBFI, PSI, ORE);
+                       GetAssumptionCache, GetTLI, GetBFI, PSI, ORE,
+                       EphValuesCache);
 }
 
 std::optional<int> llvm::getInliningCostEstimate(
@@ -3089,7 +3117,8 @@ InlineCost llvm::getInlineCost(
     function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
     function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
     function_ref<BlockFrequencyInfo &(Function &)> GetBFI,
-    ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) {
+    ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE,
+    EphValuesCacheT *EphValuesCache) {
 
   auto UserDecision =
       llvm::getAttributeBasedInliningDecision(Call, Callee, CalleeTTI, GetTLI);
@@ -3105,7 +3134,9 @@ InlineCost llvm::getInlineCost(
                           << ")\n");
 
   InlineCostCallAnalyzer CA(*Callee, Call, Params, CalleeTTI,
-                            GetAssumptionCache, GetBFI, GetTLI, PSI, ORE);
+                            GetAssumptionCache, GetBFI, GetTLI, PSI, ORE,
+                            /*BoostIndirect=*/true, /*IgnoreThreshold=*/false,
+                            EphValuesCache);
   InlineResult ShouldInline = CA.analyze();
 
   LLVM_DEBUG(CA.dump());