Skip to content

[NFC][AlwaysInliner] Reduce AlwaysInliner memory consumption. #96958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 2, 2024
Merged

[NFC][AlwaysInliner] Reduce AlwaysInliner memory consumption. #96958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3a6f85d
[NFC][AlwaysInliner] Reduce AlwaysInliner memory consumption.
dfukalov Jun 27, 2024
e3f4142
Addressed comment.
dfukalov Jun 27, 2024
3a2078e
Addressed comments.
dfukalov Jul 1, 2024
f51d541
clang-format
dfukalov Jul 1, 2024
f29f1d2
removed obsolete part of InlinedFunctions.
dfukalov Jul 1, 2024
c0aa14d
Comment updated.
dfukalov Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 55 additions & 68 deletions llvm/lib/Transforms/IPO/AlwaysInliner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Module.h"
#include "llvm/InitializePasses.h"
#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"

Expand All @@ -37,86 +37,73 @@ bool AlwaysInlineImpl(
function_ref<BlockFrequencyInfo &(Function &)> GetBFI) {
SmallSetVector<CallBase *, 16> Calls;
bool Changed = false;
SmallVector<Function *, 16> InlinedFunctions;
for (Function &F : M) {
// When callee coroutine function is inlined into caller coroutine function
// before coro-split pass,
// coro-early pass can not handle this quiet well.
// So we won't inline the coroutine function if it have not been unsplited
SmallVector<Function *, 16> InlinedComdatFunctions;

for (Function &F : make_early_inc_range(M)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add a comment about the fact that we need this since we may delete the function we're visiting

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've checked a number of places make_early_inc_range() used - found no comments about its functionality. It seems its usage instead of default iterator is quite obvious, what do you think?

Updated the comment below about InlinedComdatFunctions usage.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's not immediately obvious since the deletion happens further down so I'd write a comment, but won't object if you omit the comment

if (F.isPresplitCoroutine())
continue;

if (!F.isDeclaration() && isInlineViable(F).isSuccess()) {
Calls.clear();

for (User *U : F.users())
if (auto *CB = dyn_cast<CallBase>(U))
if (CB->getCalledFunction() == &F &&
CB->hasFnAttr(Attribute::AlwaysInline) &&
!CB->getAttributes().hasFnAttr(Attribute::NoInline))
Calls.insert(CB);

for (CallBase *CB : Calls) {
Function *Caller = CB->getCaller();
OptimizationRemarkEmitter ORE(Caller);
DebugLoc DLoc = CB->getDebugLoc();
BasicBlock *Block = CB->getParent();

InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
GetBFI ? &GetBFI(*Caller) : nullptr,
GetBFI ? &GetBFI(F) : nullptr);

InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
&GetAAR(F), InsertLifetime);
if (!Res.isSuccess()) {
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc,
Block)
<< "'" << ore::NV("Callee", &F) << "' is not inlined into '"
<< ore::NV("Caller", Caller)
<< "': " << ore::NV("Reason", Res.getFailureReason());
});
continue;
}

emitInlinedIntoBasedOnCost(
ORE, DLoc, Block, F, *Caller,
InlineCost::getAlways("always inline attribute"),
/*ForProfileContext=*/false, DEBUG_TYPE);
if (F.isDeclaration() || !isInlineViable(F).isSuccess())
continue;

Changed = true;
Calls.clear();

for (User *U : F.users())
if (auto *CB = dyn_cast<CallBase>(U))
if (CB->getCalledFunction() == &F &&
CB->hasFnAttr(Attribute::AlwaysInline) &&
!CB->getAttributes().hasFnAttr(Attribute::NoInline))
Calls.insert(CB);

for (CallBase *CB : Calls) {
Function *Caller = CB->getCaller();
OptimizationRemarkEmitter ORE(Caller);
DebugLoc DLoc = CB->getDebugLoc();
BasicBlock *Block = CB->getParent();

InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
GetBFI ? &GetBFI(*Caller) : nullptr,
GetBFI ? &GetBFI(F) : nullptr);

InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
&GetAAR(F), InsertLifetime);
if (!Res.isSuccess()) {
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
<< "'" << ore::NV("Callee", &F) << "' is not inlined into '"
<< ore::NV("Caller", Caller)
<< "': " << ore::NV("Reason", Res.getFailureReason());
});
continue;
}

if (F.hasFnAttribute(Attribute::AlwaysInline)) {
// Remember to try and delete this function afterward. This both avoids
// re-walking the rest of the module and avoids dealing with any
// iterator invalidation issues while deleting functions.
InlinedFunctions.push_back(&F);
}
emitInlinedIntoBasedOnCost(
ORE, DLoc, Block, F, *Caller,
InlineCost::getAlways("always inline attribute"),
/*ForProfileContext=*/false, DEBUG_TYPE);

Changed = true;
}
}

// Remove any live functions.
erase_if(InlinedFunctions, [&](Function *F) {
F->removeDeadConstantUsers();
return !F->isDefTriviallyDead();
});

// Delete the non-comdat ones from the module and also from our vector.
auto NonComdatBegin = partition(
InlinedFunctions, [&](Function *F) { return F->hasComdat(); });
for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) {
M.getFunctionList().erase(F);
Changed = true;
F.removeDeadConstantUsers();
if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
// Remember to try and delete this function afterward. This allows to call
// filterDeadComdatFunctions() only once.
if (F.hasComdat()) {
InlinedComdatFunctions.push_back(&F);
} else {
M.getFunctionList().erase(F);
Changed = true;
}
}
}
InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end());

if (!InlinedFunctions.empty()) {
if (!InlinedComdatFunctions.empty()) {
// Now we just have the comdat functions. Filter out the ones whose comdats
// are not actually dead.
filterDeadComdatFunctions(InlinedFunctions);
filterDeadComdatFunctions(InlinedComdatFunctions);
// The remaining functions are actually dead.
for (Function *F : InlinedFunctions) {
for (Function *F : InlinedComdatFunctions) {
M.getFunctionList().erase(F);
Changed = true;
}
Expand Down
1 change: 0 additions & 1 deletion llvm/test/Transforms/Inline/always-inline-phase-ordering.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ target triple = "arm64e-apple-macosx13"
; CHECK: remark: <unknown>:0:0: 'wibble' inlined into 'pluto' with (cost=always): always inline attribute
; CHECK: remark: <unknown>:0:0: 'snork' inlined into 'blam' with (cost=always): always inline attribute
; CHECK: remark: <unknown>:0:0: 'wobble' inlined into 'blam' with (cost=always): always inline attribute
; CHECK: remark: <unknown>:0:0: 'wobble' inlined into 'snork' with (cost=always): always inline attribute
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This remark was about inlining 'wobble' into already inlined 'snork'.

; CHECK: remark: <unknown>:0:0: 'spam' inlined into 'blam' with (cost=65, threshold=75)
; CHECK: remark: <unknown>:0:0: 'wibble.1' inlined into 'widget' with (cost=30, threshold=75)
; CHECK: remark: <unknown>:0:0: 'widget' inlined into 'bar.8' with (cost=30, threshold=75)
Expand Down
Loading