Skip to content

Commit 501f2be

Browse files
aschwaighoferaschwaighofer
authored
Merge pull request #77331 from aschwaighofer/large_types_reg2mem_aggressive
LargeTypesReg2Mem: Add a new heuristic that trys harder to keep large values on the stack
2 parents 5e8550e + 787c996 commit 501f2be

21 files changed

+782
-68
lines changed

‎SwiftCompilerSources/Sources/Optimizer/FunctionPasses/DeadStoreElimination.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ let deadStoreElimination = FunctionPass(name: "dead-store-elimination") {
7575
}
7676

7777
private func tryEliminate(store: StoreInst, complexityBudget: inout Int, _ context: FunctionPassContext) {
78-
if !store.hasValidOwnershipForDeadStoreElimination {
78+
if !store.hasValidOwnershipForDeadStoreElimination || !store.source.type.shouldExpand(context) {
7979
return
8080
}
8181

‎SwiftCompilerSources/Sources/Optimizer/FunctionPasses/RedundantLoadElimination.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ private func eliminateRedundantLoads(in function: Function, ignoreArrays: Bool,
9797
{
9898
continue
9999
}
100+
if !load.type.shouldExpand(context) {
101+
continue
102+
}
100103
tryEliminate(load: load, complexityBudget: &complexityBudget, context)
101104
}
102105
}

‎SwiftCompilerSources/Sources/Optimizer/PassManager/Options.swift

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ struct Options {
2020
_bridged.enableStackProtection()
2121
}
2222

23+
var useAggressiveReg2MemForCodeSize : Bool {
24+
_bridged.useAggressiveReg2MemForCodeSize()
25+
}
26+
2327
var enableMoveInoutStackProtection: Bool {
2428
_bridged.enableMoveInoutStackProtection()
2529
}

‎SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,3 +809,12 @@ extension CheckedCastAddrBranchInst {
809809
}
810810
}
811811
}
812+
813+
extension Type {
814+
func shouldExpand(_ context: some Context) -> Bool {
815+
if !context.options.useAggressiveReg2MemForCodeSize {
816+
return true
817+
}
818+
return context._bridged.shouldExpand(self.bridged)
819+
}
820+
}

‎include/swift/AST/SILOptions.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,10 @@ class SILOptions {
318318
/// for the debugger?
319319
bool ShouldFunctionsBePreservedToDebugger = true;
320320

321+
/// Block expanding and register promotion more aggressively throughout the
322+
/// optimizer.
323+
bool UseAggressiveReg2MemForCodeSize = false;
324+
321325
SILOptions() {}
322326

323327
/// Return a hash code of any components from these options that should

‎include/swift/Option/FrontendOptions.td

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,6 +1294,13 @@ def disable_lifetime_dependence_diagnostics :
12941294
Flag<["-"], "disable-lifetime-dependence-diagnostics">,
12951295
HelpText<"Disable lifetime dependence diagnostics for Nonescapable types.">;
12961296

1297+
def enable_aggressive_reg2mem :
1298+
Flag<["-"], "enable-aggressive-reg2mem">,
1299+
HelpText<"Enable a more aggresive reg2mem heuristic">;
1300+
def disable_aggressive_reg2mem :
1301+
Flag<["-"], "disable-aggresive-reg2mem">,
1302+
HelpText<"Disable a more aggresive reg2mem heuristic">;
1303+
12971304
def enable_collocate_metadata_functions :
12981305
Flag<["-"], "enable-collocate-metadata-functions">,
12991306
HelpText<"Enable collocate metadata functions">;

‎include/swift/SILOptimizer/OptimizerBridging.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,8 @@ struct BridgedPassContext {
261261
BRIDGED_INLINE bool optimizeMemoryAccesses(BridgedFunction f) const;
262262
BRIDGED_INLINE bool eliminateDeadAllocations(BridgedFunction f) const;
263263

264+
BRIDGED_INLINE bool shouldExpand(BridgedType type) const;
265+
264266
// IRGen
265267

266268
SwiftInt getStaticSize(BridgedType type) const;
@@ -372,6 +374,7 @@ struct BridgedPassContext {
372374
Unchecked = 2
373375
};
374376

377+
BRIDGED_INLINE bool useAggressiveReg2MemForCodeSize() const;
375378
BRIDGED_INLINE bool enableStackProtection() const;
376379
BRIDGED_INLINE bool hasFeature(BridgedFeature feature) const;
377380
BRIDGED_INLINE bool enableMoveInoutStackProtection() const;

‎include/swift/SILOptimizer/OptimizerBridgingImpl.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,11 +556,21 @@ bool BridgedPassContext::enableMoveInoutStackProtection() const {
556556
return mod->getOptions().EnableMoveInoutStackProtection;
557557
}
558558

559+
bool BridgedPassContext::useAggressiveReg2MemForCodeSize() const {
560+
swift::SILModule *mod = invocation->getPassManager()->getModule();
561+
return mod->getOptions().UseAggressiveReg2MemForCodeSize;
562+
}
563+
559564
BridgedPassContext::AssertConfiguration BridgedPassContext::getAssertConfiguration() const {
560565
swift::SILModule *mod = invocation->getPassManager()->getModule();
561566
return (AssertConfiguration)mod->getOptions().AssertConfig;
562567
}
563568

569+
bool BridgedPassContext::shouldExpand(BridgedType ty) const {
570+
swift::SILModule &mod = *invocation->getPassManager()->getModule();
571+
return swift::shouldExpand(mod, ty.unbridged());
572+
}
573+
564574
static_assert((int)BridgedPassContext::SILStage::Raw == (int)swift::SILStage::Raw);
565575
static_assert((int)BridgedPassContext::SILStage::Canonical == (int)swift::SILStage::Canonical);
566576
static_assert((int)BridgedPassContext::SILStage::Lowered == (int)swift::SILStage::Lowered);

‎lib/Frontend/CompilerInvocation.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3738,6 +3738,7 @@ bool CompilerInvocation::parseArgs(
37383738
SILOpts.SkipFunctionBodies = FunctionBodySkipping::None;
37393739
SILOpts.CMOMode = CrossModuleOptimizationMode::Everything;
37403740
SILOpts.EmbeddedSwift = true;
3741+
SILOpts.UseAggressiveReg2MemForCodeSize = true;
37413742
// OSSA modules are required for deinit de-virtualization.
37423743
SILOpts.EnableOSSAModules = true;
37433744
// -g is promoted to -gdwarf-types in embedded Swift
@@ -3751,6 +3752,11 @@ bool CompilerInvocation::parseArgs(
37513752
}
37523753
}
37533754

3755+
SILOpts.UseAggressiveReg2MemForCodeSize =
3756+
ParsedArgs.hasFlag(OPT_enable_aggressive_reg2mem,
3757+
OPT_disable_aggressive_reg2mem,
3758+
SILOpts.UseAggressiveReg2MemForCodeSize);
3759+
37543760
// With Swift 6, enable @_spiOnly by default. This also enables proper error
37553761
// reporting of ioi references from spi decls.
37563762
if (LangOpts.EffectiveLanguageVersion.isVersionAtLeast(6)) {

‎lib/IRGen/GenCall.cpp

Lines changed: 82 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5346,7 +5346,8 @@ Explosion NativeConventionSchema::mapIntoNative(IRGenModule &IGM,
53465346
IRGenFunction &IGF,
53475347
Explosion &fromNonNative,
53485348
SILType type,
5349-
bool isOutlined) const {
5349+
bool isOutlined,
5350+
bool mayPeepholeLoad) const {
53505351
if (fromNonNative.empty()) {
53515352
assert(empty() && "Empty explosion must match the native convention");
53525353
return Explosion();
@@ -5425,6 +5426,82 @@ Explosion NativeConventionSchema::mapIntoNative(IRGenModule &IGM,
54255426
? coercionTy
54265427
: overlappedCoercionTy;
54275428

5429+
if (mayPeepholeLoad) {
5430+
auto succeeded = [&]() -> bool {
5431+
if (!overlappedCoercionTy->isEmptyTy())
5432+
return false;
5433+
auto load = dyn_cast<llvm::LoadInst>(*fromNonNative.begin());
5434+
if (!load)
5435+
return false;
5436+
auto *gep = dyn_cast<llvm::GetElementPtrInst>(load->getPointerOperand());
5437+
if (!gep)
5438+
return false;
5439+
auto *alloca = dyn_cast<llvm::AllocaInst>(getUnderlyingObject(gep));
5440+
if (!alloca)
5441+
return false;
5442+
auto numExplosions = fromNonNative.size();
5443+
if (numExplosions < 2)
5444+
return false;
5445+
for (unsigned i = 0, e = numExplosions; i < e; ++i) {
5446+
auto *otherLoad = dyn_cast<llvm::LoadInst>(*(fromNonNative.begin() + i));
5447+
if (!otherLoad)
5448+
return false;
5449+
auto otherAlloca = dyn_cast<llvm::AllocaInst>(
5450+
getUnderlyingObject(otherLoad->getPointerOperand()));
5451+
if (!otherAlloca || otherAlloca != alloca)
5452+
return false;
5453+
load = otherLoad;
5454+
}
5455+
auto allocaSize =
5456+
DataLayout.getTypeSizeInBits(alloca->getAllocatedType());
5457+
5458+
Address origAlloca(alloca, alloca->getAllocatedType(),
5459+
Alignment(alloca->getAlign().value()));
5460+
5461+
IRBuilder Builder(*IGM.LLVMContext, false);
5462+
Builder.SetInsertPoint(load);
5463+
5464+
if (allocaSize < coercionSize) {
5465+
auto coerced = IGF.createAlloca(coercionTy, Alignment(alloca->getAlign().value()) , "tmp.coerce");
5466+
// Copy the defined bytes.
5467+
Builder.CreateMemCpy(coerced, origAlloca, Size(allocaSize/8));
5468+
origAlloca = coerced;
5469+
}
5470+
5471+
adjustAllocaAlignment(DataLayout, origAlloca, coercionTy);
5472+
5473+
5474+
unsigned expandedMapIdx = 0;
5475+
SmallVector<llvm::Value *, 8> expandedElts(expandedTys.size(), nullptr);
5476+
auto structAddr = Builder.CreateElementBitCast(origAlloca, coercionTy);
5477+
for (auto eltIndex : indices(coercionTy->elements())) {
5478+
auto layout = DataLayout.getStructLayout(coercionTy);
5479+
auto eltTy = coercionTy->getElementType(eltIndex);
5480+
// Skip padding fields.
5481+
if (eltTy->isArrayTy())
5482+
continue;
5483+
Address eltAddr = Builder.CreateStructGEP(structAddr, eltIndex, layout);
5484+
llvm::Value *elt = Builder.CreateLoad(eltAddr);
5485+
auto index = expandedTyIndicesMap[expandedMapIdx];
5486+
assert(expandedElts[index] == nullptr);
5487+
expandedElts[index] = elt;
5488+
++expandedMapIdx;
5489+
}
5490+
5491+
// Add the values to the explosion.
5492+
for (auto *val : expandedElts)
5493+
nativeExplosion.add(val);
5494+
assert(expandedTys.size() == nativeExplosion.size());
5495+
5496+
return true;
5497+
}();
5498+
5499+
if (succeeded) {
5500+
(void)fromNonNative.claimAll();
5501+
return nativeExplosion;
5502+
}
5503+
}
5504+
54285505
// Allocate a temporary for the coercion.
54295506
Address temporary;
54305507
Size tempSize;
@@ -5514,7 +5591,8 @@ Explosion IRGenFunction::coerceValueTo(SILType fromTy, Explosion &from,
55145591

55155592
void IRGenFunction::emitScalarReturn(SILType returnResultType,
55165593
SILType funcResultType, Explosion &result,
5517-
bool isSwiftCCReturn, bool isOutlined) {
5594+
bool isSwiftCCReturn, bool isOutlined,
5595+
bool mayPeepholeLoad) {
55185596
if (result.empty()) {
55195597
assert(IGM.getTypeInfo(returnResultType)
55205598
.nativeReturnValueSchema(IGM)
@@ -5533,7 +5611,8 @@ void IRGenFunction::emitScalarReturn(SILType returnResultType,
55335611
assert(!nativeSchema.requiresIndirect());
55345612

55355613
Explosion native = nativeSchema.mapIntoNative(IGM, *this, result,
5536-
funcResultType, isOutlined);
5614+
funcResultType, isOutlined,
5615+
mayPeepholeLoad);
55375616
if (native.size() == 1) {
55385617
Builder.CreateRet(native.claimNext());
55395618
return;

‎lib/IRGen/IRGenFunction.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class IRGenFunction {
102102
Explosion collectParameters();
103103
void emitScalarReturn(SILType returnResultType, SILType funcResultType,
104104
Explosion &scalars, bool isSwiftCCReturn,
105-
bool isOutlined);
105+
bool isOutlined, bool mayPeepholeLoad = false);
106106
void emitScalarReturn(llvm::Type *resultTy, Explosion &scalars);
107107

108108
void emitBBForReturn();

0 commit comments

Comments
 (0)