Skip to content

Commit ddca2ba

Browse files
eecksteineeckstein
authored
Merge pull request #81706 from eeckstein/mandatory-temprvalue-elimination
Optimizer: Improve performance of large InlineArrays at Onone
2 parents 1e403ec + 198d4ab commit ddca2ba

38 files changed

+600
-233
lines changed

‎SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -622,6 +622,14 @@ private enum ImmutableScope {
622622
if beginAccess.isUnsafe {
623623
return nil
624624
}
625+
626+
// This is a workaround for a bug in the move-only checker: rdar://151841926.
627+
// The move-only checker sometimes inserts destroy_addr within read-only static access scopes.
628+
// TODO: remove this once the bug is fixed.
629+
if beginAccess.isStatic {
630+
return nil
631+
}
632+
625633
switch beginAccess.accessKind {
626634
case .read:
627635
self = .readAccess(beginAccess)

‎SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ let lifetimeDependenceDiagnosticsPass = FunctionPass(
6969
// mark_dependence will hanceforth be treated as an unknown use by the optimizer. In the future, we should not
7070
// need to set this flag during diagnostics because, for escapable types, mark_dependence [unresolved] will all be
7171
// settled during an early LifetimeNormalization pass.
72-
markDep.settleToEscaping()
72+
markDep.settleToEscaping(context)
7373
continue
7474
}
7575
if let apply = instruction as? FullApplySite {

‎SwiftCompilerSources/Sources/Optimizer/FunctionPasses/TempRValueElimination.swift

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,15 @@ import SIL
3636
///
3737
let tempRValueElimination = FunctionPass(name: "temp-rvalue-elimination") {
3838
(function: Function, context: FunctionPassContext) in
39+
removeTempRValues(in: function, keepDebugInfo: false, context)
40+
}
3941

42+
let mandatoryTempRValueElimination = FunctionPass(name: "mandatory-temp-rvalue-elimination") {
43+
(function: Function, context: FunctionPassContext) in
44+
removeTempRValues(in: function, keepDebugInfo: true, context)
45+
}
46+
47+
private func removeTempRValues(in function: Function, keepDebugInfo: Bool, _ context: FunctionPassContext) {
4048
for inst in function.instructions {
4149
switch inst {
4250
case let copy as CopyAddrInst:
@@ -45,12 +53,12 @@ let tempRValueElimination = FunctionPass(name: "temp-rvalue-elimination") {
4553
// copied the `alloc_stack` back to the source location.
4654
context.erase(instruction: copy)
4755
} else {
48-
tryEliminate(copy: copy, context)
56+
tryEliminate(copy: copy, keepDebugInfo: keepDebugInfo, context)
4957
}
5058
case let store as StoreInst:
5159
// Also handle `load`-`store` pairs which are basically the same thing as a `copy_addr`.
5260
if let load = store.source as? LoadInst, load.uses.isSingleUse, load.parentBlock == store.parentBlock {
53-
tryEliminate(copy: store, context)
61+
tryEliminate(copy: store, keepDebugInfo: keepDebugInfo, context)
5462
}
5563
default:
5664
break
@@ -82,9 +90,10 @@ extension StoreInst: CopyLikeInstruction {
8290
private var load: LoadInst { source as! LoadInst }
8391
}
8492

85-
private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassContext) {
93+
private func tryEliminate(copy: CopyLikeInstruction, keepDebugInfo: Bool, _ context: FunctionPassContext) {
8694

87-
guard let (allocStack, lastUseOfAllocStack) = getRemovableAllocStackDestination(of: copy, context) else {
95+
guard let (allocStack, lastUseOfAllocStack) =
96+
getRemovableAllocStackDestination(of: copy, keepDebugInfo: keepDebugInfo, context) else {
8897
return
8998
}
9099

@@ -127,6 +136,9 @@ private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassCont
127136
}
128137
}
129138
context.erase(instruction: allocStack)
139+
if keepDebugInfo {
140+
Builder(before: copy, context).createDebugStep()
141+
}
130142
context.erase(instructionIncludingAllUsers: copy.loadingInstruction)
131143
}
132144

@@ -138,18 +150,24 @@ private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassCont
138150
/// %lastUseOfAllocStack = load %allocStack
139151
/// ```
140152
private func getRemovableAllocStackDestination(
141-
of copy: CopyLikeInstruction, _ context: FunctionPassContext
153+
of copy: CopyLikeInstruction, keepDebugInfo: Bool, _ context: FunctionPassContext
142154
) -> (allocStack: AllocStackInst, lastUseOfAllocStack: Instruction)? {
143155
guard copy.isInitializationOfDestination,
144156
let allocStack = copy.destinationAddress as? AllocStackInst
145157
else {
146158
return nil
147159
}
148160

149-
// If the `allocStack` is lexical we can eliminate it if the source of the copy is lexical and
150-
// it is live for longer than the `allocStack`.
151-
if allocStack.isLexical && !copy.sourceAddress.accessBase.storageIsLexical {
152-
return nil
161+
if keepDebugInfo {
162+
if allocStack.isFromVarDecl || allocStack.isLexical {
163+
return nil
164+
}
165+
} else {
166+
// If the `allocStack` is lexical we can eliminate it if the source of the copy is lexical and
167+
// it is live for longer than the `allocStack`.
168+
if allocStack.isLexical && !copy.sourceAddress.accessBase.storageIsLexical {
169+
return nil
170+
}
153171
}
154172

155173
var allocStackUses = UseCollector(copy: copy, context)

‎SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ swift_compiler_sources(Optimizer
2929
SimplifyInitEnumDataAddr.swift
3030
SimplifyKeyPath.swift
3131
SimplifyLoad.swift
32+
SimplifyMarkDependence.swift
3233
SimplifyMisc.swift
3334
SimplifyPartialApply.swift
3435
SimplifyPointerToAddress.swift

‎SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyMarkDependence.swift

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
//===--- SimplifyMarkDependence.swift -------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import SIL
14+
15+
// Note: this simplification cannot run before dependency diagnostics.
16+
// See `var isRedundant` below.
17+
18+
extension MarkDependenceInst : OnoneSimplifiable, SILCombineSimplifiable {
19+
func simplify(_ context: SimplifyContext) {
20+
if isRedundant ||
21+
// A literal lives forever, so no mark_dependence is needed.
22+
// This pattern can occur after StringOptimization when a utf8CString of a literal is replace
23+
// by the string_literal itself.
24+
value.isLiteral
25+
{
26+
replace(with: value, context)
27+
return
28+
}
29+
simplifyBaseOperand(context)
30+
}
31+
}
32+
33+
extension MarkDependenceAddrInst : OnoneSimplifiable, SILCombineSimplifiable {
34+
func simplify(_ context: SimplifyContext) {
35+
if isRedundant {
36+
context.erase(instruction: self)
37+
return
38+
}
39+
simplifyBaseOperand(context)
40+
}
41+
}
42+
43+
private extension MarkDependenceInstruction {
44+
var isRedundant: Bool {
45+
if base.type.isObject && base.type.isTrivial(in: base.parentFunction) {
46+
// Sometimes due to specialization/builtins, we can get a mark_dependence whose base is a trivial
47+
// typed object. Trivial values live forever. Therefore the mark_dependence does not have a meaning.
48+
// Note: the mark_dependence is still needed for lifetime diagnostics. So it's important that this
49+
// simplification does not run before the lifetime diagnostic pass.
50+
return true
51+
}
52+
// If the value is an address projection from the base the mark_dependence is not needed because the
53+
// base cannot be destroyed before the accessing the value, anyway.
54+
if valueOrAddress.type.isAddress, base.type.isAddress,
55+
// But we still need to keep the mark_dependence for non-escapable types because a non-escapable
56+
// value can be copied and copies must not outlive the base.
57+
valueOrAddress.type.isEscapable(in: parentFunction),
58+
base.accessPath.isEqualOrContains(valueOrAddress.accessPath)
59+
{
60+
return true
61+
}
62+
return false
63+
}
64+
65+
func simplifyBaseOperand(_ context: SimplifyContext) {
66+
/// In OSSA, the `base` is a borrow introducing operand. It is pretty complicated to change the base.
67+
/// So, for simplicity, we only do this optimization when OSSA is already lowered.
68+
if parentFunction.hasOwnership {
69+
return
70+
}
71+
// Replace the base operand with the operand of the base value if it's a certain kind of forwarding
72+
// instruction.
73+
let rootBase = base.lookThroughEnumAndExistentialRef
74+
if rootBase != base {
75+
baseOperand.set(to: rootBase, context)
76+
}
77+
}
78+
}
79+
80+
private extension Value {
81+
/// True, if this is a literal instruction or a struct of a literal instruction.
82+
/// What we want to catch here is a `UnsafePointer<Int8>` of a string literal.
83+
var isLiteral: Bool {
84+
switch self {
85+
case let s as StructInst:
86+
if let singleOperand = s.operands.singleElement {
87+
return singleOperand.value.isLiteral
88+
}
89+
return false
90+
case is IntegerLiteralInst, is FloatLiteralInst, is StringLiteralInst:
91+
return true
92+
default:
93+
return false
94+
}
95+
}
96+
97+
var lookThroughEnumAndExistentialRef: Value {
98+
switch self {
99+
case let e as EnumInst:
100+
if let payload = e.payload {
101+
return payload.lookThroughEnumAndExistentialRef
102+
}
103+
return self
104+
case let ier as InitExistentialRefInst:
105+
return ier.instance.lookThroughEnumAndExistentialRef
106+
case let oer as OpenExistentialRefInst:
107+
return oer.existential.lookThroughEnumAndExistentialRef
108+
default:
109+
return self
110+
}
111+
}
112+
}

‎SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,20 @@ extension CopyAddrInst {
776776
}
777777
}
778778

779+
extension MarkDependenceInstruction {
780+
func resolveToNonEscaping(_ context: some MutatingContext) {
781+
context.notifyInstructionsChanged()
782+
bridged.MarkDependenceInstruction_resolveToNonEscaping()
783+
context.notifyInstructionChanged(self)
784+
}
785+
786+
func settleToEscaping(_ context: some MutatingContext) {
787+
context.notifyInstructionsChanged()
788+
bridged.MarkDependenceInstruction_settleToEscaping()
789+
context.notifyInstructionChanged(self)
790+
}
791+
}
792+
779793
extension TermInst {
780794
func replaceBranchTarget(from fromBlock: BasicBlock, to toBlock: BasicBlock, _ context: some MutatingContext) {
781795
context.notifyBranchesChanged()

‎SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ private func registerSwiftPasses() {
101101
registerPass(lifetimeDependenceScopeFixupPass, { lifetimeDependenceScopeFixupPass.run($0) })
102102
registerPass(copyToBorrowOptimization, { copyToBorrowOptimization.run($0) })
103103
registerPass(tempRValueElimination, { tempRValueElimination.run($0) })
104+
registerPass(mandatoryTempRValueElimination, { mandatoryTempRValueElimination.run($0) })
104105
registerPass(generalClosureSpecialization, { generalClosureSpecialization.run($0) })
105106
registerPass(autodiffClosureSpecialization, { autodiffClosureSpecialization.run($0) })
106107

@@ -122,6 +123,8 @@ private func registerSwiftPasses() {
122123
registerForSILCombine(DestructureTupleInst.self, { run(DestructureTupleInst.self, $0) })
123124
registerForSILCombine(TypeValueInst.self, { run(TypeValueInst.self, $0) })
124125
registerForSILCombine(ClassifyBridgeObjectInst.self, { run(ClassifyBridgeObjectInst.self, $0) })
126+
registerForSILCombine(MarkDependenceInst.self, { run(MarkDependenceInst.self, $0) })
127+
registerForSILCombine(MarkDependenceAddrInst.self, { run(MarkDependenceAddrInst.self, $0) })
125128
registerForSILCombine(PointerToAddressInst.self, { run(PointerToAddressInst.self, $0) })
126129
registerForSILCombine(UncheckedEnumDataInst.self, { run(UncheckedEnumDataInst.self, $0) })
127130
registerForSILCombine(WitnessMethodInst.self, { run(WitnessMethodInst.self, $0) })

‎SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,9 @@ extension LifetimeDependence {
226226
return scope.computeRange(context)
227227
}
228228

229-
func resolve(_ context: some Context) {
229+
func resolve(_ context: some MutatingContext) {
230230
if let mdi = markDepInst {
231-
mdi.resolveToNonEscaping()
231+
mdi.resolveToNonEscaping(context)
232232
}
233233
}
234234
}

‎SwiftCompilerSources/Sources/SIL/Instruction.swift

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1124,58 +1124,39 @@ public enum MarkDependenceKind: Int32 {
11241124
}
11251125

11261126
public protocol MarkDependenceInstruction: Instruction {
1127-
var baseOperand: Operand { get }
1128-
var base: Value { get }
11291127
var dependenceKind: MarkDependenceKind { get }
1130-
func resolveToNonEscaping()
1131-
func settleToEscaping()
11321128
}
11331129

11341130
extension MarkDependenceInstruction {
11351131
public var isNonEscaping: Bool { dependenceKind == .NonEscaping }
11361132
public var isUnresolved: Bool { dependenceKind == .Unresolved }
1133+
1134+
public var valueOrAddressOperand: Operand { operands[0] }
1135+
public var valueOrAddress: Value { valueOrAddressOperand.value }
1136+
public var baseOperand: Operand { operands[1] }
1137+
public var base: Value { baseOperand.value }
11371138
}
11381139

11391140
final public class MarkDependenceInst : SingleValueInstruction, MarkDependenceInstruction {
1140-
public var valueOperand: Operand { operands[0] }
1141-
public var baseOperand: Operand { operands[1] }
1141+
public var valueOperand: Operand { valueOrAddressOperand }
11421142
public var value: Value { return valueOperand.value }
1143-
public var base: Value { return baseOperand.value }
11441143

11451144
public var dependenceKind: MarkDependenceKind {
11461145
MarkDependenceKind(rawValue: bridged.MarkDependenceInst_dependenceKind().rawValue)!
11471146
}
11481147

1149-
public func resolveToNonEscaping() {
1150-
bridged.MarkDependenceInst_resolveToNonEscaping()
1151-
}
1152-
1153-
public func settleToEscaping() {
1154-
bridged.MarkDependenceInst_settleToEscaping()
1155-
}
1156-
11571148
public var hasScopedLifetime: Bool {
11581149
return isNonEscaping && type.isObject && ownership == .owned && type.isEscapable(in: parentFunction)
11591150
}
11601151
}
11611152

11621153
final public class MarkDependenceAddrInst : Instruction, MarkDependenceInstruction {
1163-
public var addressOperand: Operand { operands[0] }
1164-
public var baseOperand: Operand { operands[1] }
1154+
public var addressOperand: Operand { valueOrAddressOperand }
11651155
public var address: Value { return addressOperand.value }
1166-
public var base: Value { return baseOperand.value }
11671156

11681157
public var dependenceKind: MarkDependenceKind {
11691158
MarkDependenceKind(rawValue: bridged.MarkDependenceAddrInst_dependenceKind().rawValue)!
11701159
}
1171-
1172-
public func resolveToNonEscaping() {
1173-
bridged.MarkDependenceAddrInst_resolveToNonEscaping()
1174-
}
1175-
1176-
public func settleToEscaping() {
1177-
bridged.MarkDependenceAddrInst_settleToEscaping()
1178-
}
11791160
}
11801161

11811162
final public class RefToBridgeObjectInst : SingleValueInstruction {

‎include/swift/SIL/SILBridging.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -803,11 +803,9 @@ struct BridgedInstruction {
803803
BRIDGED_INLINE SwiftInt StoreInst_getStoreOwnership() const;
804804
BRIDGED_INLINE SwiftInt AssignInst_getAssignOwnership() const;
805805
BRIDGED_INLINE MarkDependenceKind MarkDependenceInst_dependenceKind() const;
806-
BRIDGED_INLINE void MarkDependenceInst_resolveToNonEscaping() const;
807-
BRIDGED_INLINE void MarkDependenceInst_settleToEscaping() const;
806+
BRIDGED_INLINE void MarkDependenceInstruction_resolveToNonEscaping() const;
807+
BRIDGED_INLINE void MarkDependenceInstruction_settleToEscaping() const;
808808
BRIDGED_INLINE MarkDependenceKind MarkDependenceAddrInst_dependenceKind() const;
809-
BRIDGED_INLINE void MarkDependenceAddrInst_resolveToNonEscaping() const;
810-
BRIDGED_INLINE void MarkDependenceAddrInst_settleToEscaping() const;
811809
BRIDGED_INLINE SwiftInt BeginAccessInst_getAccessKind() const;
812810
BRIDGED_INLINE bool BeginAccessInst_isStatic() const;
813811
BRIDGED_INLINE bool BeginAccessInst_isUnsafe() const;

‎include/swift/SIL/SILBridgingImpl.h

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1425,26 +1425,26 @@ BridgedInstruction::MarkDependenceKind BridgedInstruction::MarkDependenceInst_de
14251425
return (MarkDependenceKind)getAs<swift::MarkDependenceInst>()->dependenceKind();
14261426
}
14271427

1428-
void BridgedInstruction::MarkDependenceInst_resolveToNonEscaping() const {
1429-
getAs<swift::MarkDependenceInst>()->resolveToNonEscaping();
1428+
void BridgedInstruction::MarkDependenceInstruction_resolveToNonEscaping() const {
1429+
if (auto *mdi = llvm::dyn_cast<swift::MarkDependenceInst>(unbridged())) {
1430+
mdi->resolveToNonEscaping();
1431+
} else {
1432+
getAs<swift::MarkDependenceAddrInst>()->resolveToNonEscaping();
1433+
}
14301434
}
14311435

1432-
void BridgedInstruction::MarkDependenceInst_settleToEscaping() const {
1433-
getAs<swift::MarkDependenceInst>()->settleToEscaping();
1436+
void BridgedInstruction::MarkDependenceInstruction_settleToEscaping() const {
1437+
if (auto *mdi = llvm::dyn_cast<swift::MarkDependenceInst>(unbridged())) {
1438+
mdi->settleToEscaping();
1439+
} else {
1440+
getAs<swift::MarkDependenceAddrInst>()->settleToEscaping();
1441+
}
14341442
}
14351443

14361444
BridgedInstruction::MarkDependenceKind BridgedInstruction::MarkDependenceAddrInst_dependenceKind() const {
14371445
return (MarkDependenceKind)getAs<swift::MarkDependenceAddrInst>()->dependenceKind();
14381446
}
14391447

1440-
void BridgedInstruction::MarkDependenceAddrInst_resolveToNonEscaping() const {
1441-
getAs<swift::MarkDependenceAddrInst>()->resolveToNonEscaping();
1442-
}
1443-
1444-
void BridgedInstruction::MarkDependenceAddrInst_settleToEscaping() const {
1445-
getAs<swift::MarkDependenceAddrInst>()->settleToEscaping();
1446-
}
1447-
14481448
SwiftInt BridgedInstruction::BeginAccessInst_getAccessKind() const {
14491449
return (SwiftInt)getAs<swift::BeginAccessInst>()->getAccessKind();
14501450
}

0 commit comments

Comments
 (0)