|
| 1 | +//===--- LoadCopyToBorrowOptimization.swift --------------------------------==// |
| 2 | +// |
| 3 | +// This source file is part of the Swift.org open source project |
| 4 | +// |
| 5 | +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors |
| 6 | +// Licensed under Apache License v2.0 with Runtime Library Exception |
| 7 | +// |
| 8 | +// See https://swift.org/LICENSE.txt for license information |
| 9 | +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| 10 | +// |
| 11 | +//===----------------------------------------------------------------------===// |
| 12 | + |
| 13 | +import SIL |
| 14 | + |
| 15 | +/// Replaces a `load [copy]` with a `load_borrow` if possible. |
| 16 | +/// |
| 17 | +/// ``` |
| 18 | +/// %1 = load [copy] %0 |
| 19 | +/// // no writes to %0 |
| 20 | +/// destroy_value %1 |
| 21 | +/// ``` |
| 22 | +/// -> |
| 23 | +/// ``` |
| 24 | +/// %1 = load_borrow %0 |
| 25 | +/// // no writes to %0 |
| 26 | +/// end_borrow %1 |
| 27 | +/// ``` |
| 28 | +/// |
| 29 | +/// The optimization can be done if: |
| 30 | +/// * During the (forward-extended) lifetime of the loaded value the memory location is not changed. |
| 31 | +/// * All (forward-extended) uses of the loaded value support guaranteed ownership. |
| 32 | +/// * The (forward-extended) lifetime of the loaded value ends with `destroy_value`(s). |
| 33 | +/// |
| 34 | +let loadCopyToBorrowOptimization = FunctionPass(name: "load-copy-to-borrow-optimization") { |
| 35 | + (function: Function, context: FunctionPassContext) in |
| 36 | + |
| 37 | + if !function.hasOwnership { |
| 38 | + return |
| 39 | + } |
| 40 | + |
| 41 | + for inst in function.instructions { |
| 42 | + if let load = inst as? LoadInst { |
| 43 | + optimize(load: load, context) |
| 44 | + } |
| 45 | + } |
| 46 | +} |
| 47 | + |
| 48 | +private func optimize(load: LoadInst, _ context: FunctionPassContext) { |
| 49 | + if load.loadOwnership != .copy { |
| 50 | + return |
| 51 | + } |
| 52 | + |
| 53 | + var ownershipUses = OwnershipUses(context) |
| 54 | + defer { ownershipUses.deinitialize() } |
| 55 | + |
| 56 | + // Handle the special case if the `load` is immediately followed by a single `move_value` |
| 57 | + // (see the comment in `LoadInst.replaceWithLoadBorrow`). |
| 58 | + let loadedValue = load.singleMoveValueUser ?? load |
| 59 | + |
| 60 | + if !ownershipUses.collectUses(of: loadedValue) { |
| 61 | + return |
| 62 | + } |
| 63 | + |
| 64 | + var liverange = InstructionWorklist(liverangeOf: load, with: ownershipUses) |
| 65 | + defer { liverange.deinitialize() } |
| 66 | + |
| 67 | + if liverange.mayWrite(toAddressOf: load, context) { |
| 68 | + return |
| 69 | + } |
| 70 | + |
| 71 | + load.replaceWithLoadBorrow(liverange: liverange, ownershipUses: ownershipUses) |
| 72 | +} |
| 73 | + |
| 74 | +private extension LoadInst { |
| 75 | + var singleMoveValueUser: MoveValueInst? { |
| 76 | + uses.ignoreDebugUses.singleUse?.instruction as? MoveValueInst |
| 77 | + } |
| 78 | + |
| 79 | + func replaceWithLoadBorrow(liverange: InstructionWorklist, ownershipUses: OwnershipUses) { |
| 80 | + let context = ownershipUses.context |
| 81 | + let builder = Builder(before: self, context) |
| 82 | + let loadBorrow = builder.createLoadBorrow(fromAddress: address) |
| 83 | + |
| 84 | + // Handle the special case if the `load` is immediately followed by a single `move_value`. |
| 85 | + // Note that `move_value` is _not_ a forwarding instruction. |
| 86 | + // In this case we have to preserve the move's flags by inserting a `begin_borrow` with the same flags. |
| 87 | + // For example: |
| 88 | + // |
| 89 | + // %1 = load [copy] %0 |
| 90 | + // %2 = move_value [lexical] %1 |
| 91 | + // ... |
| 92 | + // destroy_value %2 |
| 93 | + // -> |
| 94 | + // %1 = load_borrow %0 |
| 95 | + // %2 = begin_borrow [lexical] %1 |
| 96 | + // ... |
| 97 | + // end_borrow %2 |
| 98 | + // end_borrow %1 |
| 99 | + // |
| 100 | + let innerBorrow: BeginBorrowInst? |
| 101 | + if let moveInst = singleMoveValueUser { |
| 102 | + // An inner borrow is needed to keep the flags of the `move_value`. |
| 103 | + let bbi = builder.createBeginBorrow(of: loadBorrow, |
| 104 | + isLexical: moveInst.isLexical, |
| 105 | + hasPointerEscape: moveInst.hasPointerEscape, |
| 106 | + isFromVarDecl: moveInst.isFromVarDecl) |
| 107 | + moveInst.uses.replaceAll(with: bbi, context) |
| 108 | + context.erase(instruction: moveInst) |
| 109 | + innerBorrow = bbi |
| 110 | + } else { |
| 111 | + innerBorrow = nil |
| 112 | + } |
| 113 | + |
| 114 | + uses.replaceAll(with: loadBorrow, context) |
| 115 | + context.erase(instruction: self) |
| 116 | + |
| 117 | + for destroy in ownershipUses.destroys { |
| 118 | + context.createEndBorrows(ofInner: innerBorrow, ofOuter: loadBorrow, before: destroy, ifNotIn: liverange) |
| 119 | + context.erase(instruction: destroy) |
| 120 | + } |
| 121 | + |
| 122 | + for forwardingUse in ownershipUses.forwardingUses { |
| 123 | + forwardingUse.changeOwnership(from: .owned, to: .guaranteed, context) |
| 124 | + } |
| 125 | + |
| 126 | + for liverangeExitBlock in ownershipUses.nonDestroyingLiverangeExits { |
| 127 | + context.createEndBorrows(ofInner: innerBorrow, ofOuter: loadBorrow, |
| 128 | + before: liverangeExitBlock.instructions.first!, |
| 129 | + ifNotIn: liverange) |
| 130 | + } |
| 131 | + } |
| 132 | +} |
| 133 | + |
| 134 | +private extension FunctionPassContext { |
| 135 | + func createEndBorrows( |
| 136 | + ofInner innerBorrow: BeginBorrowInst?, |
| 137 | + ofOuter outerBorrow: LoadBorrowInst, |
| 138 | + before insertionPoint: Instruction, |
| 139 | + ifNotIn liverange: InstructionWorklist |
| 140 | + ) { |
| 141 | + // There can be multiple destroys in a row in case of decomposing an aggregate, e.g. |
| 142 | + // %1 = load [copy] %0 |
| 143 | + // ... |
| 144 | + // (%2, %3) = destructure_struct %1 |
| 145 | + // destroy_value %2 |
| 146 | + // destroy_value %3 // The final destroy. Here we need to create the `end_borrow`(s) |
| 147 | + // |
| 148 | + if liverange.hasBeenPushed(insertionPoint) { |
| 149 | + return |
| 150 | + } |
| 151 | + let builder = Builder(before: insertionPoint, self) |
| 152 | + if let innerBorrow = innerBorrow { |
| 153 | + builder.createEndBorrow(of: innerBorrow) |
| 154 | + } |
| 155 | + builder.createEndBorrow(of: outerBorrow) |
| 156 | + } |
| 157 | + |
| 158 | +} |
| 159 | + |
| 160 | +private struct OwnershipUses { |
| 161 | + let context: FunctionPassContext |
| 162 | + |
| 163 | + // Operand of all forwarding instructions, which - if possible - are converted from "owned" to "guaranteed" |
| 164 | + private(set) var forwardingUses: Stack<Operand> |
| 165 | + |
| 166 | + // All destroys of the load its forwarded values. |
| 167 | + private(set) var destroys: Stack<DestroyValueInst> |
| 168 | + |
| 169 | + // Uses of the load in a dead-end block where the final `destroy_value` is missing at the `unreachable`. |
| 170 | + // In such a case the dataflow would _not_ visit potential writes to the load's memory location. |
| 171 | + // In the following example, the `load [copy]` must not be converted to a `load_borrow`: |
| 172 | + // |
| 173 | + // %1 = load [copy] %0 |
| 174 | + // ... |
| 175 | + // store %2 to %0 |
| 176 | + // ... |
| 177 | + // use of %1 // additional last use: the lifetime of %1 ends here |
| 178 | + // ... // no destroy of %1! |
| 179 | + // unreachable |
| 180 | + // |
| 181 | + // TODO: we can remove this once with have completed OSSA lifetimes throughout the SIL pipeline. |
| 182 | + private(set) var additionalLastUses: Stack<Instruction> |
| 183 | + |
| 184 | + // Exit blocks of the load's liverange which don't have a destroy. |
| 185 | + // Those are successor blocks of terminators, like `switch_enum`, which do _not_ forward the value. |
| 186 | + // E.g. the none-case of a switch_enum of an Optional. |
| 187 | + private(set) var nonDestroyingLiverangeExits: Stack<BasicBlock> |
| 188 | + |
| 189 | + init(_ context: FunctionPassContext) { |
| 190 | + self.context = context |
| 191 | + self.forwardingUses = Stack(context) |
| 192 | + self.destroys = Stack(context) |
| 193 | + self.additionalLastUses = Stack(context) |
| 194 | + self.nonDestroyingLiverangeExits = Stack(context) |
| 195 | + } |
| 196 | + |
| 197 | + mutating func collectUses(of ownedValue: Value) -> Bool { |
| 198 | + var worklist = ValueWorklist(context) |
| 199 | + defer { worklist.deinitialize() } |
| 200 | + |
| 201 | + worklist.pushIfNotVisited(ownedValue) |
| 202 | + |
| 203 | + while let value = worklist.pop() { |
| 204 | + for use in value.uses.endingLifetime { |
| 205 | + switch use.instruction { |
| 206 | + case let destroy as DestroyValueInst: |
| 207 | + destroys.append(destroy) |
| 208 | + case let forwardingInst as ForwardingInstruction where forwardingInst.canChangeToGuaranteedOwnership: |
| 209 | + forwardingUses.append(use) |
| 210 | + findNonDestroyingLiverangeExits(of: forwardingInst) |
| 211 | + worklist.pushIfNotVisited(contentsOf: forwardingInst.forwardedResults.lazy.filter { $0.ownership == .owned}) |
| 212 | + default: |
| 213 | + return false |
| 214 | + } |
| 215 | + } |
| 216 | + // Get potential additional last uses in dead-end blocks for which the final destroy is missing. |
| 217 | + // TODO: we can remove this once with have completed OSSA lifetimes throughout the SIL pipeline. |
| 218 | + findAdditionalUsesInDeadEndBlocks(of: value) |
| 219 | + } |
| 220 | + return true |
| 221 | + } |
| 222 | + |
| 223 | + private mutating func findNonDestroyingLiverangeExits(of forwardingInst: ForwardingInstruction) { |
| 224 | + if let termInst = forwardingInst as? TermInst { |
| 225 | + // A terminator instruction can implicitly end the lifetime of its operand in a success block, |
| 226 | + // e.g. a `switch_enum` with a non-payload case block. Such success blocks need an `end_borrow`, though. |
| 227 | + for succ in termInst.successors where !succ.arguments.contains(where: {$0.ownership == .owned}) { |
| 228 | + nonDestroyingLiverangeExits.append(succ) |
| 229 | + } |
| 230 | + } |
| 231 | + } |
| 232 | + |
| 233 | + private mutating func findAdditionalUsesInDeadEndBlocks(of value: Value) { |
| 234 | + var ln = computeKnownLiveness(for: value, visitInnerUses: true, context) |
| 235 | + defer { ln.deinitialize() } |
| 236 | + additionalLastUses.append(contentsOf: ln.ends.filter { !($0 is DestroyValueInst)}) |
| 237 | + } |
| 238 | + |
| 239 | + mutating func deinitialize() { |
| 240 | + forwardingUses.deinitialize() |
| 241 | + destroys.deinitialize() |
| 242 | + additionalLastUses.deinitialize() |
| 243 | + nonDestroyingLiverangeExits.deinitialize() |
| 244 | + } |
| 245 | +} |
| 246 | + |
| 247 | +private extension ForwardingInstruction { |
| 248 | + var canChangeToGuaranteedOwnership: Bool { |
| 249 | + if !preservesReferenceCounts { |
| 250 | + return false |
| 251 | + } |
| 252 | + if !canForwardGuaranteedValues { |
| 253 | + return false |
| 254 | + } |
| 255 | + // For simplicity only support a single owned operand. Otherwise we would have to check if the other |
| 256 | + // owned operands stem from `load_borrow`s, too, which we can convert, etc. |
| 257 | + let numOwnedOperands = operands.lazy.filter({ $0.value.ownership == .owned }).count |
| 258 | + if numOwnedOperands > 1 { |
| 259 | + return false |
| 260 | + } |
| 261 | + return true |
| 262 | + } |
| 263 | +} |
| 264 | + |
| 265 | + |
| 266 | +private extension InstructionWorklist { |
| 267 | + |
| 268 | + init(liverangeOf load: LoadInst, with ownershipUses: OwnershipUses) { |
| 269 | + self.init(ownershipUses.context) |
| 270 | + |
| 271 | + for destroy in ownershipUses.destroys { |
| 272 | + pushPredecessors(of: destroy, ignoring: load) |
| 273 | + } |
| 274 | + pushIfNotVisited(contentsOf: ownershipUses.additionalLastUses) |
| 275 | + } |
| 276 | + |
| 277 | + mutating func mayWrite(toAddressOf load: LoadInst, _ context: FunctionPassContext) -> Bool { |
| 278 | + let aliasAnalysis = context.aliasAnalysis |
| 279 | + |
| 280 | + while let inst = pop() { |
| 281 | + if inst.mayWrite(toAddress: load.address, aliasAnalysis) { |
| 282 | + return true |
| 283 | + } |
| 284 | + pushPredecessors(of: inst, ignoring: load) |
| 285 | + } |
| 286 | + return false |
| 287 | + } |
| 288 | +} |
0 commit comments