|
| 1 | +//===--- LoadCopyToBorrowOptimization.swift --------------------------------==// |
| 2 | +// |
| 3 | +// This source file is part of the Swift.org open source project |
| 4 | +// |
| 5 | +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors |
| 6 | +// Licensed under Apache License v2.0 with Runtime Library Exception |
| 7 | +// |
| 8 | +// See https://swift.org/LICENSE.txt for license information |
| 9 | +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| 10 | +// |
| 11 | +//===----------------------------------------------------------------------===// |
| 12 | + |
| 13 | +import SIL |
| 14 | + |
| 15 | +/// Replaces a `load [copy]` with a `load_borrow` if possible. |
| 16 | +/// |
| 17 | +/// ``` |
| 18 | +/// %1 = load [copy] %0 |
| 19 | +/// // no writes to %0 |
| 20 | +/// destroy_value %1 |
| 21 | +/// ``` |
| 22 | +/// -> |
| 23 | +/// ``` |
| 24 | +/// %1 = load_borrow %0 |
| 25 | +/// // no writes to %0 |
| 26 | +/// end_borrow %1 |
| 27 | +/// ``` |
| 28 | +/// |
| 29 | +/// The optimization can be done if: |
| 30 | +/// * During the (forward-extended) lifetime of the loaded value the memory location is not changed. |
| 31 | +/// * All (forward-extended) uses of the loaded value support guaranteed ownership. |
| 32 | +/// * The (forward-extended) lifetime of the loaded value ends with `destroy_value`(s). |
| 33 | +/// |
| 34 | +let loadCopyToBorrowOptimization = FunctionPass(name: "load-copy-to-borrow-optimization") { |
| 35 | + (function: Function, context: FunctionPassContext) in |
| 36 | + |
| 37 | + if !function.hasOwnership { |
| 38 | + return |
| 39 | + } |
| 40 | + |
| 41 | + for inst in function.instructions { |
| 42 | + if let load = inst as? LoadInst { |
| 43 | + optimize(load: load, context) |
| 44 | + } |
| 45 | + } |
| 46 | +} |
| 47 | + |
| 48 | +private func optimize(load: LoadInst, _ context: FunctionPassContext) { |
| 49 | + if load.loadOwnership != .copy { |
| 50 | + return |
| 51 | + } |
| 52 | + |
| 53 | + var ownershipUses = OwnershipUses(context) |
| 54 | + defer { ownershipUses.deinitialize() } |
| 55 | + |
| 56 | + // Handle the special case if the `load` is immediately followed by a single `move_value`. |
| 57 | + // Note that `move_value` is _not_ a forwarding instruction. |
| 58 | + // In this case we have to preserve the move's flags by inserting a `begin_borrow` with the same flags. |
| 59 | + // For example: |
| 60 | + // |
| 61 | + // %1 = load [copy] %0 |
| 62 | + // %2 = move_value [lexical] %1 |
| 63 | + // ... |
| 64 | + // destroy_value %2 |
| 65 | + // -> |
| 66 | + // %1 = load_borrow %0 |
| 67 | + // %2 = begin_borrow [lexical] %1 |
| 68 | + // ... |
| 69 | + // end_borrow %2 |
| 70 | + // end_borrow %1 |
| 71 | + // |
| 72 | + let loadedValue = load.singleMoveValueUser ?? load |
| 73 | + |
| 74 | + // This is required if the lifetime of the `load` ends up in a dead-end block and the `destroy_value` is |
| 75 | + // missing at the `unreachable`. In such a case the dataflow would _not_ visit potential writes to the load's |
| 76 | + // memory location. In the following example, the `load [copy]` must not be converted to a `load_borrow`: |
| 77 | + // |
| 78 | + // %1 = load [copy] %0 |
| 79 | + // ... |
| 80 | + // store %2 to %0 |
| 81 | + // ... // no destroy of %1! |
| 82 | + // unreachable // the lifetime of %1 ends here |
| 83 | + // |
| 84 | + context.completeLifetime(of: loadedValue) |
| 85 | + |
| 86 | + if !ownershipUses.collectUses(of: loadedValue) { |
| 87 | + return |
| 88 | + } |
| 89 | + |
| 90 | + var liferange = InstructionWorklist(context) |
| 91 | + defer { liferange.deinitialize() } |
| 92 | + |
| 93 | + if liferange.mayWrite(toAddressOf: load, within: ownershipUses.destroys, context) { |
| 94 | + return |
| 95 | + } |
| 96 | + |
| 97 | + load.replaceWithLoadBorrow(liferange: liferange, ownershipUses: ownershipUses) |
| 98 | +} |
| 99 | + |
| 100 | +private extension LoadInst { |
| 101 | + var singleMoveValueUser: MoveValueInst? { |
| 102 | + uses.ignoreDebugUses.singleUse?.instruction as? MoveValueInst |
| 103 | + } |
| 104 | + |
| 105 | + func replaceWithLoadBorrow(liferange: InstructionWorklist, ownershipUses: OwnershipUses) { |
| 106 | + let context = ownershipUses.context |
| 107 | + let builder = Builder(before: self, context) |
| 108 | + let loadBorrow = builder.createLoadBorrow(fromAddress: address) |
| 109 | + |
| 110 | + let innerBorrow: BeginBorrowInst? |
| 111 | + if let moveInst = singleMoveValueUser { |
| 112 | + // An inner borrow is needed to keep the flags of the `move_value`. |
| 113 | + let bbi = builder.createBeginBorrow(of: loadBorrow, |
| 114 | + isLexical: moveInst.isLexical, |
| 115 | + hasPointerEscape: moveInst.hasPointerEscape, |
| 116 | + isFromVarDecl: moveInst.isFromVarDecl) |
| 117 | + moveInst.uses.replaceAll(with: bbi, context) |
| 118 | + context.erase(instruction: moveInst) |
| 119 | + innerBorrow = bbi |
| 120 | + } else { |
| 121 | + innerBorrow = nil |
| 122 | + } |
| 123 | + |
| 124 | + uses.replaceAll(with: loadBorrow, context) |
| 125 | + context.erase(instruction: self) |
| 126 | + |
| 127 | + for destroy in ownershipUses.destroys { |
| 128 | + context.createEndBorrows(ofInner: innerBorrow, ofOuter: loadBorrow, before: destroy, ifNotIn: liferange) |
| 129 | + context.erase(instruction: destroy) |
| 130 | + } |
| 131 | + |
| 132 | + for forwardingUse in ownershipUses.forwardingUses { |
| 133 | + forwardingUse.changeOwnership(from: .owned, to: .guaranteed, context) |
| 134 | + } |
| 135 | + |
| 136 | + for liferangeExitBlock in ownershipUses.nonDestroyingLiferangeExits { |
| 137 | + context.createEndBorrows(ofInner: innerBorrow, ofOuter: loadBorrow, |
| 138 | + before: liferangeExitBlock.instructions.first!, |
| 139 | + ifNotIn: liferange) |
| 140 | + } |
| 141 | + } |
| 142 | +} |
| 143 | + |
| 144 | +private extension FunctionPassContext { |
| 145 | + func createEndBorrows( |
| 146 | + ofInner innerBorrow: BeginBorrowInst?, |
| 147 | + ofOuter outerBorrow: LoadBorrowInst, |
| 148 | + before insertionPoint: Instruction, |
| 149 | + ifNotIn liferange: InstructionWorklist |
| 150 | + ) { |
| 151 | + // There can be multiple destroys in a row in case of decomposing an aggregate, e.g. |
| 152 | + // %1 = load [copy] %0 |
| 153 | + // ... |
| 154 | + // (%2, %3) = destructure_struct %1 |
| 155 | + // destroy_value %2 |
| 156 | + // destroy_value %3 // The final destroy. Here we need to create the `end_borrow`(s) |
| 157 | + // |
| 158 | + if liferange.hasBeenPushed(insertionPoint) { |
| 159 | + return |
| 160 | + } |
| 161 | + let builder = Builder(before: insertionPoint, self) |
| 162 | + if let innerBorrow = innerBorrow { |
| 163 | + builder.createEndBorrow(of: innerBorrow) |
| 164 | + } |
| 165 | + builder.createEndBorrow(of: outerBorrow) |
| 166 | + } |
| 167 | + |
| 168 | +} |
| 169 | + |
| 170 | +private struct OwnershipUses { |
| 171 | + private(set) var forwardingUses: Stack<Operand> |
| 172 | + private(set) var destroys: Stack<DestroyValueInst> |
| 173 | + private(set) var nonDestroyingLiferangeExits: Stack<BasicBlock> |
| 174 | + let context: FunctionPassContext |
| 175 | + |
| 176 | + init(_ context: FunctionPassContext) { |
| 177 | + self.forwardingUses = Stack(context) |
| 178 | + self.destroys = Stack(context) |
| 179 | + self.nonDestroyingLiferangeExits = Stack(context) |
| 180 | + self.context = context |
| 181 | + } |
| 182 | + |
| 183 | + mutating func collectUses(of ownedValue: Value) -> Bool { |
| 184 | + var worklist = ValueWorklist(context) |
| 185 | + defer { worklist.deinitialize() } |
| 186 | + |
| 187 | + worklist.pushIfNotVisited(ownedValue) |
| 188 | + |
| 189 | + while let value = worklist.pop() { |
| 190 | + for use in value.uses.endingLifetime { |
| 191 | + switch use.instruction { |
| 192 | + case let destroy as DestroyValueInst: |
| 193 | + destroys.append(destroy) |
| 194 | + case let forwardingInst as ForwardingInstruction where forwardingInst.canChangeToGuaranteedOwnership: |
| 195 | + forwardingUses.append(use) |
| 196 | + findNonDestroyingLiferangeExists(of: forwardingInst) |
| 197 | + worklist.pushIfNotVisited(contentsOf: forwardingInst.forwardedResults) |
| 198 | + default: |
| 199 | + return false |
| 200 | + } |
| 201 | + } |
| 202 | + } |
| 203 | + return true |
| 204 | + } |
| 205 | + |
| 206 | + private mutating func findNonDestroyingLiferangeExists(of forwardingInst: ForwardingInstruction) { |
| 207 | + if let termInst = forwardingInst as? TermInst { |
| 208 | + // A terminator instruction can implicitly end the lifetime of its operand in a success block, |
| 209 | + // e.g. a `switch_enum` with a non-payload case block. Such success blocks need an `end_borrow`, though. |
| 210 | + for succ in termInst.successors where succ.arguments.isEmpty { |
| 211 | + nonDestroyingLiferangeExits.append(succ) |
| 212 | + } |
| 213 | + } |
| 214 | + } |
| 215 | + |
| 216 | + mutating func deinitialize() { |
| 217 | + forwardingUses.deinitialize() |
| 218 | + destroys.deinitialize() |
| 219 | + nonDestroyingLiferangeExits.deinitialize() |
| 220 | + } |
| 221 | +} |
| 222 | + |
| 223 | +private extension ForwardingInstruction { |
| 224 | + var canChangeToGuaranteedOwnership: Bool { |
| 225 | + if !preservesRepresentation { |
| 226 | + return false |
| 227 | + } |
| 228 | + if !canForwardGuaranteedValues { |
| 229 | + return false |
| 230 | + } |
| 231 | + // For simplicity only support a single owned operand. Otherwise we would have to check if the other |
| 232 | + // owned operands stem from `load_borrow`s, too, which we can convert, etc. |
| 233 | + let numOwnedOperands = operands.lazy.filter({ $0.value.ownership == .owned }).count |
| 234 | + if numOwnedOperands > 1 { |
| 235 | + return false |
| 236 | + } |
| 237 | + return true |
| 238 | + } |
| 239 | +} |
| 240 | + |
| 241 | + |
| 242 | +private extension InstructionWorklist { |
| 243 | + |
| 244 | + mutating func mayWrite( |
| 245 | + toAddressOf load: LoadInst, |
| 246 | + within destroys: Stack<DestroyValueInst>, |
| 247 | + _ context: FunctionPassContext |
| 248 | + ) -> Bool { |
| 249 | + let aliasAnalysis = context.aliasAnalysis |
| 250 | + |
| 251 | + for destroy in destroys { |
| 252 | + pushPredecessors(of: destroy, ignoring: load) |
| 253 | + } |
| 254 | + |
| 255 | + while let inst = pop() { |
| 256 | + if inst.mayWrite(toAddress: load.address, aliasAnalysis) { |
| 257 | + return true |
| 258 | + } |
| 259 | + pushPredecessors(of: inst, ignoring: load) |
| 260 | + } |
| 261 | + |
| 262 | + return false |
| 263 | + } |
| 264 | +} |
0 commit comments