SIL optimizer: a collection of optimizer changes to prepare for COW representation

include/swift/AST/SemanticAttrs.def

@@ -51,6 +51,7 @@ SEMANTICS_ATTR(ARRAY_GET_ELEMENT_ADDRESS, "array.get_element_address")
 SEMANTICS_ATTR(ARRAY_INIT, "array.init")
 SEMANTICS_ATTR(ARRAY_INIT_EMPTY, "array.init.empty")
 SEMANTICS_ATTR(ARRAY_MAKE_MUTABLE, "array.make_mutable")
+SEMANTICS_ATTR(ARRAY_END_MUTATION, "array.end_mutation")
 SEMANTICS_ATTR(ARRAY_MUTATE_UNKNOWN, "array.mutate_unknown")
 SEMANTICS_ATTR(ARRAY_PROPS_IS_NATIVE_TYPE_CHECKED, "array.props.isNativeTypeChecked")
 SEMANTICS_ATTR(ARRAY_RESERVE_CAPACITY_FOR_APPEND, "array.reserve_capacity_for_append")
@@ -67,6 +68,8 @@ SEMANTICS_ATTR(OPTIMIZE_SIL_SPECIALIZE_GENERIC_PARTIAL_NEVER,
           "optimize.sil.specialize.generic.partial.never")
 SEMANTICS_ATTR(OPTIMIZE_SIL_SPECIALIZE_GENERIC_SIZE_NEVER,
           "optimize.sil.specialize.generic.size.never")
+SEMANTICS_ATTR(OPTIMIZE_SIL_SPECIALIZE_OWNED2GUARANTEE_NEVER,
+          "optimize.sil.specialize.owned2guarantee.never")
 
 SEMANTICS_ATTR(OSLOG_MESSAGE_INIT_INTERPOLATION, "oslog.message.init_interpolation")
 SEMANTICS_ATTR(OSLOG_MESSAGE_INIT_STRING_LITERAL, "oslog.message.init_stringliteral")

include/swift/SILOptimizer/Analysis/ArraySemantic.h

@@ -31,6 +31,7 @@ enum class ArrayCallKind {
   kGetElement,
   kGetElementAddress,
   kMakeMutable,
+  kEndMutation,
   kMutateUnknown,
   kReserveCapacityForAppend,
   kWithUnsafeMutableBufferPointer,
@@ -42,7 +43,8 @@ enum class ArrayCallKind {
   // before this comment.
   kArrayInit,
   kArrayUninitialized,
-  kArrayUninitializedIntrinsic
+  kArrayUninitializedIntrinsic,
+  kArrayFinalizeIntrinsic
 };
 
 /// Return true is the given function is an array semantics call.
@@ -78,6 +80,8 @@ class ArraySemanticsCall {
   ArraySemanticsCall(SILValue V, StringRef semanticName,
                      bool matchPartialName);
 
+  ArraySemanticsCall() : SemanticsCall(nullptr) {}
+
   /// Can we hoist this call.
   bool canHoist(SILInstruction *To, DominanceInfo *DT) const;
 

include/swift/SILOptimizer/PassManager/Passes.def

@@ -120,6 +120,8 @@ PASS(CopyForwarding, "copy-forwarding",
      "Copy Forwarding to Remove Redundant Copies")
 PASS(CopyPropagation, "copy-propagation",
      "Copy propagation to Remove Redundant SSA Copies")
+PASS(COWOpts, "cow-opts",
+     "Optimize COW operations")
 PASS(Differentiation, "differentiation",
      "Automatic Differentiation")
 PASS(EpilogueARCMatcherDumper, "sil-epilogue-arc-dumper",

lib/SIL/Utils/InstructionUtils.cpp

@@ -128,7 +128,8 @@ SILValue swift::stripCastsWithoutMarkDependence(SILValue V) {
 
     auto K = V->getKind();
     if (isRCIdentityPreservingCast(K) ||
-        K == ValueKind::UncheckedTrivialBitCastInst) {
+        K == ValueKind::UncheckedTrivialBitCastInst ||
+        K == ValueKind::EndCOWMutationInst) {
       V = cast<SingleValueInstruction>(V)->getOperand(0);
       continue;
     }
@@ -308,7 +309,8 @@ bool swift::onlyAffectsRefCount(SILInstruction *user) {
 }
 
 bool swift::mayCheckRefCount(SILInstruction *User) {
-  return isa<IsUniqueInst>(User) || isa<IsEscapingClosureInst>(User);
+  return isa<IsUniqueInst>(User) || isa<IsEscapingClosureInst>(User) ||
+         isa<BeginCOWMutationInst>(User);
 }
 
 bool swift::isSanitizerInstrumentation(SILInstruction *Instruction) {

lib/SIL/Utils/Projection.cpp

@@ -371,10 +371,13 @@ Optional<ProjectionPath> ProjectionPath::getProjectionPath(SILValue Start,
 
   auto Iter = End;
   while (Start != Iter) {
-    Projection AP(Iter);
-    if (!AP.isValid())
-      break;
-    P.Path.push_back(AP);
+    // end_cow_mutation is not a projection, but we want to "see through" it.
+    if (!isa<EndCOWMutationInst>(Iter)) {
+      Projection AP(Iter);
+      if (!AP.isValid())
+        break;
+      P.Path.push_back(AP);
+    }
     Iter = cast<SingleValueInstruction>(*Iter).getOperand(0);
   }
 

lib/SILOptimizer/Analysis/ArraySemantic.cpp

@@ -33,12 +33,14 @@ ArrayCallKind swift::getArraySemanticsKind(SILFunction *f) {
             .StartsWith("array.init", ArrayCallKind::kArrayInit)
             .Case("array.uninitialized", ArrayCallKind::kArrayUninitialized)
             .Case("array.uninitialized_intrinsic", ArrayCallKind::kArrayUninitializedIntrinsic)
+            .Case("array.finalize_intrinsic", ArrayCallKind::kArrayFinalizeIntrinsic)
             .Case("array.check_subscript", ArrayCallKind::kCheckSubscript)
             .Case("array.check_index", ArrayCallKind::kCheckIndex)
             .Case("array.get_count", ArrayCallKind::kGetCount)
             .Case("array.get_capacity", ArrayCallKind::kGetCapacity)
             .Case("array.get_element", ArrayCallKind::kGetElement)
             .Case("array.make_mutable", ArrayCallKind::kMakeMutable)
+            .Case("array.end_mutation", ArrayCallKind::kEndMutation)
             .Case("array.get_element_address",
                   ArrayCallKind::kGetElementAddress)
             .Case("array.mutate_unknown", ArrayCallKind::kMutateUnknown)
@@ -101,10 +103,12 @@ bool swift::ArraySemanticsCall::isValidSignature() {
   }
   case ArrayCallKind::kCheckSubscript: {
     // Int, Bool, Self
-    if (SemanticsCall->getNumArguments() != 3 ||
-        !SemanticsCall->getArgument(0)->getType().isTrivial(*F))
+    unsigned numArgs = SemanticsCall->getNumArguments();
+    if (numArgs != 2 && numArgs != 3)
+      return false;
+    if (!SemanticsCall->getArgument(0)->getType().isTrivial(*F))
       return false;
-    if (!SemanticsCall->getArgument(1)->getType().isTrivial(*F))
+    if (numArgs == 3 && !SemanticsCall->getArgument(1)->getType().isTrivial(*F))
       return false;
     auto SelfConvention = FnTy->getSelfParameter().getConvention();
     return SelfConvention == ParameterConvention::Direct_Guaranteed ||
@@ -324,27 +328,26 @@ bool swift::ArraySemanticsCall::canHoist(SILInstruction *InsertBefore,
     // Not implemented yet.
     return false;
 
-  case ArrayCallKind::kCheckSubscript: {
-    auto IsNativeArg = getArrayPropertyIsNativeTypeChecked();
-    ArraySemanticsCall IsNative(IsNativeArg,
-                                "array.props.isNativeTypeChecked", true);
-    if (!IsNative) {
-      // Do we have a constant parameter?
-      auto *SI = dyn_cast<StructInst>(IsNativeArg);
-      if (!SI)
-        return false;
-      if (!isa<IntegerLiteralInst>(SI->getOperand(0)))
+  case ArrayCallKind::kCheckSubscript:
+    if (SILValue IsNativeArg = getArrayPropertyIsNativeTypeChecked()) {
+      ArraySemanticsCall IsNative(IsNativeArg,
+                                  "array.props.isNativeTypeChecked", true);
+      if (!IsNative) {
+        // Do we have a constant parameter?
+        auto *SI = dyn_cast<StructInst>(IsNativeArg);
+        if (!SI)
+          return false;
+        if (!isa<IntegerLiteralInst>(SI->getOperand(0)))
+          return false;
+      } else if (!IsNative.canHoist(InsertBefore, DT))
+        // Otherwise, we must be able to hoist the function call.
         return false;
-    } else if (!IsNative.canHoist(InsertBefore, DT))
-      // Otherwise, we must be able to hoist the function call.
-      return false;
-
+    }
     return canHoistArrayArgument(SemanticsCall, getSelf(), InsertBefore, DT);
-  }
 
-  case ArrayCallKind::kMakeMutable: {
+  case ArrayCallKind::kMakeMutable:
+  case ArrayCallKind::kEndMutation:
     return canHoistArrayArgument(SemanticsCall, getSelf(), InsertBefore, DT);
-  }
   } // End switch.
 
   return false;
@@ -448,9 +451,8 @@ ApplyInst *swift::ArraySemanticsCall::hoistOrCopy(SILInstruction *InsertBefore,
         hoistOrCopySelf(SemanticsCall, InsertBefore, DT, LeaveOriginal);
 
     SILValue NewArrayProps;
-    if (Kind == ArrayCallKind::kCheckSubscript) {
+    if (SILValue IsNativeArg = getArrayPropertyIsNativeTypeChecked()) {
       // Copy the array.props argument call.
-      auto IsNativeArg = getArrayPropertyIsNativeTypeChecked();
       ArraySemanticsCall IsNative(IsNativeArg,
                                   "array.props.isNativeTypeChecked", true);
       if (!IsNative) {
@@ -492,8 +494,8 @@ ApplyInst *swift::ArraySemanticsCall::hoistOrCopy(SILInstruction *InsertBefore,
     return Call;
   }
 
-  case ArrayCallKind::kMakeMutable: {
-    assert(!LeaveOriginal && "Copying not yet implemented");
+  case ArrayCallKind::kMakeMutable:
+  case ArrayCallKind::kEndMutation: {
     // Hoist the call.
     auto Call = hoistOrCopyCall(SemanticsCall, InsertBefore, LeaveOriginal, DT);
     return Call;
@@ -515,14 +517,15 @@ void swift::ArraySemanticsCall::removeCall() {
 
   switch (getKind()) {
   default: break;
-  case ArrayCallKind::kCheckSubscript: {
-    // Remove all uses with the empty tuple ().
-    auto EmptyDep = SILBuilderWithScope(SemanticsCall)
-                        .createStruct(SemanticsCall->getLoc(),
-                                      SemanticsCall->getType(), {});
-    SemanticsCall->replaceAllUsesWith(EmptyDep);
-  }
-  break;
+  case ArrayCallKind::kCheckSubscript:
+    if (!SemanticsCall->getType().isVoid()){
+      // Remove all uses with the empty tuple ().
+      auto EmptyDep = SILBuilderWithScope(SemanticsCall)
+                          .createStruct(SemanticsCall->getLoc(),
+                                        SemanticsCall->getType(), {});
+      SemanticsCall->replaceAllUsesWith(EmptyDep);
+    }
+    break;
   case ArrayCallKind::kGetElement: {
     // Remove the matching isNativeTypeChecked and check_subscript call.
     ArraySemanticsCall IsNative(getTypeCheckedArgument(),
@@ -552,11 +555,13 @@ SILValue
 swift::ArraySemanticsCall::getArrayPropertyIsNativeTypeChecked() const {
   switch (getKind()) {
     case ArrayCallKind::kCheckSubscript:
-      return SemanticsCall->getArgument(1);
+      if (SemanticsCall->getNumArguments() == 3)
+        return SemanticsCall->getArgument(1);
+      return SILValue();
     case ArrayCallKind::kGetElement:
       return getTypeCheckedArgument();
     default:
-      llvm_unreachable("Must have an array.props argument");
+      return SILValue();
   }
 }
 
@@ -569,6 +574,7 @@ bool swift::ArraySemanticsCall::doesNotChangeArray() const {
     case ArrayCallKind::kGetCount:
     case ArrayCallKind::kGetCapacity:
     case ArrayCallKind::kGetElement:
+    case ArrayCallKind::kEndMutation:
       return true;
   }
 }

lib/SILOptimizer/Analysis/EscapeAnalysis.cpp

@@ -2285,6 +2285,7 @@ void EscapeAnalysis::analyzeInstruction(SILInstruction *I,
     case SILInstructionKind::SelectValueInst:
       analyzeSelectInst(cast<SelectValueInst>(I), ConGraph);
       return;
+    case SILInstructionKind::EndCOWMutationInst:
     case SILInstructionKind::StructInst:
     case SILInstructionKind::TupleInst:
     case SILInstructionKind::EnumInst: {

lib/SILOptimizer/Analysis/SimplifyInstruction.cpp

@@ -62,6 +62,7 @@ namespace {
     SILValue visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *UBCI);
     SILValue
     visitUncheckedTrivialBitCastInst(UncheckedTrivialBitCastInst *UTBCI);
+    SILValue visitEndCOWMutationInst(EndCOWMutationInst *ECM);
     SILValue visitThinFunctionToPointerInst(ThinFunctionToPointerInst *TFTPI);
     SILValue visitPointerToThinFunctionInst(PointerToThinFunctionInst *PTTFI);
     SILValue visitBeginAccessInst(BeginAccessInst *BAI);
@@ -329,6 +330,21 @@ visitUnconditionalCheckedCastInst(UnconditionalCheckedCastInst *UCCI) {
   return SILValue();
 }
 
+/// If the only use of a cast is a destroy, just destroy the cast operand.
+static SILValue simplifyDeadCast(SingleValueInstruction *Cast) {
+  for (Operand *op : Cast->getUses()) {
+    switch (op->getUser()->getKind()) {
+      case SILInstructionKind::DestroyValueInst:
+      case SILInstructionKind::StrongReleaseInst:
+      case SILInstructionKind::StrongRetainInst:
+        break;
+      default:
+        return SILValue();
+    }
+  }
+  return Cast->getOperand(0);
+}
+
 SILValue
 InstSimplifier::
 visitUncheckedRefCastInst(UncheckedRefCastInst *OPRI) {
@@ -351,7 +367,8 @@ visitUncheckedRefCastInst(UncheckedRefCastInst *OPRI) {
   if (OPRI->getOperand()->getType() == OPRI->getType())
     return OPRI->getOperand();
 
-  return SILValue();
+  // (destroy_value (unchecked_ref_cast x)) -> destroy_value x
+  return simplifyDeadCast(OPRI);
 }
 
 SILValue
@@ -375,7 +392,8 @@ SILValue InstSimplifier::visitUpcastInst(UpcastInst *UI) {
     if (URCI->getOperand()->getType() == UI->getType())
       return URCI->getOperand();
 
-  return SILValue();
+  // (destroy_value (upcast x)) -> destroy_value x
+  return simplifyDeadCast(UI);
 }
 
 #define LOADABLE_REF_STORAGE(Name, ...) \
@@ -410,6 +428,11 @@ visitUncheckedTrivialBitCastInst(UncheckedTrivialBitCastInst *UTBCI) {
   return SILValue();
 }
 
+SILValue InstSimplifier::visitEndCOWMutationInst(EndCOWMutationInst *ECM) {
+  // (destroy_value (end_cow_mutation x)) -> destroy_value x
+  return simplifyDeadCast(ECM);
+}
+
 SILValue
 InstSimplifier::
 visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *UBCI) {

lib/SILOptimizer/FunctionSignatureTransforms/OwnedToGuaranteedTransform.cpp

@@ -13,6 +13,7 @@
 #define DEBUG_TYPE "fso-owned-to-guaranteed-transform"
 #include "FunctionSignatureOpts.h"
 #include "swift/SIL/DebugUtils.h"
+#include "swift/AST/SemanticAttrs.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace swift;
@@ -258,6 +259,9 @@ void FunctionSignatureTransform::OwnedToGuaranteedAddResultRelease(
 bool FunctionSignatureTransform::OwnedToGuaranteedAnalyze() {
   if (FSODisableOwnedToGuaranteed)
     return false;
+  SILFunction *F = TransformDescriptor.OriginalFunction;
+  if (F->hasSemanticsAttr(semantics::OPTIMIZE_SIL_SPECIALIZE_OWNED2GUARANTEE_NEVER))
+    return false;
 
   const bool Result = OwnedToGuaranteedAnalyzeResults();
   const bool Params = OwnedToGuaranteedAnalyzeParameters();

lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp

@@ -288,6 +288,7 @@ static bool isNonMutatingArraySemanticCall(SILInstruction *Inst) {
   case ArrayCallKind::kGetCapacity:
   case ArrayCallKind::kGetElement:
   case ArrayCallKind::kGetElementAddress:
+  case ArrayCallKind::kEndMutation:
     return true;
   case ArrayCallKind::kMakeMutable:
   case ArrayCallKind::kMutateUnknown:
@@ -296,6 +297,7 @@ static bool isNonMutatingArraySemanticCall(SILInstruction *Inst) {
   case ArrayCallKind::kArrayInit:
   case ArrayCallKind::kArrayUninitialized:
   case ArrayCallKind::kArrayUninitializedIntrinsic:
+  case ArrayCallKind::kArrayFinalizeIntrinsic:
   case ArrayCallKind::kAppendContentsOf:
   case ArrayCallKind::kAppendElement:
     return false;

lib/SILOptimizer/PassManager/PassPipeline.cpp

@@ -366,6 +366,7 @@ void addFunctionPasses(SILPassPipelinePlan &P,
     P.addRedundantLoadElimination();
   }
 
+  P.addCOWOpts();
   P.addPerformanceConstantPropagation();
   // Remove redundant arguments right before CSE and DCE, so that CSE and DCE
   // can cleanup redundant and dead instructions.
@@ -514,6 +515,7 @@ static void addMidLevelFunctionPipeline(SILPassPipelinePlan &P) {
   P.addLICM();
   // Run loop unrolling after inlining and constant propagation, because loop
   // trip counts may have became constant.
+  P.addLICM();
   P.addLoopUnroll();
 }
 
@@ -595,6 +597,7 @@ static void addLateLoopOptPassPipeline(SILPassPipelinePlan &P) {
   P.addAccessEnforcementReleaseSinking();
   P.addAccessEnforcementOpts();
   P.addLICM();
+  P.addCOWOpts();
   // Simplify CFG after LICM that creates new exit blocks
   P.addSimplifyCFG();
   // LICM might have added new merging potential by hoisting

lib/SILOptimizer/SILCombiner/SILCombiner.h

@@ -183,6 +183,7 @@ class SILCombiner :
   SILInstruction *visitPointerToAddressInst(PointerToAddressInst *PTAI);
   SILInstruction *visitUncheckedAddrCastInst(UncheckedAddrCastInst *UADCI);
   SILInstruction *visitUncheckedRefCastInst(UncheckedRefCastInst *URCI);
+  SILInstruction *visitEndCOWMutationInst(EndCOWMutationInst *URCI);
   SILInstruction *visitUncheckedRefCastAddrInst(UncheckedRefCastAddrInst *URCI);
   SILInstruction *visitBridgeObjectToRefInst(BridgeObjectToRefInst *BORI);
   SILInstruction *visitUnconditionalCheckedCastInst(

lib/SILOptimizer/SILCombiner/SILCombinerCastVisitors.cpp

@@ -264,6 +264,26 @@ SILCombiner::visitUncheckedRefCastInst(UncheckedRefCastInst *URCI) {
   return nullptr;
 }
 
+SILInstruction *SILCombiner::visitEndCOWMutationInst(EndCOWMutationInst *ECM) {
+
+  // Remove a cast if it's only used by an end_cow_mutation.
+  //
+  // (end_cow_mutation (upcast X)) -> (end_cow_mutation X)
+  // (end_cow_mutation (unchecked_ref_cast X)) -> (end_cow_mutation X)
+  SILValue op = ECM->getOperand();
+  if (!isa<UncheckedRefCastInst>(op) && !isa<UpcastInst>(op))
+    return nullptr;
+  if (!op->hasOneUse())
+    return nullptr;
+
+  SingleValueInstruction *refCast = cast<SingleValueInstruction>(op);
+  auto *newECM = Builder.createEndCOWMutation(ECM->getLoc(),
+                                              refCast->getOperand(0));
+  ECM->replaceAllUsesWith(refCast);
+  refCast->setOperand(0, newECM);
+  refCast->moveAfter(newECM);
+  return eraseInstFromFunction(*ECM);
+}
 
 SILInstruction *
 SILCombiner::visitBridgeObjectToRefInst(BridgeObjectToRefInst *BORI) {