Fix critical edges

lib/SIL/SILVerifier.cpp

@@ -4530,29 +4530,26 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
       return true;
     };
 
-    SILModule &M = F->getModule();
     for (auto &BB : *F) {
       TermInst *TI = BB.getTerminator();
       CurInstruction = TI;
 
-      // Check for non-cond_br critical edges in canonical SIL.
-      if (!isa<CondBranchInst>(TI) && M.getStage() == SILStage::Canonical) {
+      // Check for non-cond_br critical edges.
+      auto *CBI = dyn_cast<CondBranchInst>(TI);
+      if (!CBI) {
         for (unsigned Idx = 0, e = BB.getSuccessors().size(); Idx != e; ++Idx) {
           require(!isCriticalEdgePred(TI, Idx),
                   "non cond_br critical edges not allowed");
         }
         continue;
       }
-
       // In ownership qualified SIL, ban critical edges from CondBranchInst that
       // have non-trivial arguments.
+      //
+      // FIXME: it would be far simpler to ban all critical edges in general.
       if (!F->hasQualifiedOwnership())
         continue;
 
-      auto *CBI = dyn_cast<CondBranchInst>(TI);
-      if (!CBI)
-        continue;
-
       if (isCriticalEdgePred(CBI, CondBranchInst::TrueIdx)) {
         require(
             llvm::all_of(CBI->getTrueArgs(),

lib/SILOptimizer/Transforms/ARCCodeMotion.cpp

@@ -96,10 +96,6 @@ STATISTIC(NumReleasesHoisted, "Number of releases hoisted");
 
 llvm::cl::opt<bool> DisableARCCodeMotion("disable-arc-cm", llvm::cl::init(false));
 
-/// Disable optimization if we have to break critical edges in the function.
-llvm::cl::opt<bool>
-DisableIfWithCriticalEdge("disable-with-critical-edge", llvm::cl::init(false));
-
 //===----------------------------------------------------------------------===//
 //                             Block State 
 //===----------------------------------------------------------------------===//
@@ -1156,11 +1152,6 @@ class ARCCodeMotion : public SILFunctionTransform {
     if (!F->shouldOptimize())
       return;
 
-    // Return if there is critical edge and we are disabling critical edge
-    // splitting.
-    if (DisableIfWithCriticalEdge && hasCriticalEdges(*F, false))
-      return;
-
     LLVM_DEBUG(llvm::dbgs() << "*** ARCCM on function: " << F->getName()
                             << " ***\n");
 

test/DebugInfo/return.swift

@@ -20,10 +20,9 @@ public func ifelseexpr() -> Int64 {
   } else {
     x.x -= 1
   }
-  // CHECK:  [[X:%.*]] = load %T6return1XC*, %T6return1XC** [[ALLOCA]]
-  // CHECK:  @swift_release to void (%T6return1XC*)*)(%T6return1XC* [[X]])
+  // CHECK:  [[L:%.*]] = load %T6return1XC*, %T6return1XC** [[ALLOCA]]
+  // CHECK:  @swift_release to void (%T6return1XC*)*)(%T6return1XC* [[L]])
   // CHECK-SAME:                    , !dbg ![[RELEASE:.*]]
-
   // The ret instruction should be in the same scope as the return expression.
   // CHECK:  ret{{.*}}, !dbg ![[RELEASE]]
   return x.x // CHECK: ![[RELEASE]] = !DILocation(line: [[@LINE]], column: 3

test/IRGen/dynamic_cast.sil

@@ -67,8 +67,10 @@ bb0(%0 : $*P):
   %1 = alloc_stack $S
   checked_cast_addr_br take_always P in %0 : $*P to S in %1 : $*S, bb1, bb2
 bb1:
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   destroy_addr %1 : $*S
   dealloc_stack %1 : $*S
   %2 = tuple ()
@@ -88,8 +90,10 @@ bb0(%0 : $*P):
   %1 = alloc_stack $S
   checked_cast_addr_br take_on_success P in %0 : $*P to S in %1 : $*S, bb1, bb2
 bb1:
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   destroy_addr %1 : $*S
   dealloc_stack %1 : $*S
   %2 = tuple ()
@@ -109,8 +113,10 @@ bb0(%0 : $*P):
   %1 = alloc_stack $S
   checked_cast_addr_br copy_on_success P in %0 : $*P to S in %1 : $*S, bb1, bb2
 bb1:
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   destroy_addr %1 : $*S
   dealloc_stack %1 : $*S
   %2 = tuple ()

test/IRGen/enum.sil

@@ -568,12 +568,10 @@ entry(%u : $SinglePayloadNoXI2):
 // CHECK:   ]
 // CHECK: ; <label>:[[DFLT]]
 // CHECK:   unreachable
-  switch_enum %u : $SinglePayloadNoXI2, case #SinglePayloadNoXI2.x!enumelt.1: x_dest, case #SinglePayloadNoXI2.y!enumelt: y_dest, case #SinglePayloadNoXI2.z!enumelt: z_dest, default end
+  switch_enum %u : $SinglePayloadNoXI2, case #SinglePayloadNoXI2.x!enumelt.1: x_dest, case #SinglePayloadNoXI2.y!enumelt: y_dest, case #SinglePayloadNoXI2.z!enumelt: z_dest, default default_dest
 
 // CHECK: ; <label>:[[X_PREDEST]]
 // CHECK:   br label %[[X_DEST:[0-9]+]]
-// CHECK: ; <label>:[[SPLITEDGE]]
-// CHECK:   br label %[[END:[0-9]+]]
 // CHECK: ; <label>:[[X_DEST]]
 // CHECK:   {{%.*}} = phi [[WORD]] [ %0, %[[X_PREDEST]] ]
 x_dest(%u2 : $Builtin.Word):
@@ -599,6 +597,11 @@ z_dest:
   apply %c() : $@convention(thin) () -> ()
   br end
 
+// CHECK: ; <label>:[[SPLITEDGE]]
+// CHECK:   br label %[[END:[0-9]+]]
+default_dest:
+  br end
+
 // CHECK: ; <label>:[[END]]
 // CHECK:   ret void
 end:
@@ -688,7 +691,7 @@ sil @int64_sink : $@convention(thin) (Builtin.Word) -> ()
 // CHECK:         ([[WORD]], [[WORD]], i8) {{.*}} {
 sil @aggregate_single_payload_unpack : $@convention(thin) (AggregateSinglePayload) -> () {
 entry(%u : $AggregateSinglePayload):
-  switch_enum %u : $AggregateSinglePayload, case #AggregateSinglePayload.x!enumelt.1: x_dest, default end
+  switch_enum %u : $AggregateSinglePayload, case #AggregateSinglePayload.x!enumelt.1: x_dest, default default_dest
 
   // CHECK: [[FIRST:%.*]] = phi [[WORD]] [ %0
   // CHECK: [[SECOND:%.*]] = phi [[WORD]] [ %1
@@ -702,6 +705,9 @@ x_dest(%v : $(Builtin.Word, Builtin.Word)):
   %d = apply %f(%b) : $@convention(thin) (Builtin.Word) -> ()
   br end
 
+default_dest:
+  br end
+
 end:
   %x = tuple ()
   return %x : $()
@@ -745,7 +751,7 @@ enum AggregateSinglePayload2 {
 // CHECK-32: define{{( dllexport)?}}{{( protected)?}}  swiftcc void @aggregate_single_payload_unpack_2(%T4enum23AggregateSinglePayload2O* noalias nocapture dereferenceable(16)) {{.*}} {
 sil @aggregate_single_payload_unpack_2 : $@convention(thin) (AggregateSinglePayload2) -> () {
 entry(%u : $AggregateSinglePayload2):
-  switch_enum %u : $AggregateSinglePayload2, case #AggregateSinglePayload2.x!enumelt.1: x_dest, default end
+  switch_enum %u : $AggregateSinglePayload2, case #AggregateSinglePayload2.x!enumelt.1: x_dest, default default_dest
 
 // CHECK-64:   [[TRUNC:%.*]] = trunc [[WORD]] %0 to i21
 // CHECK-64:   phi i21 [ [[TRUNC]]
@@ -755,6 +761,9 @@ entry(%u : $AggregateSinglePayload2):
 x_dest(%v : $(CharLike, IntLike, RangeLike)):
   br end
 
+default_dest:
+  br end
+
 end:
   %x = tuple ()
   return %x : $()
@@ -796,7 +805,7 @@ enum AggregateSinglePayload3 {
 // CHECK: define{{( dllexport)?}}{{( protected)?}} swiftcc void @aggregate_single_payload_unpack_3([[WORD]], [[WORD]]) {{.*}} {
 sil @aggregate_single_payload_unpack_3 : $@convention(thin) (AggregateSinglePayload3) -> () {
 entry(%u : $AggregateSinglePayload3):
-  switch_enum %u : $AggregateSinglePayload3, case #AggregateSinglePayload3.x!enumelt.1: x_dest, default end
+  switch_enum %u : $AggregateSinglePayload3, case #AggregateSinglePayload3.x!enumelt.1: x_dest, default default_dest
 
   // CHECK: [[TRUNC:%.*]] = trunc [[WORD]] %0 to i21
   // CHECK: [[FIRST:%.*]] = phi i21 [ [[TRUNC]]
@@ -813,6 +822,9 @@ x_dest(%v : $(Builtin.Int21, Builtin.Word)):
   %d = apply %g(%b) : $@convention(thin) (Builtin.Word) -> ()
   br end
 
+default_dest:
+  br end
+
 end:
   %x = tuple ()
   return %x : $()

test/IRGen/partial_apply.sil

@@ -516,8 +516,11 @@ bb(%0 : $Int):
 
 bb1(%err: $Error):
   %t = tuple ()
-  br bb2(%t: $())
+  br bb3(%t: $())
 
-bb2(%v : $()):
+bb2(%r : $()):
+  br bb3(%r : $())
+
+bb3(%v : $()):
   return %v : $()
 }

test/SIL/Parser/undef.sil

@@ -313,17 +313,25 @@ bb0:
 
 sil @switch_enum_test : $() -> () {
 bb0:
-  // CHECK: switch_enum undef : $E, case #E.Case!enumelt: bb1, default bb1
-  switch_enum undef : $E, case #E.Case!enumelt: bb1, default bb1
+  // CHECK: switch_enum undef : $E, case #E.Case!enumelt: bb1, default bb2
+  switch_enum undef : $E, case #E.Case!enumelt: bb1, default bb2
 bb1:
+  br bb3
+bb2:
+  br bb3
+bb3:
   return undef : $()
 }
 
 sil @switch_enum_addr_test : $() -> () {
 bb0:
-  // CHECK: switch_enum_addr undef : $*E, case #E.Case!enumelt: bb1, default bb1
-  switch_enum_addr undef : $*E, case #E.Case!enumelt: bb1, default bb1
+  // CHECK: switch_enum_addr undef : $*E, case #E.Case!enumelt: bb1, default bb2
+  switch_enum_addr undef : $*E, case #E.Case!enumelt: bb1, default bb2
 bb1:
+  br bb3
+bb2:
+  br bb3
+bb3:
   return undef : $()
 }
 
@@ -332,8 +340,10 @@ bb0:
   // CHECK: dynamic_method_br undef : $P, #C.fn!1.foreign, bb1, bb2
   dynamic_method_br undef : $P, #C.fn!1.foreign, bb1, bb2
 bb1(%x : $@convention(objc_method) (P) -> ()):
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   return undef: $()
 }
 
@@ -342,8 +352,10 @@ bb0:
   // CHECK: checked_cast_br undef : $C to $C, bb1, bb2
   checked_cast_br undef : $C to $C, bb1, bb2
 bb1(%x : $C):
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   return undef : $()
 }
 
@@ -352,8 +364,10 @@ bb0:
   // CHECK: checked_cast_addr_br take_always C in undef : $*C to C in undef : $*C, bb1, bb2
   checked_cast_addr_br take_always C in undef : $*C to C in undef : $*C, bb1, bb2
 bb1:
-  br bb2
+  br bb3
 bb2:
+  br bb3
+bb3:
   return undef : $()
 }
 

test/SILGen/errors.swift

@@ -539,6 +539,7 @@ func supportFirstStructure<B: Buildable>(_ b: inout B) throws {
 // CHECK: [[BB_ERROR]]([[ERROR:%.*]] : @owned $Error):
 // CHECK: abort_apply [[TOKEN]]
 // CHECK: throw [[ERROR]]
+
 // CHECK: } // end sil function '$S6errors21supportFirstStructure{{.*}}F'
 
 func supportStructure<B: Buildable>(_ b: inout B, name: String) throws {
@@ -565,6 +566,7 @@ func supportStructure<B: Buildable>(_ b: inout B, name: String) throws {
 // CHECK:   end_borrow [[BORROWED_INDEX_COPY]] from [[INDEX_COPY]]
 // CHECK:   destroy_value [[INDEX_COPY]] : $String
 // CHECK:   throw [[ERROR]]
+
 // CHECK: } // end sil function '$S6errors16supportStructure{{.*}}F'
 
 struct Pylon {

test/SILOptimizer/cse_objc.sil

@@ -127,28 +127,34 @@ bb0(%0 : $Walkable):
 // CHECK: open_existential_ref %0 : $AnyObject to $[[OPENED:@opened\(.*\) AnyObject]]
 // CHECK: bb1(%{{[0-9]*}} : $@convention(objc_method) ([[OPENED]]) -> @autoreleased Optional<NSString>):
 // CHECK-NOT: open_existential_ref
-// CHECK: bb2(%{{[0-9]*}} : $@convention(objc_method) ([[OPENED]]) -> @autoreleased Optional<NSString>):
+// CHECK: bb3(%{{[0-9]*}} : $@convention(objc_method) ([[OPENED]]) -> @autoreleased Optional<NSString>):
 // CHECK-NOT: open_existential_ref
 // CHECK: return
 sil @cse_open_existential_in_bbarg : $@convention(thin) (@owned AnyObject) -> () {
 bb0(%0 : $AnyObject):
   %1 = open_existential_ref %0 : $AnyObject to $@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject
-  dynamic_method_br %1 : $@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject, #NSProxy.description!getter.1.foreign, bb1, bb3
+  dynamic_method_br %1 : $@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject, #NSProxy.description!getter.1.foreign, bb1, bb2
 
 bb1(%3 : $@convention(objc_method) (@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject) -> @autoreleased Optional<NSString>):
   strong_retain %1 : $@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject
   %5 = partial_apply %3(%1) : $@convention(objc_method) (@opened("CCEAC0E2-4BB2-11E6-86F8-B8E856428C60") AnyObject) -> @autoreleased Optional<NSString>
   %6 = apply %5() : $@callee_owned () -> @owned Optional<NSString>
   %7 = open_existential_ref %0 : $AnyObject to $@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject
-  dynamic_method_br %7 : $@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject, #NSProxy.description!getter.1.foreign, bb2, bb3
+  dynamic_method_br %7 : $@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject, #NSProxy.description!getter.1.foreign, bb3, bb4
 
-bb2(%9 : $@convention(objc_method) (@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject) -> @autoreleased Optional<NSString>):
+bb2:
+  br bb5
+
+bb3(%9 : $@convention(objc_method) (@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject) -> @autoreleased Optional<NSString>):
   strong_retain %7 : $@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject
   %10 = partial_apply %9(%7) : $@convention(objc_method) (@opened("CCEDAF1E-4BB2-11E6-86F8-B8E856428C60") AnyObject) -> @autoreleased Optional<NSString>
   %11 = apply %10() : $@callee_owned () -> @owned Optional<NSString>
-  br bb3
+  br bb5
+
+bb4:
+  br bb5
 
-bb3:
+bb5:
   strong_release %0 : $AnyObject
   %13 = tuple ()
   return %13 : $()

test/SILOptimizer/diagnose_unreachable.sil

@@ -395,39 +395,53 @@ sil_scope 1 { loc "f.swift":2:1 parent 0 }
 // CHECK: sil_scope [[S:[0-9]+]] { loc "f.swift":2:1 parent [[F]] }
 // CHECK-LABEL:sil @try_apply_2
 // CHECK:    bb3({{.*}}$Never):
+// CHECK-NEXT: {{ unreachable }}, scope [[F]]
+// CHECK:    bb5({{.*}}$Never):
 // CHECK-NEXT: {{ unreachable }}, scope [[S]]
-// CHECK:    bb4(
+// CHECK:    bb7({{.*}} : $Error):
 // CHECK-NEXT: throw
-
+// CHECK-LABEL: } // end sil function 'try_apply_2'
 sil @try_apply_2 : $@convention(thin) (Builtin.Int1) -> @error Error {
 bb0(%0 : $Builtin.Int1):
   %1 = function_ref @throwing_noreturn : $@convention(thin) () -> (Never, @error Error)
   cond_br %0, bb1, bb2
 bb1:
   try_apply %1() : $@convention(thin) () -> (Never, @error Error), normal bb3, error bb4, scope 0
 bb2:
-  try_apply %1() : $@convention(thin) () -> (Never, @error Error), normal bb3, error bb4, scope 1
+  try_apply %1() : $@convention(thin) () -> (Never, @error Error), normal bb5, error bb6, scope 1
 bb3(%2 : $Never):
-  %3 = tuple ()
-  return %3 : $()
+  br bb7
 bb4(%4 : $Error):
-  throw %4 : $Error
+  br bb8(%4 : $Error)
+bb5(%5 : $Never):
+  br bb7
+bb6(%7 : $Error):
+  br bb8(%7 : $Error)
+bb7:
+  %8 = tuple ()
+  return %8 : $()
+bb8(%9 : $Error):
+  throw %9 : $Error
 }
 
-// This should arguably be ill-formed.
 // CHECK-LABEL:sil @try_apply_3
 // CHECK:      br bb1
-// CHECK:    bb1(
+// CHECK:    bb1:
 // CHECK-NOT:  {{ unreachable}}
 // CHECK:      try_apply
-// CHECK:    bb2(
+// CHECK:    bb3({{.*}} : $Error):
 // CHECK-NEXT: throw
 sil @try_apply_3 : $@convention(thin) (Never) -> @error Error {
 bb0(%0 : $Never):
   br bb1(%0 : $Never)
+
 bb1(%1 : $Never):
   %2 = function_ref @throwing_noreturn : $@convention(thin) () -> (Never, @error Error)
-  try_apply %2() : $@convention(thin) () -> (Never, @error Error), normal bb1, error bb2
-bb2(%3 : $Error):
-  throw %3 : $Error
+  try_apply %2() : $@convention(thin) () -> (Never, @error Error), normal bb2, error bb3
+
+bb2(%3 : $Never):
+  br bb1(%3 : $Never)
+
+bb3(%4 : $Error):
+  throw %4 : $Error
 }