[DynamicCast] Rely on runtime when casts can't be optimized

The Swift compiler renders `as?` operations in Swift into variations of the
`checked_cast_br` instructions in SIL.  Subsequent optimization passes may alter
or eliminate these instructions.  Any remaining instructions after optimization
are translated by IRGen into runtime calls.

At least, that's the theory.  Unfortunately, the current IRGen logic does not
recognize all of the casting instruction variants and renders one particular
unrecognized variant as a simple nil load.  Specifically, this occurs for
optimized casts of metatypes to AnyObject:
```
  let a = Int.self
  let b = a as? AnyObject
  // b should not be nil here
```

This PR changes this case in IRGen to instead generate a call to
`swift_dynamicCast`, deferring this case to the runtime.

Future:  Someday, the compiler should be taught to correctly optimize
this cast away.

Author: tbkka

lib/IRGen/GenCast.cpp

@@ -478,13 +478,8 @@ llvm::Value *irgen::emitMetatypeToAnyObjectDowncast(IRGenFunction &IGF,
                                                     CheckedCastMode mode) {
   // If ObjC interop is enabled, casting a metatype to AnyObject succeeds
   // if the metatype is for a class.
-
-  auto triviallyFail = [&]() -> llvm::Value* {
-    return llvm::ConstantPointerNull::get(IGF.IGM.ObjCPtrTy);
-  };
-  
   if (!IGF.IGM.ObjCInterop)
-    return triviallyFail();
+    return nullptr;
 
   switch (type->getRepresentation()) {
   case MetatypeRepresentation::ObjC:
@@ -496,7 +491,7 @@ llvm::Value *irgen::emitMetatypeToAnyObjectDowncast(IRGenFunction &IGF,
     // TODO: Final class metatypes could in principle be thin.
     assert(!type.getInstanceType()->mayHaveSuperclass()
            && "classes should not have thin metatypes (yet)");
-    return triviallyFail();
+    return nullptr;
 
   case MetatypeRepresentation::Thick: {
     auto instanceTy = type.getInstanceType();
@@ -508,10 +503,10 @@ llvm::Value *irgen::emitMetatypeToAnyObjectDowncast(IRGenFunction &IGF,
       return IGF.Builder.CreateBitCast(heapMetadata, IGF.IGM.ObjCPtrTy);
     }
 
-    // Is the type obviously not a class?
-    if (!isa<ArchetypeType>(instanceTy)
-        && !isa<ExistentialMetatypeType>(type))
-      return triviallyFail();
+    // If it's not a class, we can't handle it here
+    if (!isa<ArchetypeType>(instanceTy) && !isa<ExistentialMetatypeType>(type)) {
+      return nullptr;
+    }
 
     // Ask the runtime whether this is class metadata.
     llvm::Constant *castFn;
@@ -966,10 +961,43 @@ void irgen::emitScalarCheckedCast(IRGenFunction &IGF,
     // Otherwise, this is a metatype-to-object cast.
     assert(targetLoweredType.isAnyClassReferenceType());
 
-    // Convert the metatype value to AnyObject.
+    // Can we convert the metatype value to AnyObject using Obj-C machinery?
     llvm::Value *object =
       emitMetatypeToAnyObjectDowncast(IGF, metatypeVal, sourceMetatype, mode);
 
+    if (object == nullptr) {
+      // Obj-C cast routine failed, use swift_dynamicCast instead
+
+      if (sourceMetatype->getRepresentation() == MetatypeRepresentation::Thin
+          || metatypeVal == nullptr) {
+        // Earlier stages *should* never generate a checked cast with a thin metatype argument.
+        // TODO: Move this assertion up to apply to all checked cast operations.
+        // In assert builds, enforce this by failing here:
+        assert(false && "Invalid SIL: General checked_cast_br cannot have thin argument");
+        // In non-assert builds, stay compatible with previous behavior by emitting a null load.
+        object = llvm::ConstantPointerNull::get(IGF.IGM.ObjCPtrTy);
+      } else {
+        Address src = IGF.createAlloca(metatypeVal->getType(),
+                                       IGF.IGM.getPointerAlignment(),
+                                       "castSrc");
+        IGF.Builder.CreateStore(metatypeVal, src);
+        llvm::PointerType *destPtrType = IGF.IGM.getStoragePointerType(targetLoweredType);
+        Address dest = IGF.createAlloca(destPtrType,
+                                        IGF.IGM.getPointerAlignment(),
+                                        "castDest");
+        IGF.Builder.CreateStore(llvm::ConstantPointerNull::get(destPtrType), dest);
+        llvm::Value *success = emitCheckedCast(IGF,
+                                               src, sourceFormalType,
+                                               dest, targetFormalType,
+                                               CastConsumptionKind::TakeAlways,
+                                               mode);
+        llvm::Value *successResult = IGF.Builder.CreateLoad(dest);
+        llvm::Value *failureResult = llvm::ConstantPointerNull::get(destPtrType);
+        llvm::Value *result = IGF.Builder.CreateSelect(success, successResult, failureResult);
+        object = std::move(result);
+      }
+    }
+
     sourceFormalType = IGF.IGM.Context.getAnyObjectType();
     sourceLoweredType = SILType::getPrimitiveObjectType(sourceFormalType);
 

test/IRGen/casts.sil

@@ -298,9 +298,9 @@ bb3(%9 : $Optional<CP>):
 // CHECK-LABEL: define{{( dllexport)?}}{{( protected)?}} swiftcc void @checked_metatype_to_object_casts
 sil @checked_metatype_to_object_casts : $@convention(thin) <T> (@thick Any.Type) -> () {
 entry(%e : $@thick Any.Type):
-  %a = metatype $@thin NotClass.Type
-  // CHECK: br i1 false
-  checked_cast_br %a : $@thin NotClass.Type to AnyObject, a_yea, a_nay
+  %a = metatype $@thick NotClass.Type
+  // CHECK: call i1 @swift_dynamicCast({{.*}})
+  checked_cast_br %a : $@thick NotClass.Type to AnyObject, a_yea, a_nay
 a_yea(%1 : $AnyObject):
   %b = metatype $@thick A.Type
   // CHECK: bitcast %swift.type* {{%.*}} to %objc_object*

validation-test/Casting/BoxingCasts.swift.gyb

@@ -16,16 +16,27 @@
 // RUN: %empty-directory(%t)
 //
 // RUN: %gyb %s -o %t/BoxingCasts.swift
-// RUN: %line-directive %t/BoxingCasts.swift -- %target-build-swift -g -module-name a -swift-version 5 -Onone %t/BoxingCasts.swift -o %t/a.swift5.Onone.out
+//
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-build-swift -g -module-name a -Onone -swift-version 5 %t/BoxingCasts.swift -o %t/a.swift5.Onone.out
 // RUN: %target-codesign %t/a.swift5.Onone.out
 // RUN: %line-directive %t/BoxingCasts.swift -- %target-run %t/a.swift5.Onone.out
 //
-// Note: The RUN directives above override the default test optimizations.
-// This test is deliberately run non-optimized in order to verify the
-// behavior of runtime methods that may not be called for optimized casts.
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-build-swift -g -O -module-name a -O -swift-version 5 %t/BoxingCasts.swift -o %t/a.swift5.O.out
+// RUN: %target-codesign %t/a.swift5.O.out
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-run %t/a.swift5.O.out
+//
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-build-swift -g -module-name a -Onone -swift-version 4 %t/BoxingCasts.swift -o %t/a.swift4.Onone.out
+// RUN: %target-codesign %t/a.swift4.Onone.out
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-run %t/a.swift4.Onone.out
 //
-// XXX FIXME XXX TODO XXX  _Also_ build this with optimizations in order to
-// verify compiler behaviors.
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-build-swift -g -O -module-name a -O -swift-version 4 %t/BoxingCasts.swift -o %t/a.swift4.O.out
+// RUN: %target-codesign %t/a.swift4.O.out
+// RUN: %line-directive %t/BoxingCasts.swift -- %target-run %t/a.swift4.O.out
+//
+// Note: The RUN directives above override the default test optimizations.
+// This test is deliberately run both ways:
+//  * optimized to verify compiler cast optimizations, and
+//  * non-optimized to verify the runtime methods used for non-optimized casts.
 //
 // REQUIRES: executable_test