SIL: Fix SILType substitution bug with DynamicSelfType vs MetatypeType

Previously, SIL type lowering would assume a MetatypeType was
completely lowered if it had a representation, but this is not
quite right; after substitution, we can have a MetatypeType
whose instance type is a DynamicSelfType. Strip these away more
eagerly, since they show up in SILType::subst(), where we first
substitute AST-level generic parameters, and then lower the
result to get the final SIL type.

Fixes <https://bugs.swift.org/browse/SR-2733>.

Author: slavapestov

lib/SIL/SILFunctionType.cpp

@@ -2209,6 +2209,28 @@ namespace {
                                            substObjectType));
     }
 
+    /// Metatypes get DynamicSelfType stripped off the instance type.
+    CanType visitMetatypeType(CanMetatypeType origType) {
+      CanType origInstanceType = origType.getInstanceType();
+      CanType substInstanceType = origInstanceType.subst(
+            Subst, Conformances, None)->getCanonicalType();
+
+      // If the substitution didn't change anything, we know that the
+      // original type was a lowered type, so we're good.
+      if (origInstanceType == substInstanceType) {
+        return origType;
+      }
+
+      // If this is a DynamicSelf metatype, turn it into a metatype of the
+      // underlying self type.
+      if (auto dynamicSelf = dyn_cast<DynamicSelfType>(substInstanceType)) {
+        substInstanceType = dynamicSelf.getSelfType();
+      }
+
+      return CanMetatypeType::get(substInstanceType,
+                                  origType->getRepresentation());
+    }
+
     /// Any other type is would be a valid type in the AST.  Just
     /// apply the substitution on the AST level and then lower that.
     CanType visitType(CanType origType) {

lib/SIL/TypeLowering.cpp

@@ -1235,30 +1235,6 @@ void TypeConverter::insert(TypeKey k, const TypeLowering *tl) {
   Types[k.getCachingKey()] = tl;
 }
 
-#ifndef NDEBUG
-/// Is this type a lowered type?
-static bool isLoweredType(CanType type) {
-  if (isa<LValueType>(type) || isa<InOutType>(type))
-    return false;
-  if (isa<AnyFunctionType>(type))
-    return false;
-  if (auto tuple = dyn_cast<TupleType>(type)) {
-    for (auto elt : tuple.getElementTypes())
-      if (!isLoweredType(elt))
-        return false;
-    return true;
-  }
-  OptionalTypeKind optKind;
-  if (auto objectType = type.getAnyOptionalObjectType(optKind)) {
-    return (optKind == OTK_Optional && isLoweredType(objectType));
-  }
-  if (auto meta = dyn_cast<AnyMetatypeType>(type)) {
-    return meta->hasRepresentation();
-  }
-  return true;
-}
-#endif
-
 /// Lower each of the elements of the substituted type according to
 /// the abstraction pattern of the given original type.
 static CanTupleType getLoweredTupleType(TypeConverter &tc,
@@ -1525,7 +1501,7 @@ const TypeLowering &TypeConverter::getTypeLowering(SILType type) {
 const TypeLowering &
 TypeConverter::getTypeLoweringForLoweredType(TypeKey key) {
   auto type = key.SubstType;
-  assert(isLoweredType(type) && "type is not lowered!");
+  assert(type->isLegalSILType() && "type is not lowered!");
   (void)type;
 
   // Re-using uncurry level 0 is reasonable because our uncurrying
@@ -1542,7 +1518,7 @@ TypeConverter::getTypeLoweringForLoweredType(TypeKey key) {
 const TypeLowering &
 TypeConverter::getTypeLoweringForUncachedLoweredType(TypeKey key) {
   assert(!find(key) && "re-entrant or already cached");
-  assert(isLoweredType(key.SubstType) && "type is not already lowered");
+  assert(key.SubstType->isLegalSILType() && "type is not already lowered");
 
 #ifndef NDEBUG
   // Catch reentrancy bugs.

test/SILGen/dynamic_self.swift

@@ -207,6 +207,20 @@ class Z {
 
 }
 
+// Make sure we erase dynamic Self in a metatype instance type when
+// performing SIL type substitution.
+
+func makeInstance<T: FactoryBase>(_ cls: T.Type) -> T {
+    return cls.init()
+}
+
+class FactoryBase {
+    required init() { }
+    func before() -> Self {
+        return makeInstance(type(of: self))
+    }
+}
+
 // CHECK-LABEL: sil_witness_table hidden X: P module dynamic_self {
 // CHECK: method #P.f!1: @_TTWC12dynamic_self1XS_1PS_FS1_1f