Always desugar the base type of an extension when serializing. (#6336)

This "fixes" two issues:

- The name of a non-public typealias would leak into the public
  interface if the extension had any public members.

- A common pattern of defining a platform-specific typealias for an
  imported class and then extending that type would lead to
  circularity when trying to deserialize the typealias. We /shouldn't/
  be loading the extension at that point, but fixing that would be
  much harder.

The "right" answer is to (a) check that the typealias is public if the
extension has any public members, and (b) somehow ensure there is no
circularity issue (either by not importing the extension as a result
of importing the typealias, or by the extension being able to set its
sugared base type later).

rdar://problem/29694978

Author: jrose-apple

lib/Serialization/Serialization.cpp

@@ -2242,9 +2242,20 @@ void Serializer::writeDecl(const Decl *D) {
     auto contextID = addDeclContextRef(extension->getDeclContext());
     Type baseTy = extension->getExtendedType();
 
+    // FIXME: Use the canonical type here in order to minimize circularity
+    // issues at deserialization time. A known problematic case here is
+    // "extension of typealias Foo"; "typealias Foo = SomeKit.Bar"; and then
+    // trying to import Bar accidentally asking for all of its extensions
+    // (perhaps because we're searching for a conformance).
+    //
+    // We could limit this to only the problematic cases, but it seems like a
+    // simpler user model to just always desugar extension types.
+    baseTy = baseTy->getCanonicalType();
+
     // Make sure the base type has registered itself as a provider of generic
     // parameters.
-    (void)addDeclRef(baseTy->getAnyNominal());
+    auto baseNominal = baseTy->getAnyNominal();
+    (void)addDeclRef(baseNominal);
 
     auto conformances = extension->getLocalConformances(
                           ConformanceLookupKind::All,
@@ -2265,7 +2276,7 @@ void Serializer::writeDecl(const Decl *D) {
                                 inheritedTypes);
 
     bool isClassExtension = false;
-    if (auto baseNominal = baseTy->getAnyNominal()) {
+    if (baseNominal) {
       isClassExtension = isa<ClassDecl>(baseNominal) ||
                          isa<ProtocolDecl>(baseNominal);
     }

test/Serialization/extension-of-typealias.swift

@@ -0,0 +1,47 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %target-swift-frontend -emit-module -module-name Library -o %t -D LIBRARY %s
+// RUN: %target-swift-ide-test -print-module -module-to-print=Library -I %t -source-filename=%s | %FileCheck %s
+// RUN: %target-swift-frontend -typecheck -I %t %s -verify
+
+// Check that base types of extensions are desugared. This isn't necessarily
+// the behavior we want long-term, but it's the behavior we need right now.
+
+#if LIBRARY
+
+public typealias Zahl = Int
+
+// CHECK-LABEL: extension Int {
+extension Zahl {
+  // CHECK-NEXT: addedMember()
+  public func addedMember() {}
+} // CHECK-NEXT: {{^}$}}
+
+public typealias List<T> = Array<T>
+
+// CHECK-LABEL: extension Array {
+extension List {
+  // CHECK-NEXT: addedMember()
+  public func addedMember() {}
+} // CHECK-NEXT: {{^}$}}
+
+// CHECK-LABEL: extension Array where Element == Int {
+extension List where Element == Int {
+  // CHECK-NEXT: addedMemberInt()
+  public func addedMemberInt() {}
+} // CHECK-NEXT: {{^}$}}
+
+// CHECK: typealias List
+// CHECK: typealias Zahl
+
+#else
+
+import Library
+
+func test(x: Int) {
+  x.addedMember()
+  [x].addedMember()
+  [x].addedMemberInt()
+  ([] as [Bool]).addedMemberInt() // expected-error {{'[Bool]' is not convertible to 'Array<Int>'}}
+}
+
+#endif

validation-test/Serialization/Inputs/rdar29694978.h

@@ -0,0 +1,13 @@
+@import Foundation;
+
+@protocol BaseProto
+@end
+
+@protocol SubProto <BaseProto>
+@end
+
+@interface NonGenericType: NSObject <SubProto>
+@end
+
+@interface GenericType<Element>: NSObject <SubProto>
+@end

validation-test/Serialization/rdar29694978.swift

@@ -0,0 +1,28 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %target-build-swift %s -import-objc-header %S/Inputs/rdar29694978.h -emit-module -o %t/Library.swiftmodule
+// RUN: %target-swift-ide-test -print-module -module-to-print=Library -source-filename=x -I %S/Inputs/ -I %t | %FileCheck %s
+
+// REQUIRES: objc_interop
+
+// This would be wonderful to fold into a larger test, but I'm worried that
+// trying to do so would perturb the original conditions that cause it to
+// crash: deserializing a typealias (1) to an imported Objective-C type (2)
+// that has a protocol (3) that refines another protocol (4) which causes us
+// to load all extensions, including one defined in this file (5) in terms of
+// the original typealias (1).
+//
+// It's probably best to just leave this test about that.
+
+import Foundation
+
+// CHECK-DAG: typealias MyNonGenericType = NonGenericType
+typealias MyNonGenericType = NonGenericType
+// CHECK-DAG: extension NonGenericType
+extension MyNonGenericType {}
+
+// CHECK-DAG: typealias MyGenericType<T> = GenericType<T>
+typealias MyGenericType<T: NSObject> = GenericType<T>
+// CHECK-DAG: extension GenericType where Element : AnyObject
+extension MyGenericType {}
+// CHECK-DAG: extension GenericType where Element == NSObject
+extension MyGenericType where Element == NSObject {}