AST: Types nested inside a subclass shadow types in the superclass with same name

The existing rules sometimes behaved this way, because validateDecl()
bails out silently without error on circularity.

Let's make this explicit instead, and get it working in all cases.

Fixes <rdar://https://bugs.swift.org/browse/SR-3492> and
<rdar://problem/45889192>.

Author: slavapestov

lib/AST/NameLookup.cpp

@@ -193,6 +193,8 @@ static void recordShadowedDeclsAfterSignatureMatch(
   for (unsigned firstIdx : indices(decls)) {
     auto firstDecl = decls[firstIdx];
     auto firstModule = firstDecl->getModuleContext();
+    bool firstTopLevel = firstDecl->getDeclContext()->isModuleScopeContext();
+
     auto name = firstDecl->getBaseName();
 
     auto isShadowed = [&](ArrayRef<ModuleDecl::AccessPathTy> paths) {
@@ -219,15 +221,37 @@ static void recordShadowedDeclsAfterSignatureMatch(
       // Determine whether one module takes precedence over another.
       auto secondDecl = decls[secondIdx];
       auto secondModule = secondDecl->getModuleContext();
+      bool secondTopLevel = secondDecl->getDeclContext()->isModuleScopeContext();
+
+      // For member types, we skip most of the below rules. Instead, we allow
+      // member types defined in a subclass to shadow member types defined in
+      // a superclass.
+      if (isa<TypeDecl>(firstDecl) &&
+          isa<TypeDecl>(secondDecl) &&
+          !firstTopLevel &&
+          !secondTopLevel) {
+        auto *firstClass = firstDecl->getDeclContext()->getSelfClassDecl();
+        auto *secondClass = secondDecl->getDeclContext()->getSelfClassDecl();
+        if (firstClass && secondClass && firstClass != secondClass) {
+          if (firstClass->isSuperclassOf(secondClass)) {
+            shadowed.insert(firstDecl);
+            continue;
+          } else if (secondClass->isSuperclassOf(firstClass)) {
+            shadowed.insert(secondDecl);
+            continue;
+          }
+        }
+
+        continue;
+      }
 
       // Top-level type declarations in a module shadow other declarations
       // visible through the module's imports.
       //
       // [Backward compatibility] Note that members of types have the same
       // shadowing check, but we do it after dropping unavailable members.
       if (firstModule != secondModule &&
-          firstDecl->getDeclContext()->isModuleScopeContext() &&
-          secondDecl->getDeclContext()->isModuleScopeContext()) {
+          firstTopLevel && secondTopLevel) {
         auto firstPaths = imports.getAllAccessPathsNotShadowedBy(
           firstModule, secondModule, dc);
         auto secondPaths = imports.getAllAccessPathsNotShadowedBy(
@@ -304,8 +328,7 @@ static void recordShadowedDeclsAfterSignatureMatch(
       // shadowing check is performed after unavailable candidates have
       // already been dropped.
       if (firstModule != secondModule &&
-          !firstDecl->getDeclContext()->isModuleScopeContext() &&
-          !secondDecl->getDeclContext()->isModuleScopeContext()) {
+          !firstTopLevel && !secondTopLevel) {
         auto firstPaths = imports.getAllAccessPathsNotShadowedBy(
           firstModule, secondModule, dc);
         auto secondPaths = imports.getAllAccessPathsNotShadowedBy(
@@ -478,9 +501,6 @@ static void recordShadowedDecls(ArrayRef<ValueDecl *> decls,
       // type. This is layering a partial fix upon a total hack.
       if (auto asd = dyn_cast<AbstractStorageDecl>(decl))
         signature = asd->getOverloadSignatureType();
-    } else if (decl->getDeclContext()->isTypeContext()) {
-      // Do not apply shadowing rules for member types.
-      continue;
     }
 
     // Record this declaration based on its signature.

test/NameBinding/member_type_shadowing.swift

@@ -12,4 +12,20 @@ extension RootClass: P {
   typealias Member = SubClass.Member
 }
 
+class A {
+  enum Reusable {
+    case option1
+    case option2
+  }
+}
+
+class B: A {
+  enum Reusable {
+    case option1 // expected-note {{'option1' declared here}}
+  }
 
+  func process() {
+    _ = B.Reusable.option1
+    _ = B.Reusable.option2 // expected-error {{type 'B.Reusable' has no member 'option2'; did you mean 'option1'?}}
+  }
+}