'open' classes within 'public' structs should be considered open. (swiftlang#11997)

Without this, our hack to give all open class vtable members public
linkage at the LLVM level wasn't kicking in, and subclasses from
other modules were getting link errors.

rdar://problem/32885384

Author: jrose-apple

lib/AST/Decl.cpp

@@ -1973,17 +1973,31 @@ Accessibility ValueDecl::getEffectiveAccess() const {
     break;
   }
 
+  auto restrictToEnclosing =
+      [this](Accessibility effectiveAccess,
+             Accessibility enclosingAccess) -> Accessibility {
+    if (effectiveAccess == Accessibility::Open &&
+        enclosingAccess == Accessibility::Public &&
+        isa<NominalTypeDecl>(this)) {
+      // Special case: an open class may be contained in a public
+      // class/struct/enum. Leave effectiveAccess as is.
+      return effectiveAccess;
+    }
+    return std::min(effectiveAccess, enclosingAccess);
+  };
+
   if (auto enclosingNominal = dyn_cast<NominalTypeDecl>(getDeclContext())) {
-    effectiveAccess = std::min(effectiveAccess,
-                               enclosingNominal->getEffectiveAccess());
+    effectiveAccess =
+        restrictToEnclosing(effectiveAccess,
+                            enclosingNominal->getEffectiveAccess());
 
   } else if (auto enclosingExt = dyn_cast<ExtensionDecl>(getDeclContext())) {
     // Just check the base type. If it's a constrained extension, Sema should
     // have already enforced access more strictly.
     if (auto extendedTy = enclosingExt->getExtendedType()) {
       if (auto nominal = extendedTy->getAnyNominal()) {
-        effectiveAccess = std::min(effectiveAccess,
-                                   nominal->getEffectiveAccess());
+        effectiveAccess =
+            restrictToEnclosing(effectiveAccess, nominal->getEffectiveAccess());
       }
     }
 

test/IRGen/Inputs/vtable_symbol_linkage_base.swift

@@ -10,3 +10,18 @@ open class Base {
   private var privateVar: Int = 29
   internal var internalVar: Int = 30
 }
+
+
+public struct Namespace {
+  open class Nested {
+    public init() {}
+    var storedProp: Int?
+  }
+}
+
+extension Namespace {
+  open class ExtNested {
+    public init() {}
+    var storedProp: Int?
+  }
+}

test/IRGen/vtable_symbol_linkage.swift

@@ -9,4 +9,5 @@ import BaseModule
 public class Derived : Base {
 }
 
-
+public class DerivedNested : Namespace.Nested {}
+public class DerivedExtNested : Namespace.ExtNested {}