[SIL]/[SILOpt] Thunks and specializations shouldn't be connected to their parent class.

The "subclass scope" is meant to represent a connection to a vtable (and how
public something needs to be), for things that end up in class
vtables. Specializations and thunks are mostly internal implementation details
and do not end up there, so subclass scope is not applicable to them. This stops
the thunks and specializations being incorrectly public.

(Note, there are some thunks that _are_ public facing: if a function has its
signature optimized, the original entry point becomes a thunk, and this entry
point is what ends up in vtables etc., so needs to remain around, which means
keeping the same hacks for `private` members of an `open` class.)

Fixes rdar://problem/40738913.

Author: huonw

include/swift/SIL/SILFunction.h

@@ -229,6 +229,33 @@ class SILFunction
   /// serialization.
   bool WasDeserializedCanonical = false;
 
+  static void
+  validateSubclassScope(SubclassScope scope, IsThunk_t isThunk,
+                        const GenericSpecializationInformation *genericInfo) {
+#ifndef NDEBUG
+    // The _original_ function for a method can turn into a thunk through
+    // signature optimization, meaning it needs to retain its subclassScope, but
+    // other thunks and specializations are implementation details and so
+    // shouldn't be connected to their parent class.
+    bool thunkCanHaveSubclassScope;
+    switch (isThunk) {
+    case IsNotThunk:
+    case IsSignatureOptimizedThunk:
+      thunkCanHaveSubclassScope = true;
+      break;
+    case IsThunk:
+    case IsReabstractionThunk:
+      thunkCanHaveSubclassScope = false;
+      break;
+    }
+    auto allowsInterestingScopes = thunkCanHaveSubclassScope && !genericInfo;
+    assert(
+        allowsInterestingScopes ||
+        scope == SubclassScope::NotApplicable &&
+            "SubclassScope on specialization or non-signature-optimized thunk");
+#endif
+  }
+
   SILFunction(SILModule &module, SILLinkage linkage, StringRef mangledName,
               CanSILFunctionType loweredType, GenericEnvironment *genericEnv,
               Optional<SILLocation> loc, IsBare_t isBareSILFunction,
@@ -578,13 +605,20 @@ class SILFunction
 
   /// Get this function's thunk attribute.
   IsThunk_t isThunk() const { return IsThunk_t(Thunk); }
-  void setThunk(IsThunk_t isThunk) { Thunk = isThunk; }
+  void setThunk(IsThunk_t isThunk) {
+    validateSubclassScope(getClassSubclassScope(), isThunk, SpecializationInfo);
+    Thunk = isThunk;
+  }
 
   /// Get the class visibility (relevant for class methods).
   SubclassScope getClassSubclassScope() const {
     return SubclassScope(ClassSubclassScope);
   }
-    
+  void setClassSubclassScope(SubclassScope scope) {
+    validateSubclassScope(scope, isThunk(), SpecializationInfo);
+    ClassSubclassScope = static_cast<unsigned>(scope);
+  }
+
   /// Get this function's noinline attribute.
   Inline_t getInlineStrategy() const { return Inline_t(InlineStrategy); }
   void setInlineStrategy(Inline_t inStr) { InlineStrategy = inStr; }
@@ -660,6 +694,7 @@ class SILFunction
 
   void setSpecializationInfo(const GenericSpecializationInformation *Info) {
     assert(!isSpecialization());
+    validateSubclassScope(getClassSubclassScope(), isThunk(), Info);
     SpecializationInfo = Info;
   }
 

lib/ParseSIL/ParseSIL.cpp

@@ -912,6 +912,8 @@ static bool parseDeclSILOptional(bool *isTransparent,
       *isCanonical = true;
     else if (isThunk && SP.P.Tok.getText() == "thunk")
       *isThunk = IsThunk;
+    else if (isThunk && SP.P.Tok.getText() == "signature_optimized_thunk")
+      *isThunk = IsSignatureOptimizedThunk;
     else if (isThunk && SP.P.Tok.getText() == "reabstraction_thunk")
       *isThunk = IsReabstractionThunk;
     else if (isGlobalInit && SP.P.Tok.getText() == "global_init")

lib/SIL/SILDeclRef.cpp

@@ -771,6 +771,10 @@ SubclassScope SILDeclRef::getSubclassScope() const {
   if (context->isExtensionContext())
     return SubclassScope::NotApplicable;
 
+  // Various forms of thunks don't either.
+  if (isThunk() || isForeign)
+    return SubclassScope::NotApplicable;
+
   auto *classType = context->getAsClassOrClassExtensionContext();
   if (!classType || classType->isFinal())
     return SubclassScope::NotApplicable;

lib/SIL/SILFunction.cpp

@@ -102,6 +102,8 @@ SILFunction::SILFunction(SILModule &Module, SILLinkage Linkage, StringRef Name,
       HasCReferences(false), IsWeakLinked(false),
       OptMode(OptimizationMode::NotSet), EffectsKindAttr(E),
       EntryCount(entryCount) {
+  validateSubclassScope(classSubclassScope, isThunk, nullptr);
+
   if (InsertBefore)
     Module.functions.insert(SILModule::iterator(InsertBefore), this);
   else

lib/SILOptimizer/FunctionSignatureTransforms/FunctionSignatureOpts.cpp

@@ -506,6 +506,10 @@ void FunctionSignatureTransform::createFunctionSignatureOptimizedFunction() {
     NewF->setUnqualifiedOwnership();
   }
 
+  if (F->isSpecialization()) {
+    NewF->setSpecializationInfo(F->getSpecializationInfo());
+  }
+
   // Then we transfer the body of F to NewF.
   NewF->spliceBody(F);
 

lib/SILOptimizer/IPO/ClosureSpecializer.cpp

@@ -635,11 +635,15 @@ ClosureSpecCloner::initCloned(const CallSiteDescriptor &CallSiteDesc,
       // and not the original linkage.
       // Otherwise the new function could have an external linkage (in case the
       // original function was de-serialized) and would not be code-gen'd.
+      // It's also important to disconnect this specialized function from any
+      // classes (the classSubclassScope), because that may incorrectly
+      // influence the linkage.
       getSpecializedLinkage(ClosureUser, ClosureUser->getLinkage()), ClonedName,
       ClonedTy, ClosureUser->getGenericEnvironment(),
       ClosureUser->getLocation(), IsBare, ClosureUser->isTransparent(),
       CallSiteDesc.isSerialized(), ClosureUser->getEntryCount(),
-      ClosureUser->isThunk(), ClosureUser->getClassSubclassScope(),
+      ClosureUser->isThunk(),
+      /*classSubclassScope=*/SubclassScope::NotApplicable,
       ClosureUser->getInlineStrategy(), ClosureUser->getEffectsKind(),
       ClosureUser, ClosureUser->getDebugScope());
   if (!ClosureUser->hasQualifiedOwnership()) {

lib/SILOptimizer/Utils/Generics.cpp

@@ -1840,6 +1840,7 @@ SILFunction *GenericFuncSpecializer::tryCreateSpecialization() {
   auto *Caller = ReInfo.getApply() ? ReInfo.getApply().getFunction() : nullptr;
   SubstitutionMap Subs = Caller ? ReInfo.getApply().getSubstitutionMap()
                                 : ReInfo.getClonerParamSubstitutionMap();
+  SpecializedF->setClassSubclassScope(SubclassScope::NotApplicable);
   SpecializedF->setSpecializationInfo(
       GenericSpecializationInformation::create(Caller, GenericFunc, Subs));
   return SpecializedF;

test/TBD/class_objc.swift.gyb

@@ -1,9 +1,9 @@
 // RUN: %empty-directory(%t)
 // RUN: %gyb %s > %t/main.swift
 
-// RUN: %target-swift-frontend -emit-ir -o- -parse-as-library -module-name test -import-objc-header %S/Inputs/objc_class_header.h -validate-tbd-against-ir=all %t/main.swift -disable-objc-attr-requires-foundation-module
+// RUN: %target-swift-frontend -emit-ir -o- -parse-as-library -module-name test -import-objc-header %S/Inputs/objc_class_header.h -validate-tbd-against-ir=missing %t/main.swift -disable-objc-attr-requires-foundation-module
 
-// RUN: %target-swift-frontend -enable-resilience -emit-ir -o- -parse-as-library -module-name test -import-objc-header %S/Inputs/objc_class_header.h -validate-tbd-against-ir=all %t/main.swift -disable-objc-attr-requires-foundation-module
+// RUN: %target-swift-frontend -enable-resilience -emit-ir -o- -parse-as-library -module-name test -import-objc-header %S/Inputs/objc_class_header.h -validate-tbd-against-ir=missing %t/main.swift -disable-objc-attr-requires-foundation-module
 
 // With -enable-testing:
 
@@ -15,29 +15,56 @@
 
 import Foundation
 
-% names = ["Public", "Internal", "Private"]
+
+%{
+names = ["Open", "Public", "Internal", "Private"]
+
+METHODS = "{access} {override} func method{name}() {{}}"
+OBJC_METHOD = "@objc {access} {override} func method{name}ObjC() {{}}"
+def classBody(i, override=False):
+    methods = []
+    for name in names[i:]:
+        is_private = name == "Private"
+        args = {"access": name.lower(), "name": name,
+                "override": "override" if override and not is_private else ""}
+        methods.append(METHODS.format(**args))
+        if not is_private:
+            methods.append(OBJC_METHOD.format(**args))
+
+    return "\n".join(methods)
+}%
 
 % for i, name in enumerate(names):
 % access = name.lower()
 
 // a class by itself
-${access} class ${name}Empty: NSObject {}
+${access} class ${name}Empty: NSObject {
+${classBody(i)}
+}
 
 // subclasses of that
 % for subname in names[i:]:
 % subaccess = subname.lower()
-${subaccess} class ${subname}Sub${name}Empty: ${name}Empty {}
+${subaccess} class ${subname}Sub${name}Empty: ${name}Empty {
+${classBody(i, override=True)}
+}
 %end
 
-${access} class ${name}InheritObjCProtocol: NSObject, ObjCProtocol {}
+${access} class ${name}InheritObjCProtocol: NSObject, ObjCProtocol {
+${classBody(i)}
+}
 
 // some bugs were revealed when there's @objc without inheriting from
 // NSObject.
-@objc ${access} class ${name}ObjCOnly {}
+@objc ${access} class ${name}ObjCOnly {
+${classBody(i)}
+}
 
 %end
 
 @usableFromInline
 @objc
-internal class InternalObjCOnlyUsableFromInline {}
+internal class InternalObjCOnlyUsableFromInline {
+${classBody(names.index("Internal"))}
+}
 

test/TBD/specialization.swift

@@ -0,0 +1,35 @@
+// Validate the the specializations actually exist (if they don't then we're not
+// validating that they end up with the correct linkages):
+// RUN: %target-swift-frontend -emit-sil -o- -O -validate-tbd-against-ir=none %s | %FileCheck %s
+
+// RUN: %target-swift-frontend -emit-ir -o/dev/null -O -validate-tbd-against-ir=all %s
+// RUN: %target-swift-frontend -emit-ir -o/dev/null -O -validate-tbd-against-ir=all -enable-resilience %s
+
+// With -enable-testing:
+// RUN: %target-swift-frontend -emit-ir -o/dev/null -O -validate-tbd-against-ir=all -enable-testing %s
+// RUN: %target-swift-frontend -emit-ir -o/dev/null -O -validate-tbd-against-ir=all -enable-resilience -enable-testing %s
+
+// rdar://problem/40738913
+
+open class Foo {
+    @inline(never)
+    fileprivate func foo<T>(_: T.Type) {}
+}
+
+open class Bar<T> {
+    @inline(never)
+    fileprivate func bar() {}
+}
+
+
+public func f() {
+    Foo().foo(Int.self)
+    Bar<Int>().bar()
+}
+
+
+// CHECK-LABEL: // specialized Foo.foo<A>(_:)
+// CHECK-NEXT: sil private [noinline] @$S14specialization3FooC3foo33_A6E3E43DB6679655BDF5A878ABC489A0LLyyxmlFSi_Tg5Tf4dd_n : $@convention(thin) () -> ()
+
+// CHECK-LABEL: // specialized Bar.bar()
+// CHECK-NEXT: sil private [noinline] @$S14specialization3BarC3bar33_A6E3E43DB6679655BDF5A878ABC489A0LLyyFSi_Tg5Tf4d_n : $@convention(thin) () -> ()