[TypeChecker] Fix assertion failure when using Self in extension that’s not on top-level

We fail to resolve the extended type if an extension is not declared at the top level. This causes us to diagnose a lookup failure on `Self`. If the extended type is a protocol, we thus end up with a nominal Self type that’s not a class, violating the assertion that’s currently in place. To fix the crasher, convert the assertion to an `if` condition and issue a generic "cannot find type 'Self' in scope" in the else case.

Cherry-picking 78d6fb2 from the `main` branch.

Author: ahoppen

lib/Sema/TypeCheckType.cpp

@@ -1131,21 +1131,26 @@ static Type diagnoseUnknownType(TypeResolution resolution,
       NominalTypeDecl *nominal = nullptr;
       if ((nominalDC = dc->getInnermostTypeContext()) &&
           (nominal = nominalDC->getSelfNominalTypeDecl())) {
-        // Attempt to refer to 'Self' within a non-protocol nominal
-        // type. Fix this by replacing 'Self' with the nominal type name.
-        assert(isa<ClassDecl>(nominal) && "Must be a class");
+        if (isa<ClassDecl>(nominal)) {
+          // Attempt to refer to 'Self' within a non-protocol nominal
+          // type. Fix this by replacing 'Self' with the nominal type name.
 
-        // Produce a Fix-It replacing 'Self' with the nominal type name.
-        auto name = getDeclNameFromContext(dc, nominal);
-        diags.diagnose(comp->getNameLoc(), diag::dynamic_self_invalid, name)
-          .fixItReplace(comp->getNameLoc().getSourceRange(), name);
+          // Produce a Fix-It replacing 'Self' with the nominal type name.
+          auto name = getDeclNameFromContext(dc, nominal);
+          diags.diagnose(comp->getNameLoc(), diag::dynamic_self_invalid, name)
+            .fixItReplace(comp->getNameLoc().getSourceRange(), name);
 
-        auto type = resolution.mapTypeIntoContext(
-          dc->getInnermostTypeContext()->getSelfInterfaceType());
+          auto type = resolution.mapTypeIntoContext(
+            dc->getInnermostTypeContext()->getSelfInterfaceType());
 
-        comp->overwriteNameRef(DeclNameRef(nominal->getName()));
-        comp->setValue(nominal, nominalDC->getParent());
-        return type;
+          comp->overwriteNameRef(DeclNameRef(nominal->getName()));
+          comp->setValue(nominal, nominalDC->getParent());
+          return type;
+        } else {
+          diags.diagnose(comp->getNameLoc(), diag::cannot_find_type_in_scope,
+                         comp->getNameRef());
+          return ErrorType::get(ctx);
+        }
       }
       // Attempt to refer to 'Self' from a free function.
       diags.diagnose(comp->getNameLoc(), diag::dynamic_self_non_method,

validation-test/compiler_crashers_2_fixed/sr14454.swift

@@ -0,0 +1,12 @@
+// RUN: not %target-swift-frontend -typecheck %s
+protocol Foo {}
+
+func withTrailingClosure(x: (Int) -> Void) {}
+
+_ = withTrailingClosure { (x) in
+  extension Foo {
+    func foo() {
+      _ = MemoryLayout<Self>.size
+    }
+  }
+}