Runtime: Don't attempt to round-trip mangled names for private types.

By design, we don't want private or function-nested types to be accessible by mangled name, since they don't have stable identities, and they could inadvertently become ABI if someone serialized a mangled string and expected to deserialize it into a type. Fixes rdar://problem/39826794 .

Author: jckarter

stdlib/public/runtime/Metadata.cpp

@@ -4196,6 +4196,16 @@ bool Metadata::satisfiesClassConstraint() const {
 }
 
 #if !NDEBUG
+static bool referencesAnonymousContext(Demangle::Node *node) {
+  if (node->getKind() == Demangle::Node::Kind::AnonymousContext)
+    return true;
+  for (unsigned i = 0, e = node->getNumChildren(); i < e; ++i)
+    if (referencesAnonymousContext(node->getChild(i)))
+      return true;
+
+  return false;
+}
+
 void swift::verifyMangledNameRoundtrip(const Metadata *metadata) {
   // Enable verification when a special environment variable is set.
   // Some metatypes crash when going through the mangler or demangler. A
@@ -4211,12 +4221,17 @@ void swift::verifyMangledNameRoundtrip(const Metadata *metadata) {
 
   Demangle::Demangler Dem;
   auto node = _swift_buildDemanglingForMetadata(metadata, Dem);
+  // If the mangled node involves types in an AnonymousContext, then by design,
+  // it cannot be looked up by name.
+  if (referencesAnonymousContext(node))
+    return;
+  
   auto mangledName = Demangle::mangleNode(node);
   auto result = _getTypeByMangledName(mangledName,
                                       [](unsigned, unsigned){ return nullptr; });
   if (metadata != result)
     swift::warning(RuntimeErrorFlagNone,
-                   "Metadata mangled name failed to roundtrip: %p -> %s -> %p",
+                   "Metadata mangled name failed to roundtrip: %p -> %s -> %p\n",
                    metadata, mangledName.c_str(), (const Metadata *)result);
 }
 #endif