Unique extended existential shapes using the generalized AST type.

I wrote out this whole analysis of why different existential types
might have the same logical content, and then I turned around and
immediately uniqued existential shapes purely by logical content
rather than the (generalized) formal type.  Oh well.  At least it's
not too late to make ABI changes like this.

We now store a reference to a mangling of the generalized formal
type directly in the shape.  This type alone is sufficient to unique
the shape:

- By the nature of the generalization algorithm, every type parameter
  in the generalization signature should be mentioned in the
  generalized formal type in a deterministic order.

- By the nature of the generalization algorithm, every other
  requirement in the generalization signature should be implied
  by the positions in which generalization type parameters appear
  (e.g. because the formal type is C<T> & P, where C constrains
  its type parameter for well-formedness).

- The requirement signature and type expression are extracted from
  the existential type.

As a result, we no longer rely on computing a unique hash at
compile time.

Storing this separately from the requirement signature potentially
allows runtimes with general shape support to work with future
extensions to existential types even if they cannot demangle the
generalized formal type.

Storing the generalized formal type also allows us to easily and
reliably extract the formal type of the existential.  Otherwise,
it's quite a heroic endeavor to match requirements back up with
primary associated types.  Doing so would also only allows us to
extract *some* matching formal type, not necessarily the *right*
formal type.  So there's some good synergy here.

Author: rjmccall

docs/ABI/Mangling.rst

@@ -637,7 +637,7 @@ Types
   type ::= protocol-list 'p'                 // existential type
   type ::= protocol-list superclass 'Xc'     // existential type with superclass
   type ::= protocol-list 'Xl'                // existential type with AnyObject
-  type ::= protocol-list 'y' (type* '_')* type* retroactive-conformance* 'Xp'   // parameterized protocol type
+  type ::= protocol-list 'y' (type* '_')* type* retroactive-conformance* 'XP'   // parameterized protocol type
   type ::= type-list 't'                     // tuple
   type ::= type generic-signature 'u'        // generic type
   type ::= 'x'                               // generic param, depth=0, idx=0
@@ -918,21 +918,9 @@ root protocol conformance, and the suffix 'g'.
 
 ::
 
-  // No generalization signature, no type expression.
-  extended-existential-shape ::= generic-signature 'Xg' extended-existential-value-storage
-
-  // Generalization signature (the second one), no type expression.
-  extended-existential-shape ::= generic-signature generic-signature 'XG' extended-existential-value-storage
-
-  // No generalization signature, type expression.
-  extended-existential-shape ::= generic-signature type 'Xh' extended-existential-value-storage
-
-  // Generalization signature (the second one), type expression.
-  extended-existential-shape ::= generic-signature generic-signature type 'Xh' extended-existential-value-storage
-
-  extended-existential-value-storage ::= 'o' // opaque
-  extended-existential-value-storage ::= 'c' // class
-  extended-existential-value-storage ::= 'm' // metatype
+  // No generalization signature.
+  extended-existential-shape ::= type 'Xg' // no generalization signature
+  extended-existential-shape ::= generic-signature type 'XG'
 
 Identifiers
 ~~~~~~~~~~~

include/swift/ABI/Metadata.h

@@ -1855,6 +1855,32 @@ struct TargetExtendedExistentialTypeShape
   /// Flags for the existential shape.
   ExtendedExistentialTypeShapeFlags Flags;
 
+  /// The mangling of the generalized existential type, expressed
+  /// (if necessary) in terms of the type parameters of the
+  /// generalization signature.
+  ///
+  /// If this shape is non-unique, this is always a flat string, not a
+  /// "symbolic" mangling which can contain relative references.  This
+  /// allows uniquing to simply compare the string content.
+  ///
+  /// In principle, the content of the requirement signature and type
+  /// expression are derivable from this type.  We store them separately
+  /// so that code which only needs to work with the logical content of
+  /// the type doesn't have to break down the existential type string.
+  /// This both (1) allows those operations to work substantially more
+  /// efficiently (and without needing code to produce a requirement
+  /// signature from an existential type to exist in the runtime) and
+  /// (2) potentially allows old runtimes to support new existential
+  /// types without as much invasive code.
+  ///
+  /// The content of this string is *not* necessarily derivable from
+  /// the requirement signature.  This is because there may be multiple
+  /// existential types that have equivalent logical content but which
+  /// we nonetheless distinguish at compile time.  Storing this also
+  /// allows us to far more easily produce a formal type from this
+  /// shape reflectively.
+  RelativeStringPointer ExistentialType;
+
   /// The header describing the requirement signature of the existential.
   TargetGenericContextDescriptorHeader<Runtime> ReqSigHeader;
 
@@ -2024,6 +2050,9 @@ struct UniqueHash {
 
 /// A descriptor for an extended existential type descriptor which
 /// needs to be uniqued at runtime.
+///
+/// Uniquing is performed by comparing the existential type strings
+/// of the shapes.
 template <typename Runtime>
 struct TargetNonUniqueExtendedExistentialTypeShape {
   /// A reference to memory that can be used to cache a globally-unique
@@ -2032,10 +2061,12 @@ struct TargetNonUniqueExtendedExistentialTypeShape {
     std::atomic<ConstTargetMetadataPointer<Runtime,
                   TargetExtendedExistentialTypeShape>>> UniqueCache;
 
-  /// A hash of the mangling of the existential shape.
-  ///
-  /// TODO: describe that mangling here
-  UniqueHash Hash;
+  llvm::StringRef getExistentialTypeStringForUniquing() const {
+    // When we have a non-unique shape, we're guaranteed that
+    // ExistentialType contains no symbolic references, so we can just
+    // recover it this way rather than having to parse it.
+    return LocalCopy.ExistentialType.get();
+  }
 
   /// The local copy of the existential shape descriptor.
   TargetExtendedExistentialTypeShape<Runtime> LocalCopy;

include/swift/Demangling/DemangleNodes.def

@@ -336,7 +336,6 @@ NODE(OpaqueReturnTypeIndexed)
 NODE(BackDeploymentThunk)
 NODE(BackDeploymentFallback)
 NODE(ExtendedExistentialTypeShape)
-NODE(ExtendedExistentialValueStorage)
 NODE(Uniquable)
 
 #undef CONTEXT_NODE

include/swift/IRGen/Linking.h

@@ -125,10 +125,6 @@ class LinkEntity {
     ExtendedExistentialIsUniqueMask = 0x100,
     ExtendedExistentialIsSharedShift = 9,
     ExtendedExistentialIsSharedMask = 0x200,
-    ExtendedExistentialMetatypeDepthShift = 10,
-    ExtendedExistentialMetatypeDepthMask =
-      ~(KindMask | ExtendedExistentialIsUniqueMask |
-        ExtendedExistentialIsSharedMask),
   };
 #define LINKENTITY_SET_FIELD(field, value) (value << field##Shift)
 #define LINKENTITY_GET_FIELD(value, field) ((value & field##Mask) >> field##Shift)
@@ -1368,9 +1364,7 @@ class LinkEntity {
   }
 
   static LinkEntity forExtendedExistentialTypeShape(CanGenericSignature genSig,
-                                                    CanExistentialType
-                                                      existentialType,
-                                                    unsigned metatypeDepth,
+                                                    CanType existentialType,
                                                     bool isUnique,
                                                     bool isShared) {
     LinkEntity entity;
@@ -1380,8 +1374,7 @@ class LinkEntity {
     entity.Data =
         LINKENTITY_SET_FIELD(Kind, unsigned(Kind::ExtendedExistentialTypeShape))
       | LINKENTITY_SET_FIELD(ExtendedExistentialIsUnique, unsigned(isUnique))
-      | LINKENTITY_SET_FIELD(ExtendedExistentialIsShared, unsigned(isShared))
-      | LINKENTITY_SET_FIELD(ExtendedExistentialMetatypeDepth, metatypeDepth);
+      | LINKENTITY_SET_FIELD(ExtendedExistentialIsShared, unsigned(isShared));
     return entity;
   }
 
@@ -1484,9 +1477,9 @@ class LinkEntity {
              reinterpret_cast<const GenericSignatureImpl*>(SecondaryPointer));
   }
 
-  CanExistentialType getExtendedExistentialTypeShapeType() const {
+  CanType getExtendedExistentialTypeShapeType() const {
     assert(getKind() == Kind::ExtendedExistentialTypeShape);
-    return cast<ExistentialType>(CanType(reinterpret_cast<TypeBase*>(Pointer)));
+    return CanType(reinterpret_cast<TypeBase*>(Pointer));
   }
 
   bool isExtendedExistentialTypeShapeUnique() const {
@@ -1499,11 +1492,6 @@ class LinkEntity {
     return LINKENTITY_GET_FIELD(Data, ExtendedExistentialIsShared);
   }
 
-  unsigned getExtendedExistentialTypeShapeMetatypeDepth() const {
-    assert(getKind() == Kind::ExtendedExistentialTypeShape);
-    return LINKENTITY_GET_FIELD(Data, ExtendedExistentialMetatypeDepth);
-  }
-
   bool isDynamicallyReplaceable() const {
     assert(getKind() == Kind::SILFunction);
     return LINKENTITY_GET_FIELD(Data, IsDynamicallyReplaceableImpl);

lib/Demangling/Demangler.cpp

@@ -3231,8 +3231,6 @@ NodePointer Demangler::demangleSpecialType() {
       return popFunctionType(Node::Kind::CFunctionPointer);
     case 'g':
     case 'G':
-    case 'h':
-    case 'H':
       return demangleExtendedExistentialShape(specialChar);
     case 'z':
       switch (auto cchar = nextChar()) {
@@ -3382,47 +3380,19 @@ NodePointer Demangler::demangleSpecialType() {
 }
 
 NodePointer Demangler::demangleExtendedExistentialShape(char nodeKind) {
-  assert(nodeKind == 'g' || nodeKind == 'G' ||
-         nodeKind == 'h' || nodeKind == 'H');
+  assert(nodeKind == 'g' || nodeKind == 'G');
 
-  NodePointer type = nullptr;
-  if (nodeKind == 'h' || nodeKind == 'H')
-    type = popNode(Node::Kind::Type);
+  NodePointer type = popNode(Node::Kind::Type);
 
   NodePointer genSig = nullptr;
-  if (nodeKind == 'G' || nodeKind == 'H')
+  if (nodeKind == 'G')
     genSig = popNode(Node::Kind::DependentGenericSignature);
 
-  NodePointer reqSig = popNode(Node::Kind::DependentGenericSignature);
-
-  NodePointer valueStorage = [&]() -> NodePointer {
-    switch (nextChar()) {
-    case 'o':
-      return createNode(Node::Kind::ExtendedExistentialValueStorage,
-                        "opaque");
-    case 'c':
-      return createNode(Node::Kind::ExtendedExistentialValueStorage,
-                        "class");
-    case 'm':
-      return createNode(Node::Kind::ExtendedExistentialValueStorage,
-                        "metatype");
-    default:
-      return nullptr;
-    }
-  }();
-  if (!valueStorage) return nullptr;
-
-  // Make a default type expression if one wasn't given.
-  if (!type) {
-    type = createType(getDependentGenericParamType(genSig ? 1 : 0, 0));
-  }
-
   if (genSig) {
     return createWithChildren(Node::Kind::ExtendedExistentialTypeShape,
-                              reqSig, genSig, type, valueStorage);
+                              genSig, type);
   } else {
-    return createWithChildren(Node::Kind::ExtendedExistentialTypeShape,
-                              reqSig, type, valueStorage);
+    return createWithChild(Node::Kind::ExtendedExistentialTypeShape, type);
   }
 }
 

lib/Demangling/NodePrinter.cpp

@@ -595,7 +595,6 @@ class NodePrinter {
     case Node::Kind::BackDeploymentThunk:
     case Node::Kind::BackDeploymentFallback:
     case Node::Kind::ExtendedExistentialTypeShape:
-    case Node::Kind::ExtendedExistentialValueStorage:
     case Node::Kind::Uniquable:
       return false;
     }
@@ -2975,9 +2974,8 @@ NodePointer NodePrinter::print(NodePointer Node, unsigned depth,
     auto savedDisplayWhereClauses = Options.DisplayWhereClauses;
     Options.DisplayWhereClauses = true;
 
-    NodePointer reqSig = Node->getChild(0);
     NodePointer genSig = nullptr, type = nullptr;
-    if (Node->getNumChildren() == 4) {
+    if (Node->getNumChildren() == 2) {
       genSig = Node->getChild(1);
       type = Node->getChild(2);
     } else {
@@ -2989,17 +2987,12 @@ NodePointer NodePrinter::print(NodePointer Node, unsigned depth,
       print(genSig, depth + 1);
       Printer << " ";
     }
-    Printer << "any";
-    print(reqSig, depth + 1);
-    Printer << " ";
+    Printer << "any ";
     print(type, depth + 1);
 
     Options.DisplayWhereClauses = savedDisplayWhereClauses;
     return nullptr;
   }
-  case Node::Kind::ExtendedExistentialValueStorage:
-    Printer << Node->getText();
-    return nullptr;
   }
 
   printer_unreachable("bad node kind!");

lib/Demangling/OldRemangler.cpp

@@ -2725,10 +2725,6 @@ ManglingError Remangler::mangleExtendedExistentialTypeShape(Node *node,
                                                      unsigned int depth) {
   return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
 }
-ManglingError Remangler::mangleExtendedExistentialValueStorage(Node *node,
-                                                     unsigned int depth) {
-  return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
-}
 ManglingError
 Remangler::mangleCanonicalSpecializedGenericMetaclass(Node *node,
                                                       unsigned depth) {

lib/Demangling/Remangler.cpp

@@ -3445,65 +3445,25 @@ ManglingError Remangler::mangleUniquable(Node *node, unsigned depth) {
 
 ManglingError Remangler::mangleExtendedExistentialTypeShape(Node *node,
                                                             unsigned depth) {
-  NodePointer reqSig, genSig, type, storage;
-  reqSig = node->getChild(0);
-  if (node->getNumChildren() == 3) {
+  NodePointer genSig, type;
+  if (node->getNumChildren() == 1) {
     genSig = nullptr;
-    type = node->getChild(1);
-    storage = node->getChild(2);
+    type = node->getChild(0);
   } else {
-    genSig = node->getChild(1);
-    type = node->getChild(2);
-    storage = node->getChild(3);
+    genSig = node->getChild(0);
+    type = node->getChild(1);
   }
 
-  RETURN_IF_ERROR(mangle(reqSig, depth + 1));
   if (genSig) {
     RETURN_IF_ERROR(mangle(genSig, depth + 1));  
   }
+  RETURN_IF_ERROR(mangle(type, depth + 1));
 
-  // Recognize the special case of a type expression that's just the
-  // unique requirement type parameter.
-  {
-    if (type->getKind() != Node::Kind::Type)
-      return MANGLING_ERROR(ManglingError::WrongNodeType, type);
-    auto typeNode = type->getChild(0);
-
-    if (typeNode->getKind() == Node::Kind::DependentGenericParamType) {
-      auto paramDepth = typeNode->getChild(0)->getIndex();
-      auto index = typeNode->getChild(1)->getIndex();
-      if (paramDepth == (genSig ? 1 : 0) && index == 0)
-        type = nullptr;
-    }
-  }
-  if (type)
-    RETURN_IF_ERROR(mangle(type, depth + 1));
-
-  if (genSig && type)
-    Buffer << "XH";
-  else if (type)
-    Buffer << "Xh";
-  else if (genSig)
+  if (genSig)
     Buffer << "XG";
   else
     Buffer << "Xg";
 
-  RETURN_IF_ERROR(mangle(storage, depth + 1));
-
-  return ManglingError::Success;
-}
-
-ManglingError Remangler::mangleExtendedExistentialValueStorage(Node *node,
-                                                           unsigned depth) {
-  if (node->getText() == "opaque")
-    Buffer << "o";
-  else if (node->getText() == "class")
-    Buffer << "c";
-  else if (node->getText() == "metatype")
-    Buffer << "m";
-  else
-    return MANGLING_ERROR(ManglingError::UnknownEncoding, node);
-
   return ManglingError::Success;
 }