Give opaque existential containers extra inhabitants.

We can use the extra inhabitants of the type metadata field as extra inhabitants of the entire
existential container, allowing `Any?` and similar types to be the same size as non-optional
existentials.

Author: jckarter

lib/IRGen/GenExistential.cpp

@@ -109,6 +109,11 @@ namespace {
     Address projectMetadataRef(IRGenFunction &IGF, Address addr) {
       return IGF.Builder.CreateStructGEP(addr, 1, getFixedBufferSize(IGF.IGM));
     }
+    
+    /// Give the offset of the metadata field of an existential object.
+    Size getMetadataRefOffset(IRGenModule &IGM) {
+      return getFixedBufferSize(IGM);
+    }
 
     /// Given the address of an existential object, load its metadata
     /// object.
@@ -811,12 +816,12 @@ class OpaqueExistentialTypeInfo final :
              IndirectTypeInfo<OpaqueExistentialTypeInfo, FixedTypeInfo>>;
   friend super;
 
-  // FIXME: We could get spare bits out of the metadata and/or witness
-  // pointers.
   OpaqueExistentialTypeInfo(ArrayRef<const ProtocolDecl *> protocols,
-                            llvm::Type *ty, Size size, Alignment align)
+                            llvm::Type *ty, Size size,
+                            SpareBitVector &&spareBits,
+                            Alignment align)
     : super(protocols, ty, size,
-            SpareBitVector::getConstant(size.getValueInBits(), false), align,
+            std::move(spareBits), align,
             IsNotPOD, IsBitwiseTakable, IsFixedSize) {}
 
 public:
@@ -903,6 +908,37 @@ class OpaqueExistentialTypeInfo final :
     call->setDoesNotThrow();
     return;
   }
+               
+  // Opaque existentials have extra inhabitants and spare bits in their type
+  // metadata pointer, matching those of a standalone thick metatype (which
+  // in turn match those of a heap object).
+  unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
+    return getHeapObjectExtraInhabitantCount(IGM);
+  }
+  APInt getFixedExtraInhabitantValue(IRGenModule &IGM,
+                                     unsigned bits,
+                                     unsigned index) const override {
+    auto offset = getLayout().getMetadataRefOffset(IGM).getValueInBits();
+    return getHeapObjectFixedExtraInhabitantValue(IGM, bits, index, offset);
+  }
+  APInt getFixedExtraInhabitantMask(IRGenModule &IGM) const override {
+    auto mask = APInt::getAllOnesValue(IGM.getPointerSize().getValueInBits());
+    mask = mask.zext(getFixedSize().getValueInBits());
+    mask = mask.shl(getLayout().getMetadataRefOffset(IGM).getValueInBits());
+    return mask;
+  }
+  llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
+                                       Address src, SILType T,
+                                       bool isOutlined) const override {
+    auto type = getLayout().projectMetadataRef(IGF, src);
+    return getHeapObjectExtraInhabitantIndex(IGF, type);
+  }
+  void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index,
+                            Address dest, SILType T, bool isOutlined)
+  const override {
+    auto type = getLayout().projectMetadataRef(IGF, dest);
+    return storeHeapObjectExtraInhabitant(IGF, index, type);
+  }
 };
 
 
@@ -1408,7 +1444,27 @@ static const TypeInfo *createExistentialTypeInfo(IRGenModule &IGM, CanType T) {
   OpaqueExistentialLayout opaque(protosWithWitnessTables.size());
   Alignment align = opaque.getAlignment(IGM);
   Size size = opaque.getSize(IGM);
+  // There are spare bits in the metadata pointer and witness table pointers
+  // consistent with a native object reference.
+  SpareBitVector spareBits;
+  spareBits.appendClearBits(size.getValueInBits());
+  /* TODO: There are spare bits we could theoretically use in the type metadata
+     and witness table pointers, but opaque existentials are currently address-
+     only, and we can't soundly take advantage of spare bits for in-memory
+     representations.
+   
+  auto metadataOffset = opaque.getMetadataRefOffset(IGM);
+  spareBits.appendClearBits(metadataOffset.getValueInBits());
+  auto typeSpareBits = IGM.getHeapObjectSpareBits();
+  spareBits.append(typeSpareBits);
+  auto witnessSpareBits =
+    IGM.getWitnessTablePtrSpareBits();
+  for (unsigned i = 0, e = protosWithWitnessTables.size(); i < e; ++i)
+    spareBits.append(witnessSpareBits);
+  assert(spareBits.size() == size.getValueInBits());
+   */
   return OpaqueExistentialTypeInfo::create(protosWithWitnessTables, type, size,
+                                           std::move(spareBits),
                                            align);
 }
 

lib/IRGen/GenOpaque.cpp

@@ -38,7 +38,7 @@
 using namespace swift;
 using namespace irgen;
 
-/// A fixed-size buffer is always 16 bytes and pointer-aligned.
+/// A fixed-size buffer is always three pointers in size and pointer-aligned.
 /// If we align them more, we'll need to introduce padding to
 /// make protocol types work.
 Size irgen::getFixedBufferSize(IRGenModule &IGM) {

stdlib/public/Reflection/TypeLowering.cpp

@@ -436,7 +436,10 @@ unsigned RecordTypeInfoBuilder::addField(unsigned fieldSize,
   switch (Kind) {
   // The extra inhabitants of a struct are the same as the extra
   // inhabitants of the field that has the most.
+  // Opaque existentials pick up the extra inhabitants of their type metadata
+  // field.
   case RecordKind::Struct:
+  case RecordKind::OpaqueExistential:
     NumExtraInhabitants = std::max(NumExtraInhabitants, numExtraInhabitants);
     break;
 
@@ -450,7 +453,6 @@ unsigned RecordTypeInfoBuilder::addField(unsigned fieldSize,
   case RecordKind::Invalid:
   case RecordKind::MultiPayloadEnum:
   case RecordKind::NoPayloadEnum:
-  case RecordKind::OpaqueExistential:
   case RecordKind::SinglePayloadEnum:
   case RecordKind::ThickFunction:
   case RecordKind::Tuple:

stdlib/public/runtime/ExistentialMetadataImpl.h

@@ -328,7 +328,18 @@ struct LLVM_LIBRARY_VISIBILITY OpaqueExistentialBox
   static constexpr size_t stride = sizeof(Container);
   static constexpr size_t isPOD = false;
   static constexpr bool isBitwiseTakable = true;
-  static constexpr unsigned numExtraInhabitants = 0;
+  static constexpr unsigned numExtraInhabitants =
+    swift_getHeapObjectExtraInhabitantCount();
+  
+  static void storeExtraInhabitant(Container *dest, int index) {
+    swift_storeHeapObjectExtraInhabitant((HeapObject**)&dest->Header.Type,
+                                         index);
+  }
+
+  static int getExtraInhabitantIndex(const Container *src) {
+    return swift_getHeapObjectExtraInhabitantIndex(
+                                        (HeapObject* const *)&src->Header.Type);
+  }
 };
 
 /// A non-fixed box implementation class for an opaque existential
@@ -371,7 +382,18 @@ struct LLVM_LIBRARY_VISIBILITY NonFixedOpaqueExistentialBox
   };
 
   using type = Container;
-  static constexpr unsigned numExtraInhabitants = 0;
+  static constexpr unsigned numExtraInhabitants =
+    swift_getHeapObjectExtraInhabitantCount();
+  
+  static void storeExtraInhabitant(Container *dest, int index) {
+    swift_storeHeapObjectExtraInhabitant((HeapObject**)&dest->Header.Type,
+                                         index);
+  }
+
+  static int getExtraInhabitantIndex(const Container *src) {
+    return swift_getHeapObjectExtraInhabitantIndex(
+                                  (HeapObject* const *)&src->Header.Type);
+  }
 };
 
 /// A common base class for fixed and non-fixed class-existential box

stdlib/public/runtime/Metadata.cpp

@@ -2741,7 +2741,7 @@ class ExistentialCacheEntry {
 
 class OpaqueExistentialValueWitnessTableCacheEntry {
 public:
-  ValueWitnessTable Data;
+  ExtraInhabitantsValueWitnessTable Data;
 
   OpaqueExistentialValueWitnessTableCacheEntry(unsigned numTables);
 
@@ -2798,9 +2798,11 @@ class ClassExistentialValueWitnessTableCacheEntry {
 /// The uniquing structure for existential type metadata.
 static SimpleGlobalCache<ExistentialCacheEntry> ExistentialTypes;
 
-static const ValueWitnessTable OpaqueExistentialValueWitnesses_0 =
+static const ExtraInhabitantsValueWitnessTable
+OpaqueExistentialValueWitnesses_0 =
   ValueWitnessTableForBox<OpaqueExistentialBox<0>>::table;
-static const ValueWitnessTable OpaqueExistentialValueWitnesses_1 =
+static const ExtraInhabitantsValueWitnessTable
+OpaqueExistentialValueWitnesses_1 =
   ValueWitnessTableForBox<OpaqueExistentialBox<1>>::table;
 
 /// The standard metadata for Any.
@@ -2855,7 +2857,6 @@ OpaqueExistentialValueWitnessTableCacheEntry::
 OpaqueExistentialValueWitnessTableCacheEntry(unsigned numWitnessTables) {
   using Box = NonFixedOpaqueExistentialBox;
   using Witnesses = NonFixedValueWitnesses<Box, /*known allocated*/ true>;
-  static_assert(!Witnesses::hasExtraInhabitants, "no extra inhabitants");
 
 #define WANT_ONLY_REQUIRED_VALUE_WITNESSES
 #define VALUE_WITNESS(LOWER_ID, UPPER_ID) \
@@ -2869,8 +2870,13 @@ OpaqueExistentialValueWitnessTableCacheEntry(unsigned numWitnessTables) {
     .withPOD(false)
     .withBitwiseTakable(true)
     .withInlineStorage(false)
-    .withExtraInhabitants(false);
+    .withExtraInhabitants(true);
   Data.stride = Box::Container::getStride(numWitnessTables);
+  
+  // Extra inhabitant behavior does not change with the number of witnesses.
+  Data.extraInhabitantFlags = Witnesses::extraInhabitantFlags;
+  Data.getExtraInhabitantIndex = Witnesses::getExtraInhabitantIndex;
+  Data.storeExtraInhabitant = Witnesses::storeExtraInhabitant;
 
   assert(getNumWitnessTables() == numWitnessTables);
 }

stdlib/public/runtime/MetadataImpl.h

@@ -918,13 +918,12 @@ struct NonFixedValueWitnesses :
 
   static void storeExtraInhabitant(OpaqueValue *dest, int index,
                                    const Metadata *self) {
-    Box::storeExtraInhabitant((typename Box::type*) dest, index, self);
+    Box::storeExtraInhabitant((typename Box::type*) dest, index);
   }
 
   static int getExtraInhabitantIndex(const OpaqueValue *src,
                                      const Metadata *self) {
-    return Box::getExtraInhabitantIndex((typename Box::type const *) src,
-                                        self);
+    return Box::getExtraInhabitantIndex((typename Box::type const *) src);
   }
 };
 

test/IRGen/existentials.sil

@@ -8,8 +8,8 @@ import Swift
 protocol P {}
 
 // NonBitwiseTakableBit = 0x00100000. This struct is bitwise takable because
-// 0x30007 = 196615.
-// CHECK: @"$S12existentials14BitwiseTakableVWV" = internal constant [11 x i8*] {{.*}} i8* inttoptr (i64 196615 to i8*)
+// 0x70007 = 458759
+// CHECK: @"$S12existentials14BitwiseTakableVWV" = internal constant [14 x i8*] {{.*}} i8* inttoptr (i64 458759 to i8*)
 struct BitwiseTakable {
   var p: P
 }