Add runtime functions to compute tuple layouts from element layouts.

Previously, when a tuple type had non-fixed layout, we would compute
a layout by building the metadata for that tuple type and then
extracting the layout from the VWT.  This can be quite expensive
because it involves constructing the exact metadata for types like
arrays and functions despite those types being fixed-layout across
all instantiations.  It also tends to cause unnecessary recursive-type
issues, especially with enums where tuples are currently used to model
cases with mutliple payloads.  Since we just need a layout, computing
it directly from element layouts instead of constructing metadata for
the formal type lets us take advantage of all the other fast paths for
layout construction, e.g. for fixed types and single-field aggregates.

This is a good improvement overall, but it also serves to alleviate
some of the problems of rdar://40810002 / SR-7876 in a way that
might be suitable for integration to 4.2.

Author: rjmccall

include/swift/Runtime/Metadata.h

@@ -531,6 +531,46 @@ swift_getTupleTypeMetadata3(MetadataRequest request,
                             const Metadata *elt2, const char *labels,
                             const ValueWitnessTable *proposedWitnesses);
 
+/// Perform layout as if for a tuple whose elements have the given layouts.
+///
+/// \param tupleLayout - A structure into which to write the tuple layout.
+///   Must be non-null.
+/// \param elementOffsets - An array into which to write the offsets of
+///   the elements.  May be null.  Must have space for all elements,
+///   including element 0 (which will always have offset 0).
+SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
+void swift_getTupleTypeLayout(TypeLayout *tupleLayout,
+                              uint32_t *elementOffsets,
+                              TupleTypeFlags flags,
+                              const TypeLayout * const *elements);
+
+/// Perform layout as if for a two-element tuple whose elements have
+/// the given layouts.
+///
+/// \param tupleLayout - A structure into which to write the tuple layout.
+///   Must be non-null.
+/// \returns The offset of the second element.
+///   The first element always has offset 0.
+SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
+size_t swift_getTupleTypeLayout2(TypeLayout *tupleLayout,
+                                 const TypeLayout *elt0,
+                                 const TypeLayout *elt1);
+
+struct OffsetPair { size_t First; size_t Second; };
+
+/// Perform layout as if for a three-element tuple whose elements have
+/// the given layouts.
+///
+/// \param tupleLayout - A structure into which to write the tuple layout.
+///   Must be non-null.
+/// \returns The offsets of the second and third elements.
+///   The first element always has offset 0.
+SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
+OffsetPair swift_getTupleTypeLayout3(TypeLayout *tupleLayout,
+                                     const TypeLayout *elt0Layout,
+                                     const TypeLayout *elt1Layout,
+                                     const TypeLayout *elt2Layout);
+
 /// Initialize the value witness table and struct field offset vector for a
 /// struct, using the "Universal" layout strategy.
 SWIFT_RUNTIME_EXPORT

include/swift/Runtime/RuntimeFunctions.def

@@ -765,6 +765,34 @@ FUNCTION(GetTupleMetadata3, swift_getTupleTypeMetadata3, SwiftCC,
               Int8PtrTy, WitnessTablePtrTy),
          ATTRS(NoUnwind, ReadOnly))
 
+// void swift_getTupleTypeLayout(TypeLayout *result,
+//                               uint32_t offsets,
+//                               TupleTypeFlags flags,
+//                               const TypeLayout * const *elts);
+FUNCTION(GetTupleLayout, swift_getTupleTypeLayout, SwiftCC,
+         RETURNS(VoidTy),
+         ARGS(FullTypeLayoutTy->getPointerTo(0), Int32Ty->getPointerTo(0),
+              SizeTy, Int8PtrPtrTy->getPointerTo(0)),
+         ATTRS(NoUnwind))
+
+// size_t swift_getTupleTypeLayout2(TypeLayout *layout,
+//                                  const TypeLayout *elt0,
+//                                  const TypeLayout *elt1);
+FUNCTION(GetTupleLayout2, swift_getTupleTypeLayout2, SwiftCC,
+         RETURNS(SizeTy),
+         ARGS(FullTypeLayoutTy->getPointerTo(0), Int8PtrPtrTy, Int8PtrPtrTy),
+         ATTRS(NoUnwind))
+
+// OffsetPair swift_getTupleTypeLayout3(TypeLayout *layout,
+//                                      const TypeLayout *elt0,
+//                                      const TypeLayout *elt1,
+//                                      const TypeLayout *elt2);
+FUNCTION(GetTupleLayout3, swift_getTupleTypeLayout3, SwiftCC,
+         RETURNS(OffsetPairTy),
+         ARGS(FullTypeLayoutTy->getPointerTo(0),
+              Int8PtrPtrTy, Int8PtrPtrTy, Int8PtrPtrTy),
+         ATTRS(NoUnwind))
+
 // Metadata *swift_getExistentialTypeMetadata(
 //                              ProtocolClassConstraint classConstraint,
 //                              const Metadata *superclassConstraint,

lib/IRGen/IRGenModule.cpp

@@ -199,6 +199,16 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
     SizeTy
   });
 
+  OffsetPairTy = llvm::StructType::get(getLLVMContext(), { SizeTy, SizeTy });
+
+  // The TypeLayout structure, including all possible trailing components.
+  FullTypeLayoutTy = createStructType(*this, "swift.full_type_layout", {
+    SizeTy, // size
+    SizeTy, // flags
+    SizeTy, // alignment
+    SizeTy  // extra inhabitant flags (optional)
+  });
+
   // A protocol descriptor describes a protocol. It is not type metadata in
   // and of itself, but is referenced in the structure of existential type
   // metadata records.

lib/IRGen/IRGenModule.h

@@ -530,6 +530,8 @@ class IRGenModule {
     llvm::StructType *TypeMetadataResponseTy;   /// { %swift.type*, iSize }
     llvm::StructType *TypeMetadataDependencyTy; /// { %swift.type*, iSize }
   };
+  llvm::StructType *OffsetPairTy;      /// { iSize, iSize }
+  llvm::StructType *FullTypeLayoutTy;  /// %swift.full_type_layout = { ... }
   llvm::PointerType *TupleTypeMetadataPtrTy; /// %swift.tuple_type*
   llvm::StructType *FullHeapMetadataStructTy; /// %swift.full_heapmetadata = type { ... }
   llvm::PointerType *FullHeapMetadataPtrTy;/// %swift.full_heapmetadata*

lib/IRGen/MetadataRequest.cpp

@@ -2126,6 +2126,23 @@ namespace {
       return emitFromValueWitnessTablePointer(vwtable);
     }
 
+    /// Given that the type is fixed-layout, emit the type layout by
+    /// emitting a global layout for it.
+    llvm::Value *emitFromFixedLayout(CanType t) {
+      auto layout = tryEmitFromFixedLayout(t);
+      assert(layout && "type must be fixed-size to call emitFromFixedLayout");
+      return layout;
+    }
+
+    /// If the type is fixed-layout, emit the type layout by
+    /// emitting a global layout for it.
+    llvm::Value *tryEmitFromFixedLayout(CanType t) {
+      auto &ti = IGF.getTypeInfo(SILType::getPrimitiveObjectType(t));
+      if (auto fixedTI = dyn_cast<FixedTypeInfo>(&ti))
+        return IGF.IGM.emitFixedTypeLayout(t, *fixedTI);
+      return nullptr;
+    }
+
     bool hasVisibleValueWitnessTable(CanType t) const {
       // Some builtin and structural types have value witnesses exported from
       // the runtime.
@@ -2233,7 +2250,7 @@ namespace {
       }
       case MetatypeRepresentation::Thick:
         if (isa<ExistentialMetatypeType>(type)) {
-          return emitFromTypeMetadata(type, request);
+          return emitFromFixedLayout(type);
         }
         // Otherwise, this is a metatype that looks like a pointer.
         LLVM_FALLTHROUGH;
@@ -2340,6 +2357,103 @@ namespace {
                                                        type->getOwnership());
       return emitFromValueWitnessTable(valueWitnessType);
     }
+
+    llvm::Value *visitTupleType(CanTupleType type,
+                                DynamicMetadataRequest request) {
+      // Single-element tuples have exactly the same layout as their elements.
+      if (type->getNumElements() == 1) {
+        return visit(type.getElementType(0), request);
+      }
+
+      // If the type is fixed-layout, use a global layout.
+      if (auto layout = tryEmitFromFixedLayout(type))
+        return layout;
+
+      // TODO: check for cached VWT / metadata for the type.
+
+      // Use swift_getTupleTypeLayout to compute a layout.
+
+      // Create a buffer to hold the result.  We don't have any reasonable
+      // way to scope the lifetime of this.
+      auto resultPtr = IGF.createAlloca(IGF.IGM.FullTypeLayoutTy,
+                                        IGF.IGM.getPointerAlignment())
+                          .getAddress();
+
+      switch (type->getNumElements()) {
+      case 0:
+      case 1:
+        llvm_unreachable("filtered out above");
+
+      case 2: {
+        auto elt0 = visit(type.getElementType(0), request);
+        auto elt1 = visit(type.getElementType(1), request);
+
+        // Ignore the offset.
+        auto call = IGF.Builder.CreateCall(IGF.IGM.getGetTupleLayout2Fn(),
+                                           {resultPtr, elt0, elt1});
+        call->setDoesNotThrow();
+
+        break;
+      }
+
+      case 3: {
+        auto elt0 = visit(type.getElementType(0), request);
+        auto elt1 = visit(type.getElementType(1), request);
+        auto elt2 = visit(type.getElementType(2), request);
+
+        // Ignore the offsets.
+        auto call = IGF.Builder.CreateCall(IGF.IGM.getGetTupleLayout3Fn(),
+                                           {resultPtr, elt0, elt1, elt2});
+        call->setDoesNotThrow();
+
+        break;
+      }
+
+      default: {
+        // Allocate a temporary array for the element layouts.
+        auto eltLayoutsArraySize =
+          IGF.IGM.getPointerSize() * type->getNumElements();
+        auto eltLayoutsArray =
+          IGF.createAlloca(IGF.IGM.Int8PtrPtrTy,
+                           IGF.IGM.getSize(Size(type->getNumElements())),
+                           IGF.IGM.getPointerAlignment());
+        IGF.Builder.CreateLifetimeStart(eltLayoutsArray, eltLayoutsArraySize);
+
+        // Emit layouts for all the elements and store them into the array.
+        for (auto i : indices(type.getElementTypes())) {
+          auto eltLayout = visit(type.getElementType(i), request);
+          auto eltLayoutSlot =
+            i == 0 ? eltLayoutsArray
+                   : IGF.Builder.CreateConstArrayGEP(eltLayoutsArray, i,
+                                                     IGF.IGM.getPointerSize());
+          IGF.Builder.CreateStore(eltLayout, eltLayoutSlot);
+        }
+
+        // Ignore the offsets.
+        auto offsetsPtr =
+          llvm::ConstantPointerNull::get(IGF.IGM.Int32Ty->getPointerTo());
+
+        // Flags.
+        auto flags = TupleTypeFlags().withNumElements(type->getNumElements());
+        auto flagsValue = IGF.IGM.getSize(Size(flags.getIntValue()));
+
+        // Compute the layout.
+        auto call = IGF.Builder.CreateCall(IGF.IGM.getGetTupleLayoutFn(),
+                                           {resultPtr, offsetsPtr, flagsValue,
+                                            eltLayoutsArray.getAddress()});
+        call->setDoesNotThrow();
+
+        // We're done with the buffer.
+        IGF.Builder.CreateLifetimeEnd(eltLayoutsArray, eltLayoutsArraySize);
+
+        break;
+      }
+      }
+
+      // Cast resultPtr to i8**, our general currency type for type layouts.
+      resultPtr = IGF.Builder.CreateBitCast(resultPtr, IGF.IGM.Int8PtrPtrTy);
+      return resultPtr;
+    }
   };
 
 } // end anonymous namespace

stdlib/public/runtime/Metadata.cpp

@@ -1378,6 +1378,43 @@ static void performBasicLayout(TypeLayout &layout,
   layout.stride = std::max(size_t(1), roundUpToAlignMask(size, alignMask));
 }
 
+
+size_t swift::swift_getTupleTypeLayout2(TypeLayout *result,
+                                        const TypeLayout *elt0,
+                                        const TypeLayout *elt1) {
+  const TypeLayout *elts[] = { elt0, elt1 };
+  uint32_t offsets[2];
+  swift_getTupleTypeLayout(result, offsets,
+                           TupleTypeFlags().withNumElements(2), elts);
+  assert(offsets[0] == 0);
+  return offsets[1];
+}
+
+OffsetPair swift::swift_getTupleTypeLayout3(TypeLayout *result,
+                                            const TypeLayout *elt0,
+                                            const TypeLayout *elt1,
+                                            const TypeLayout *elt2) {
+  const TypeLayout *elts[] = { elt0, elt1, elt2 };
+  uint32_t offsets[3];
+  swift_getTupleTypeLayout(result, offsets,
+                           TupleTypeFlags().withNumElements(3), elts);
+  assert(offsets[0] == 0);
+  return {offsets[1], offsets[2]};
+}
+
+void swift::swift_getTupleTypeLayout(TypeLayout *result,
+                                     uint32_t *elementOffsets,
+                                     TupleTypeFlags flags,
+                                     const TypeLayout * const *elements) {
+  *result = TypeLayout();
+  performBasicLayout(*result, elements, flags.getNumElements(),
+    [](const TypeLayout *elt) { return elt; },
+    [elementOffsets](size_t i, const TypeLayout *elt, size_t offset) {
+      if (elementOffsets)
+        elementOffsets[i] = uint32_t(offset);
+    });
+}
+
 MetadataResponse
 swift::swift_getTupleTypeMetadata(MetadataRequest request,
                                   TupleTypeFlags flags,

test/IRGen/enum_resilience.swift

@@ -331,21 +331,27 @@ public func constructFullyFixed() -> FullyFixedLayout {
 }
 
 // CHECK-LABEL: define internal swiftcc %swift.metadata_response @"$S15enum_resilience24EnumWithResilientPayloadOMr"(%swift.type*, i8*, i8**)
+// CHECK:        [[TUPLE_LAYOUT:%.*]] = alloca %swift.full_type_layout
 // CHECK:        [[SIZE_RESPONSE:%.*]] = call swiftcc %swift.metadata_response @"$S16resilient_struct4SizeVMa"([[INT]] 319)
 // CHECK-NEXT:   [[SIZE_METADATA:%.*]] = extractvalue %swift.metadata_response [[SIZE_RESPONSE]], 0
 // CHECK-NEXT:   [[SIZE_STATE:%.*]] = extractvalue %swift.metadata_response [[SIZE_RESPONSE]], 1
 // CHECK-NEXT:   [[T0:%.*]] = icmp ule [[INT]] [[SIZE_STATE]], 63
 // CHECK-NEXT:   br i1 [[T0]], label %[[SATISFIED1:.*]], label
 // CHECK:      [[SATISFIED1]]:
-// CHECK:        [[TUPLE_RESPONSE:%.*]] = call swiftcc %swift.metadata_response @swift_getTupleTypeMetadata2([[INT]] 319, %swift.type* [[SIZE_METADATA]], %swift.type* [[SIZE_METADATA]], i8* null, i8** null)
-// CHECK-NEXT:   [[TUPLE_METADATA:%.*]] = extractvalue %swift.metadata_response [[TUPLE_RESPONSE]], 0
-// CHECK-NEXT:   [[TUPLE_STATE:%.*]] = extractvalue %swift.metadata_response [[TUPLE_RESPONSE]], 1
-// CHECK-NEXT:   [[T0:%.*]] = icmp ule [[INT]] [[TUPLE_STATE]], 63
-// CHECK-NEXT:   br i1 [[T0]], label %[[SATISFIED2:.*]], label
-// CHECK:      [[SATISFIED2]]:
+// CHECK-NEXT:   [[T0:%.*]] = bitcast %swift.type* [[SIZE_METADATA]] to i8***
+// CHECK-NEXT:   [[T1:%.*]] = getelementptr inbounds i8**, i8*** [[T0]], [[INT]] -1
+// CHECK-NEXT:   [[SIZE_VWT:%.*]] = load i8**, i8*** [[T1]],
+// CHECK-NEXT:   [[SIZE_LAYOUT_1:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
+// CHECK-NEXT:   store i8** [[SIZE_LAYOUT_1]],
+// CHECK-NEXT:   getelementptr
+// CHECK-NEXT:   [[SIZE_LAYOUT_2:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
+// CHECK-NEXT:   [[SIZE_LAYOUT_3:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
+// CHECK-NEXT:   call swiftcc [[INT]] @swift_getTupleTypeLayout2(%swift.full_type_layout* [[TUPLE_LAYOUT]], i8** [[SIZE_LAYOUT_2]], i8** [[SIZE_LAYOUT_3]])
+// CHECK-NEXT:   [[T0:%.*]] = bitcast %swift.full_type_layout* [[TUPLE_LAYOUT]] to i8**
+// CHECK-NEXT:   store i8** [[T0]],
 // CHECK:        call void @swift_initEnumMetadataMultiPayload
-// CHECK:        phi %swift.type* [ [[SIZE_METADATA]], %entry ], [ [[TUPLE_METADATA]], %[[SATISFIED1]] ], [ null, %[[SATISFIED2]] ]
-// CHECK:        phi [[INT]] [ 63, %entry ], [ 63, %[[SATISFIED1]] ], [ 0, %[[SATISFIED2]] ]
+// CHECK:        phi %swift.type* [ [[SIZE_METADATA]], %entry ], [ null, %[[SATISFIED1]] ]
+// CHECK:        phi [[INT]] [ 63, %entry ], [ 0, %[[SATISFIED1]] ]
 
 
 public protocol Prot {

test/IRGen/struct_resilience.swift

@@ -198,20 +198,29 @@ public func resilientAny(s : ResilientWeakRef) {
 
 // CHECK-LABEL:  define internal swiftcc %swift.metadata_response @"$S17struct_resilience26StructWithResilientStorageVMr"(%swift.type*, i8*, i8**)
 // CHECK: [[FIELDS:%.*]] = alloca [4 x i8**]
+// CHECK: [[TUPLE_LAYOUT:%.*]] = alloca %swift.full_type_layout,
 
 // CHECK: [[FIELDS_ADDR:%.*]] = getelementptr inbounds [4 x i8**], [4 x i8**]* [[FIELDS]], i32 0, i32 0
 
 // public let s: Size
 
-// CHECK: call swiftcc %swift.metadata_response @"$S16resilient_struct4SizeVMa"([[INT]] 319)
+// CHECK: [[T0:%.*]] = call swiftcc %swift.metadata_response @"$S16resilient_struct4SizeVMa"([[INT]] 319)
+// CHECK: [[SIZE_METADATA:%.*]] = extractvalue %swift.metadata_response [[T0]], 0
+// CHECK: [[T0:%.*]] = bitcast %swift.type* [[SIZE_METADATA]] to i8***
+// CHECK: [[T1:%.*]] = getelementptr inbounds i8**, i8*** [[T0]], [[INT]] -1
+// CHECK: [[SIZE_VWT:%.*]] = load i8**, i8*** [[T1]],
+// CHECK: [[SIZE_LAYOUT_1:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
 // CHECK: [[FIELD_1:%.*]] = getelementptr inbounds i8**, i8*** [[FIELDS_ADDR]], i32 0
-// CHECK: store i8** [[SIZE_AND_ALIGNMENT:%.*]], i8*** [[FIELD_1]]
+// CHECK: store i8** [[SIZE_LAYOUT_1:%.*]], i8*** [[FIELD_1]]
 
 // public let ss: (Size, Size)
 
-// CHECK: call swiftcc %swift.metadata_response @swift_getTupleTypeMetadata2([[INT]] 319,
+// CHECK: [[SIZE_LAYOUT_2:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
+// CHECK: [[SIZE_LAYOUT_3:%.*]] = getelementptr inbounds i8*, i8** [[SIZE_VWT]], i32 8
+// CHECK: call swiftcc [[INT]] @swift_getTupleTypeLayout2(%swift.full_type_layout* [[TUPLE_LAYOUT]], i8** [[SIZE_LAYOUT_2]], i8** [[SIZE_LAYOUT_3]])
+// CHECK: [[T0:%.*]] = bitcast %swift.full_type_layout* [[TUPLE_LAYOUT]] to i8**
 // CHECK: [[FIELD_2:%.*]] = getelementptr inbounds i8**, i8*** [[FIELDS_ADDR]], i32 1
-// CHECK: store i8** [[SIZE_AND_ALIGNMENT:%.*]], i8*** [[FIELD_2]]
+// CHECK: store i8** [[T0]], i8*** [[FIELD_2]]
 
 // Fixed-layout aggregate -- we can reference a static value witness table
 // public let n: Int