IRGen: Remove initializeBufferWithTakeOfBuffer in favor of memcpy

And update the existential container's initializeWithTake implementation
in the runtime. After only allowing bitwise takable values in the
inline buffer we can use memcpy to move existential container values.

rdar://31414907
SR-343

Author: aschwaighofer

include/swift/ABI/ValueWitness.def

@@ -172,14 +172,6 @@ FUNCTION_VALUE_WITNESS(assignWithTake,
                        MUTABLE_VALUE_TYPE,
                        (MUTABLE_VALUE_TYPE, MUTABLE_VALUE_TYPE, TYPE_TYPE))
 
-///   T *(*initializeBufferWithTakeOfBuffer)(B *dest, B *src, M *self);
-/// Given an invalid buffer, initialize it by taking the value out of
-/// the source buffer.
-FUNCTION_VALUE_WITNESS(initializeBufferWithTakeOfBuffer,
-                       InitializeBufferWithTakeOfBuffer,
-                       MUTABLE_VALUE_TYPE,
-                       (MUTABLE_BUFFER_TYPE, MUTABLE_BUFFER_TYPE, TYPE_TYPE))
-
 /// unsigned (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases)
 /// Given an instance of valid single payload enum with a payload of this
 /// witness table's type (e.g Optional<ThisType>) , get the tag of the enum.

lib/IRGen/GenExistential.cpp

@@ -343,16 +343,11 @@ class OpaqueExistentialTypeInfo final :
   void initializeWithTake(IRGenFunction &IGF, Address dest, Address src,
                           SILType T, bool isOutlined) const override {
     if (isOutlined) {
-      llvm::Value *metadata = copyType(IGF, dest, src);
-
-      auto layout = getLayout();
-
-      // Project down to the buffers and ask the witnesses to do a
-      // take-initialize.
-      Address srcBuffer = layout.projectExistentialBuffer(IGF, src);
-      Address destBuffer = layout.projectExistentialBuffer(IGF, dest);
-      emitInitializeBufferWithTakeOfBufferCall(IGF, metadata, destBuffer,
-                                               srcBuffer);
+      // memcpy the existential container. This is safe because: either the
+      // value is stored inline and is therefore by convention bitwise takable
+      // or the value is stored in a reference counted heap buffer, in which
+      // case a memcpy of the reference is also correct.
+      IGF.emitMemCpy(dest, src, getLayout().getSize(IGF.IGM));
     } else {
       // Create an outlined function to avoid explosion
       OutliningMetadataCollector collector(IGF);

lib/IRGen/GenOpaque.cpp

@@ -66,9 +66,7 @@ static llvm::Type *createWitnessType(IRGenModule &IGM, ValueWitness index) {
   }
 
   // T *(*initializeBufferWithCopyOfBuffer)(B *dest, B *src, M *self);
-  // T *(*initializeBufferWithTakeOfBuffer)(B *dest, B *src, M *self);
-  case ValueWitness::InitializeBufferWithCopyOfBuffer:
-  case ValueWitness::InitializeBufferWithTakeOfBuffer: {
+  case ValueWitness::InitializeBufferWithCopyOfBuffer: {
     llvm::Type *bufPtrTy = IGM.getFixedBufferTy()->getPointerTo(0);
     llvm::Type *args[] = { bufPtrTy, bufPtrTy, IGM.TypeMetadataPtrTy };
     return llvm::FunctionType::get(IGM.OpaquePtrTy, args, /*isVarArg*/ false);
@@ -212,7 +210,6 @@ static llvm::AttributeList getValueWitnessAttrs(IRGenModule &IGM,
   // These have two arguments and they don't alias each other.
   case ValueWitness::AssignWithTake:
   case ValueWitness::InitializeBufferWithCopyOfBuffer:
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
   case ValueWitness::InitializeWithCopy:
   case ValueWitness::InitializeWithTake:
     return attrs.addAttribute(ctx, 1, llvm::Attribute::NoAlias)
@@ -259,8 +256,6 @@ static StringRef getValueWitnessLabel(ValueWitness index) {
     return "initializeWithCopy";
   case ValueWitness::InitializeWithTake:
     return "initializeWithTake";
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
-    return "initializeBufferWithTakeOfBuffer";
   case ValueWitness::Size:
     return "size";
   case ValueWitness::Flags:
@@ -417,31 +412,6 @@ irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
   return call;
 }
 
-llvm::Value *
-irgen::emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                SILType T,
-                                                Address destBuffer,
-                                                Address srcBuffer) {
-  auto metadata = IGF.emitTypeMetadataRefForLayout(T);
-  return emitInitializeBufferWithTakeOfBufferCall(IGF, metadata,
-                                                  destBuffer, srcBuffer);
-}
-
-/// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-llvm::Value *
-irgen::emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                llvm::Value *metadata,
-                                                Address destBuffer,
-                                                Address srcBuffer) {
-  auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
-                             ValueWitness::InitializeBufferWithTakeOfBuffer);
-  llvm::CallInst *call =
-    IGF.Builder.CreateCall(copyFn,
-      {destBuffer.getAddress(), srcBuffer.getAddress(), metadata});
-
-  return call;
-}
-
 /// Emit a dynamic alloca call to allocate enough memory to hold an object of
 /// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
 /// false.

lib/IRGen/GenOpaque.h

@@ -58,18 +58,6 @@ namespace irgen {
                                                         Address destBuffer,
                                                         Address srcBuffer);
 
-  /// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-  llvm::Value *emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                        llvm::Value *metadata,
-                                                        Address destBuffer,
-                                                        Address srcBuffer);
-
-  /// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-  llvm::Value *emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                        SILType T,
-                                                        Address destBuffer,
-                                                        Address srcBuffer);
-
   /// Emit a call to do an 'initializeWithCopy' operation.
   void emitInitializeWithCopyCall(IRGenFunction &IGF,
                                   SILType T,

lib/IRGen/GenRecord.h

@@ -323,22 +323,6 @@ class RecordTypeInfo<Impl, Base, FieldImpl,
 public:
   using super::getStorageType;
 
-  Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                           Address destBuffer,
-                                           Address srcBuffer,
-                                           SILType type) const override {
-    if (auto field = getUniqueNonEmptyField()) {
-      auto &fieldTI = field->getTypeInfo();
-      Address fieldResult =
-        fieldTI.initializeBufferWithTakeOfBuffer(IGF, destBuffer, srcBuffer,
-                                                 field->getType(IGF.IGM, type));
-      return IGF.Builder.CreateElementBitCast(fieldResult, getStorageType());
-    } else {
-      return super::initializeBufferWithTakeOfBuffer(IGF, destBuffer,
-                                                     srcBuffer, type);
-    }
-  }
-
   Address initializeBufferWithCopyOfBuffer(IRGenFunction &IGF,
                                            Address destBuffer,
                                            Address srcBuffer,

lib/IRGen/GenValueWitness.cpp

@@ -56,7 +56,6 @@ const char *irgen::getValueWitnessName(ValueWitness witness) {
   CASE(InitializeBufferWithCopyOfBuffer)
   CASE(InitializeWithCopy)
   CASE(InitializeWithTake)
-  CASE(InitializeBufferWithTakeOfBuffer)
   CASE(StoreExtraInhabitant)
   CASE(GetExtraInhabitantIndex)
   CASE(GetEnumTag)
@@ -331,47 +330,6 @@ static Address emitDefaultInitializeBufferWithCopyOfBuffer(
   }
 }
 
-/// Emit an 'initializeBufferWithTakeOfBuffer' operation.
-/// Returns the address of the destination object.
-static Address
-emitDefaultInitializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                            Address destBuffer,
-                                            Address srcBuffer,
-                                            SILType T,
-                                            const TypeInfo &type,
-                                            FixedPacking packing) {
-  switch (packing) {
-
-  case FixedPacking::Dynamic:
-    // Special-case dynamic packing in order to thread the jumps.
-    return emitForDynamicPacking(IGF,
-                                 &emitDefaultInitializeBufferWithTakeOfBuffer,
-                                 T, type, destBuffer, srcBuffer);
-
-  case FixedPacking::OffsetZero: {
-    // Both of these allocations/projections should be no-ops.
-    Address destObject =
-      emitDefaultAllocateBuffer(IGF, destBuffer, T, type, packing);
-    Address srcObject =
-      emitDefaultProjectBuffer(IGF, srcBuffer, T, type, packing);
-    type.initializeWithTake(IGF, destObject, srcObject, T, true);
-    return destObject;
-  }
-
-  case FixedPacking::Allocate: {
-    // Just copy the out-of-line storage pointers.
-    srcBuffer = IGF.Builder.CreateBitCast(
-        srcBuffer, IGF.IGM.RefCountedPtrTy->getPointerTo());
-    llvm::Value *addr = IGF.Builder.CreateLoad(srcBuffer);
-    destBuffer = IGF.Builder.CreateBitCast(
-        destBuffer, IGF.IGM.RefCountedPtrTy->getPointerTo());
-    IGF.Builder.CreateStore(addr, destBuffer);
-    return emitDefaultProjectBuffer(IGF, destBuffer, T, type, packing);
-  }
-  }
-  llvm_unreachable("bad fixed packing");
-}
-
 // Metaprogram some of the common boilerplate here:
 //   - the default implementation in TypeInfo
 //   - the value-witness emitter which tries to avoid some dynamic
@@ -392,8 +350,6 @@ static Address emit##TITLE(IRGenFunction &IGF, Address dest, Address src, \
 }
 DEFINE_BINARY_BUFFER_OP(initializeBufferWithCopyOfBuffer,
                         InitializeBufferWithCopyOfBuffer)
-DEFINE_BINARY_BUFFER_OP(initializeBufferWithTakeOfBuffer,
-                        InitializeBufferWithTakeOfBuffer)
 #undef DEFINE_BINARY_BUFFER_OP
 
 
@@ -532,19 +488,6 @@ static void buildValueWitnessFunction(IRGenModule &IGM,
     return;
   }
 
-  case ValueWitness::InitializeBufferWithTakeOfBuffer: {
-    Address dest = getArgAsBuffer(IGF, argv, "dest");
-    Address src = getArgAsBuffer(IGF, argv, "src");
-    getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);
-
-    Address result =
-      emitInitializeBufferWithTakeOfBuffer(IGF, dest, src, concreteType,
-                                           type, packing);
-    result = IGF.Builder.CreateBitCast(result, IGF.IGM.OpaquePtrTy);
-    IGF.Builder.CreateRet(result.getAddress());
-    return;
-  }
-
   case ValueWitness::InitializeWithCopy: {
     Address dest = getArgAs(IGF, argv, type, "dest");
     Address src = getArgAs(IGF, argv, type, "src");
@@ -805,38 +748,6 @@ static llvm::Constant *getMemCpyFunction(IRGenModule &IGM,
   });
 }
 
-/// Return a function which takes two buffer arguments, copies
-/// a pointer from the second to the first, and returns the pointer.
-static llvm::Constant *
-getCopyOutOfLineBoxPointerFunction(IRGenModule &IGM,
-                                   const FixedTypeInfo &fixedTI) {
-  llvm::Type *argTys[] = { IGM.Int8PtrPtrTy, IGM.Int8PtrPtrTy,
-                           IGM.TypeMetadataPtrTy };
-  llvm::SmallString<40> name;
-  {
-    llvm::raw_svector_ostream nameStream(name);
-    nameStream << "__swift_copy_outline_existential_box_pointer";
-    nameStream << fixedTI.getFixedAlignment().getValue();
-  }
-  return IGM.getOrCreateHelperFunction(
-      name, IGM.Int8PtrTy, argTys, [&](IRGenFunction &IGF) {
-        auto it = IGF.CurFn->arg_begin();
-        Address dest(&*it++, IGM.getPointerAlignment());
-        Address src(&*it++, IGM.getPointerAlignment());
-        auto *ptr = IGF.Builder.CreateLoad(src);
-        IGF.Builder.CreateStore(ptr, dest);
-        auto *alignmentMask = fixedTI.getStaticAlignmentMask(IGM);
-        auto *heapHeaderSize = llvm::ConstantInt::get(
-            IGM.SizeTy, IGM.RefCountedStructSize.getValue());
-        auto *startOffset = IGF.Builder.CreateAnd(
-            IGF.Builder.CreateAdd(heapHeaderSize, alignmentMask),
-            IGF.Builder.CreateNot(alignmentMask));
-        auto *objectAddr =
-            IGF.emitByteOffsetGEP(ptr, startOffset, IGM.Int8Ty);
-        IGF.Builder.CreateRet(objectAddr);
-      });
-}
-
 /// Find a witness to the fact that a type is a value type.
 /// Always adds an i8*.
 static void addValueWitness(IRGenModule &IGM,
@@ -870,17 +781,6 @@ static void addValueWitness(IRGenModule &IGM,
     }
     goto standard;
 
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
-    if (packing == FixedPacking::Allocate) {
-      return addFunction(getCopyOutOfLineBoxPointerFunction(
-          IGM, cast<FixedTypeInfo>(concreteTI)));
-    } else
-      if (packing == FixedPacking::OffsetZero &&
-               concreteTI.isBitwiseTakable(ResilienceExpansion::Maximal)) {
-      return addFunction(getMemCpyFunction(IGM, concreteTI));
-    }
-    goto standard;
-
   case ValueWitness::InitializeWithTake:
     if (concreteTI.isBitwiseTakable(ResilienceExpansion::Maximal)) {
       return addFunction(getMemCpyFunction(IGM, concreteTI));

lib/IRGen/IRGenMangler.cpp

@@ -45,7 +45,6 @@ std::string IRGenMangler::mangleValueWitness(Type type, ValueWitness witness) {
     GET_MANGLING(AssignWithCopy) \
     GET_MANGLING(InitializeWithTake) \
     GET_MANGLING(AssignWithTake) \
-    GET_MANGLING(InitializeBufferWithTakeOfBuffer) \
     GET_MANGLING(GetEnumTagSinglePayload) \
     GET_MANGLING(StoreEnumTagSinglePayload) \
     GET_MANGLING(StoreExtraInhabitant) \

lib/IRGen/ResilientTypeInfo.h

@@ -88,13 +88,6 @@ class ResilientTypeInfo : public WitnessSizedTypeInfo<Impl> {
     return this->getAddressForPointer(addr);
   }
 
-  Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                   Address dest, Address src,
-                                   SILType T) const override {
-    auto addr = emitInitializeBufferWithTakeOfBufferCall(IGF, T, dest, src);
-    return this->getAddressForPointer(addr);
-  }
-
   void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src,
                           SILType T, bool isOutlined) const override {
     emitInitializeWithCopyCall(IGF, T, dest, src);

lib/IRGen/TypeInfo.h

@@ -370,23 +370,6 @@ class TypeInfo {
                                                    Address srcBuffer,
                                                    SILType T) const;
 
-  /// Perform a take-initialization from the given fixed-size buffer
-  /// into an uninitialized fixed-size buffer, allocating the buffer if
-  /// necessary and deallocating the destination buffer.  Returns the
-  /// address of the value inside the destination buffer.
-  ///
-  /// This is equivalent to:
-  ///   auto srcAddress = projectBuffer(IGF, srcBuffer, T);
-  ///   initializeBufferWithTake(IGF, destBuffer, srcAddress, T);
-  ///   deallocateBuffer(IGF, srcBuffer, T);
-  /// but may be able to re-use the buffer from the source buffer, and may
-  /// be more efficient for dynamic types, since it uses a single
-  /// value witness call.
-  virtual Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                                   Address destBuffer,
-                                                   Address srcBuffer,
-                                                   SILType T) const;
-
   /// Take-initialize an address from a parameter explosion.
   virtual void initializeFromParams(IRGenFunction &IGF, Explosion &params,
                                     Address src, SILType T,

stdlib/public/runtime/ExistentialMetadataImpl.h

@@ -107,20 +107,9 @@ struct LLVM_LIBRARY_VISIBILITY OpaqueExistentialBoxBase
   static Container *initializeWithTake(Container *dest, Container *src,
                                        A... args) {
     src->copyTypeInto(dest, args...);
-    auto *type = src->getType();
-    auto *vwt = type->getValueWitnesses();
-
-    if (vwt->isValueInline()) {
-      auto *destValue =
-          reinterpret_cast<OpaqueValue *>(dest->getBuffer(args...));
-      auto *srcValue =
-          reinterpret_cast<OpaqueValue *>(src->getBuffer(args...));
-
-      type->vw_initializeWithTake(destValue, srcValue);
-    } else {
-      // initWithTake of the reference to the cow box.
-      copyReference(dest, src, Dest::Init, Source::Take, args...);
-    }
+    auto from = src->getBuffer(args...);
+    auto to = dest->getBuffer(args...);
+    memcpy(to, from, sizeof(ValueBuffer));
     return dest;
   }