Merge pull request #16775 from aschwaighofer/remove_initBufferWithTake

IRGen: Remove initializeBufferWithTakeOfBuffer in favor of memcpy

Author: aschwaighofer

include/swift/ABI/ValueWitness.def

@@ -172,14 +172,6 @@ FUNCTION_VALUE_WITNESS(assignWithTake,
                        MUTABLE_VALUE_TYPE,
                        (MUTABLE_VALUE_TYPE, MUTABLE_VALUE_TYPE, TYPE_TYPE))
 
-///   T *(*initializeBufferWithTakeOfBuffer)(B *dest, B *src, M *self);
-/// Given an invalid buffer, initialize it by taking the value out of
-/// the source buffer.
-FUNCTION_VALUE_WITNESS(initializeBufferWithTakeOfBuffer,
-                       InitializeBufferWithTakeOfBuffer,
-                       MUTABLE_VALUE_TYPE,
-                       (MUTABLE_BUFFER_TYPE, MUTABLE_BUFFER_TYPE, TYPE_TYPE))
-
 /// unsigned (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases)
 /// Given an instance of valid single payload enum with a payload of this
 /// witness table's type (e.g Optional<ThisType>) , get the tag of the enum.

lib/IRGen/GenExistential.cpp

@@ -343,16 +343,11 @@ class OpaqueExistentialTypeInfo final :
   void initializeWithTake(IRGenFunction &IGF, Address dest, Address src,
                           SILType T, bool isOutlined) const override {
     if (isOutlined) {
-      llvm::Value *metadata = copyType(IGF, dest, src);
-
-      auto layout = getLayout();
-
-      // Project down to the buffers and ask the witnesses to do a
-      // take-initialize.
-      Address srcBuffer = layout.projectExistentialBuffer(IGF, src);
-      Address destBuffer = layout.projectExistentialBuffer(IGF, dest);
-      emitInitializeBufferWithTakeOfBufferCall(IGF, metadata, destBuffer,
-                                               srcBuffer);
+      // memcpy the existential container. This is safe because: either the
+      // value is stored inline and is therefore by convention bitwise takable
+      // or the value is stored in a reference counted heap buffer, in which
+      // case a memcpy of the reference is also correct.
+      IGF.emitMemCpy(dest, src, getLayout().getSize(IGF.IGM));
     } else {
       // Create an outlined function to avoid explosion
       OutliningMetadataCollector collector(IGF);

lib/IRGen/GenOpaque.cpp

@@ -66,9 +66,7 @@ static llvm::Type *createWitnessType(IRGenModule &IGM, ValueWitness index) {
   }
 
   // T *(*initializeBufferWithCopyOfBuffer)(B *dest, B *src, M *self);
-  // T *(*initializeBufferWithTakeOfBuffer)(B *dest, B *src, M *self);
-  case ValueWitness::InitializeBufferWithCopyOfBuffer:
-  case ValueWitness::InitializeBufferWithTakeOfBuffer: {
+  case ValueWitness::InitializeBufferWithCopyOfBuffer: {
     llvm::Type *bufPtrTy = IGM.getFixedBufferTy()->getPointerTo(0);
     llvm::Type *args[] = { bufPtrTy, bufPtrTy, IGM.TypeMetadataPtrTy };
     return llvm::FunctionType::get(IGM.OpaquePtrTy, args, /*isVarArg*/ false);
@@ -212,7 +210,6 @@ static llvm::AttributeList getValueWitnessAttrs(IRGenModule &IGM,
   // These have two arguments and they don't alias each other.
   case ValueWitness::AssignWithTake:
   case ValueWitness::InitializeBufferWithCopyOfBuffer:
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
   case ValueWitness::InitializeWithCopy:
   case ValueWitness::InitializeWithTake:
     return attrs.addAttribute(ctx, 1, llvm::Attribute::NoAlias)
@@ -259,8 +256,6 @@ static StringRef getValueWitnessLabel(ValueWitness index) {
     return "initializeWithCopy";
   case ValueWitness::InitializeWithTake:
     return "initializeWithTake";
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
-    return "initializeBufferWithTakeOfBuffer";
   case ValueWitness::Size:
     return "size";
   case ValueWitness::Flags:
@@ -417,31 +412,6 @@ irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
   return call;
 }
 
-llvm::Value *
-irgen::emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                SILType T,
-                                                Address destBuffer,
-                                                Address srcBuffer) {
-  auto metadata = IGF.emitTypeMetadataRefForLayout(T);
-  return emitInitializeBufferWithTakeOfBufferCall(IGF, metadata,
-                                                  destBuffer, srcBuffer);
-}
-
-/// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-llvm::Value *
-irgen::emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                llvm::Value *metadata,
-                                                Address destBuffer,
-                                                Address srcBuffer) {
-  auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
-                             ValueWitness::InitializeBufferWithTakeOfBuffer);
-  llvm::CallInst *call =
-    IGF.Builder.CreateCall(copyFn,
-      {destBuffer.getAddress(), srcBuffer.getAddress(), metadata});
-
-  return call;
-}
-
 /// Emit a dynamic alloca call to allocate enough memory to hold an object of
 /// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
 /// false.

lib/IRGen/GenOpaque.h

@@ -58,18 +58,6 @@ namespace irgen {
                                                         Address destBuffer,
                                                         Address srcBuffer);
 
-  /// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-  llvm::Value *emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                        llvm::Value *metadata,
-                                                        Address destBuffer,
-                                                        Address srcBuffer);
-
-  /// Emit a call to do an 'initializeBufferWithTakeOfBuffer' operation.
-  llvm::Value *emitInitializeBufferWithTakeOfBufferCall(IRGenFunction &IGF,
-                                                        SILType T,
-                                                        Address destBuffer,
-                                                        Address srcBuffer);
-
   /// Emit a call to do an 'initializeWithCopy' operation.
   void emitInitializeWithCopyCall(IRGenFunction &IGF,
                                   SILType T,

lib/IRGen/GenRecord.h

@@ -323,22 +323,6 @@ class RecordTypeInfo<Impl, Base, FieldImpl,
 public:
   using super::getStorageType;
 
-  Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                           Address destBuffer,
-                                           Address srcBuffer,
-                                           SILType type) const override {
-    if (auto field = getUniqueNonEmptyField()) {
-      auto &fieldTI = field->getTypeInfo();
-      Address fieldResult =
-        fieldTI.initializeBufferWithTakeOfBuffer(IGF, destBuffer, srcBuffer,
-                                                 field->getType(IGF.IGM, type));
-      return IGF.Builder.CreateElementBitCast(fieldResult, getStorageType());
-    } else {
-      return super::initializeBufferWithTakeOfBuffer(IGF, destBuffer,
-                                                     srcBuffer, type);
-    }
-  }
-
   Address initializeBufferWithCopyOfBuffer(IRGenFunction &IGF,
                                            Address destBuffer,
                                            Address srcBuffer,

lib/IRGen/GenValueWitness.cpp

@@ -56,7 +56,6 @@ const char *irgen::getValueWitnessName(ValueWitness witness) {
   CASE(InitializeBufferWithCopyOfBuffer)
   CASE(InitializeWithCopy)
   CASE(InitializeWithTake)
-  CASE(InitializeBufferWithTakeOfBuffer)
   CASE(StoreExtraInhabitant)
   CASE(GetExtraInhabitantIndex)
   CASE(GetEnumTag)
@@ -331,47 +330,6 @@ static Address emitDefaultInitializeBufferWithCopyOfBuffer(
   }
 }
 
-/// Emit an 'initializeBufferWithTakeOfBuffer' operation.
-/// Returns the address of the destination object.
-static Address
-emitDefaultInitializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                            Address destBuffer,
-                                            Address srcBuffer,
-                                            SILType T,
-                                            const TypeInfo &type,
-                                            FixedPacking packing) {
-  switch (packing) {
-
-  case FixedPacking::Dynamic:
-    // Special-case dynamic packing in order to thread the jumps.
-    return emitForDynamicPacking(IGF,
-                                 &emitDefaultInitializeBufferWithTakeOfBuffer,
-                                 T, type, destBuffer, srcBuffer);
-
-  case FixedPacking::OffsetZero: {
-    // Both of these allocations/projections should be no-ops.
-    Address destObject =
-      emitDefaultAllocateBuffer(IGF, destBuffer, T, type, packing);
-    Address srcObject =
-      emitDefaultProjectBuffer(IGF, srcBuffer, T, type, packing);
-    type.initializeWithTake(IGF, destObject, srcObject, T, true);
-    return destObject;
-  }
-
-  case FixedPacking::Allocate: {
-    // Just copy the out-of-line storage pointers.
-    srcBuffer = IGF.Builder.CreateBitCast(
-        srcBuffer, IGF.IGM.RefCountedPtrTy->getPointerTo());
-    llvm::Value *addr = IGF.Builder.CreateLoad(srcBuffer);
-    destBuffer = IGF.Builder.CreateBitCast(
-        destBuffer, IGF.IGM.RefCountedPtrTy->getPointerTo());
-    IGF.Builder.CreateStore(addr, destBuffer);
-    return emitDefaultProjectBuffer(IGF, destBuffer, T, type, packing);
-  }
-  }
-  llvm_unreachable("bad fixed packing");
-}
-
 // Metaprogram some of the common boilerplate here:
 //   - the default implementation in TypeInfo
 //   - the value-witness emitter which tries to avoid some dynamic
@@ -392,8 +350,6 @@ static Address emit##TITLE(IRGenFunction &IGF, Address dest, Address src, \
 }
 DEFINE_BINARY_BUFFER_OP(initializeBufferWithCopyOfBuffer,
                         InitializeBufferWithCopyOfBuffer)
-DEFINE_BINARY_BUFFER_OP(initializeBufferWithTakeOfBuffer,
-                        InitializeBufferWithTakeOfBuffer)
 #undef DEFINE_BINARY_BUFFER_OP
 
 
@@ -532,19 +488,6 @@ static void buildValueWitnessFunction(IRGenModule &IGM,
     return;
   }
 
-  case ValueWitness::InitializeBufferWithTakeOfBuffer: {
-    Address dest = getArgAsBuffer(IGF, argv, "dest");
-    Address src = getArgAsBuffer(IGF, argv, "src");
-    getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);
-
-    Address result =
-      emitInitializeBufferWithTakeOfBuffer(IGF, dest, src, concreteType,
-                                           type, packing);
-    result = IGF.Builder.CreateBitCast(result, IGF.IGM.OpaquePtrTy);
-    IGF.Builder.CreateRet(result.getAddress());
-    return;
-  }
-
   case ValueWitness::InitializeWithCopy: {
     Address dest = getArgAs(IGF, argv, type, "dest");
     Address src = getArgAs(IGF, argv, type, "src");
@@ -805,38 +748,6 @@ static llvm::Constant *getMemCpyFunction(IRGenModule &IGM,
   });
 }
 
-/// Return a function which takes two buffer arguments, copies
-/// a pointer from the second to the first, and returns the pointer.
-static llvm::Constant *
-getCopyOutOfLineBoxPointerFunction(IRGenModule &IGM,
-                                   const FixedTypeInfo &fixedTI) {
-  llvm::Type *argTys[] = { IGM.Int8PtrPtrTy, IGM.Int8PtrPtrTy,
-                           IGM.TypeMetadataPtrTy };
-  llvm::SmallString<40> name;
-  {
-    llvm::raw_svector_ostream nameStream(name);
-    nameStream << "__swift_copy_outline_existential_box_pointer";
-    nameStream << fixedTI.getFixedAlignment().getValue();
-  }
-  return IGM.getOrCreateHelperFunction(
-      name, IGM.Int8PtrTy, argTys, [&](IRGenFunction &IGF) {
-        auto it = IGF.CurFn->arg_begin();
-        Address dest(&*it++, IGM.getPointerAlignment());
-        Address src(&*it++, IGM.getPointerAlignment());
-        auto *ptr = IGF.Builder.CreateLoad(src);
-        IGF.Builder.CreateStore(ptr, dest);
-        auto *alignmentMask = fixedTI.getStaticAlignmentMask(IGM);
-        auto *heapHeaderSize = llvm::ConstantInt::get(
-            IGM.SizeTy, IGM.RefCountedStructSize.getValue());
-        auto *startOffset = IGF.Builder.CreateAnd(
-            IGF.Builder.CreateAdd(heapHeaderSize, alignmentMask),
-            IGF.Builder.CreateNot(alignmentMask));
-        auto *objectAddr =
-            IGF.emitByteOffsetGEP(ptr, startOffset, IGM.Int8Ty);
-        IGF.Builder.CreateRet(objectAddr);
-      });
-}
-
 /// Find a witness to the fact that a type is a value type.
 /// Always adds an i8*.
 static void addValueWitness(IRGenModule &IGM,
@@ -870,17 +781,6 @@ static void addValueWitness(IRGenModule &IGM,
     }
     goto standard;
 
-  case ValueWitness::InitializeBufferWithTakeOfBuffer:
-    if (packing == FixedPacking::Allocate) {
-      return addFunction(getCopyOutOfLineBoxPointerFunction(
-          IGM, cast<FixedTypeInfo>(concreteTI)));
-    } else
-      if (packing == FixedPacking::OffsetZero &&
-               concreteTI.isBitwiseTakable(ResilienceExpansion::Maximal)) {
-      return addFunction(getMemCpyFunction(IGM, concreteTI));
-    }
-    goto standard;
-
   case ValueWitness::InitializeWithTake:
     if (concreteTI.isBitwiseTakable(ResilienceExpansion::Maximal)) {
       return addFunction(getMemCpyFunction(IGM, concreteTI));

lib/IRGen/IRGenMangler.cpp

@@ -45,7 +45,6 @@ std::string IRGenMangler::mangleValueWitness(Type type, ValueWitness witness) {
     GET_MANGLING(AssignWithCopy) \
     GET_MANGLING(InitializeWithTake) \
     GET_MANGLING(AssignWithTake) \
-    GET_MANGLING(InitializeBufferWithTakeOfBuffer) \
     GET_MANGLING(GetEnumTagSinglePayload) \
     GET_MANGLING(StoreEnumTagSinglePayload) \
     GET_MANGLING(StoreExtraInhabitant) \

lib/IRGen/ResilientTypeInfo.h

@@ -88,13 +88,6 @@ class ResilientTypeInfo : public WitnessSizedTypeInfo<Impl> {
     return this->getAddressForPointer(addr);
   }
 
-  Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                   Address dest, Address src,
-                                   SILType T) const override {
-    auto addr = emitInitializeBufferWithTakeOfBufferCall(IGF, T, dest, src);
-    return this->getAddressForPointer(addr);
-  }
-
   void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src,
                           SILType T, bool isOutlined) const override {
     emitInitializeWithCopyCall(IGF, T, dest, src);

lib/IRGen/TypeInfo.h

@@ -370,23 +370,6 @@ class TypeInfo {
                                                    Address srcBuffer,
                                                    SILType T) const;
 
-  /// Perform a take-initialization from the given fixed-size buffer
-  /// into an uninitialized fixed-size buffer, allocating the buffer if
-  /// necessary and deallocating the destination buffer.  Returns the
-  /// address of the value inside the destination buffer.
-  ///
-  /// This is equivalent to:
-  ///   auto srcAddress = projectBuffer(IGF, srcBuffer, T);
-  ///   initializeBufferWithTake(IGF, destBuffer, srcAddress, T);
-  ///   deallocateBuffer(IGF, srcBuffer, T);
-  /// but may be able to re-use the buffer from the source buffer, and may
-  /// be more efficient for dynamic types, since it uses a single
-  /// value witness call.
-  virtual Address initializeBufferWithTakeOfBuffer(IRGenFunction &IGF,
-                                                   Address destBuffer,
-                                                   Address srcBuffer,
-                                                   SILType T) const;
-
   /// Take-initialize an address from a parameter explosion.
   virtual void initializeFromParams(IRGenFunction &IGF, Explosion &params,
                                     Address src, SILType T,

stdlib/public/runtime/ExistentialMetadataImpl.h

@@ -107,20 +107,9 @@ struct LLVM_LIBRARY_VISIBILITY OpaqueExistentialBoxBase
   static Container *initializeWithTake(Container *dest, Container *src,
                                        A... args) {
     src->copyTypeInto(dest, args...);
-    auto *type = src->getType();
-    auto *vwt = type->getValueWitnesses();
-
-    if (vwt->isValueInline()) {
-      auto *destValue =
-          reinterpret_cast<OpaqueValue *>(dest->getBuffer(args...));
-      auto *srcValue =
-          reinterpret_cast<OpaqueValue *>(src->getBuffer(args...));
-
-      type->vw_initializeWithTake(destValue, srcValue);
-    } else {
-      // initWithTake of the reference to the cow box.
-      copyReference(dest, src, Dest::Init, Source::Take, args...);
-    }
+    auto from = src->getBuffer(args...);
+    auto to = dest->getBuffer(args...);
+    memcpy(to, from, sizeof(ValueBuffer));
     return dest;
   }