[6.0] IRGen: Set a "not bitwise borrowable" bit in value witnesses for @_rawLayout types.

include/swift/ABI/MetadataValues.h

@@ -168,17 +168,18 @@ class TargetValueWitnessFlags {
   // flags for the struct. (The "non-inline" and "has-extra-inhabitants" bits
   // still require additional fixup.)
   enum : uint32_t {
-    AlignmentMask =       0x000000FF,
-    // unused             0x0000FF00,
-    IsNonPOD =            0x00010000,
-    IsNonInline =         0x00020000,
-    // unused             0x00040000,
-    HasSpareBits =        0x00080000,
-    IsNonBitwiseTakable = 0x00100000,
-    HasEnumWitnesses =    0x00200000,
-    Incomplete =          0x00400000,
-    IsNonCopyable =       0x00800000,
-    // unused             0xFF000000,
+    AlignmentMask =          0x000000FF,
+    // unused                0x0000FF00,
+    IsNonPOD =               0x00010000,
+    IsNonInline =            0x00020000,
+    // unused                0x00040000,
+    HasSpareBits =           0x00080000,
+    IsNonBitwiseTakable =    0x00100000,
+    HasEnumWitnesses =       0x00200000,
+    Incomplete =             0x00400000,
+    IsNonCopyable =          0x00800000,
+    IsNonBitwiseBorrowable = 0x01000000,
+    // unused                0xFE000000,
   };
 
   static constexpr const uint32_t MaxNumExtraInhabitants = 0x7FFFFFFF;
@@ -243,6 +244,27 @@ class TargetValueWitnessFlags {
                                    (isBT ? 0 : IsNonBitwiseTakable));
   }
 
+  /// True if values of this type can be passed by value when borrowed.
+  /// If this bit is true, then borrows of the value are independent of the
+  /// value's address, so a value can be passed in registers or memcpy'd
+  /// while borrowed. This is in contrast to Rust, for instance, where a
+  /// `&T` type is always represented as a pointer, and borrowing a
+  /// value always moves the borrowed value into memory.
+  bool isBitwiseBorrowable() const {
+    /// This bit was introduced with Swift 6; prior to the introduction of
+    /// `Atomic` and `Mutex`, a type was always bitwise-borrowable if it
+    /// was bitwise-takable. Compilers and runtimes before Swift 6 would
+    /// never set the `IsNonBitwiseBorrowable` bit in the value witness
+    /// table, but any type that sets `IsNonBitwiseTakable` is definitely
+    /// not bitwise borrowable.
+    return isBitwiseTakable()
+      && !(Data & IsNonBitwiseBorrowable);
+  }
+  constexpr TargetValueWitnessFlags withBitwiseBorrowable(bool isBB) const {
+    return TargetValueWitnessFlags((Data & ~IsNonBitwiseBorrowable) |
+                                   (isBB ? 0 : IsNonBitwiseBorrowable));
+  }
+
   /// True if values of this type can be copied.
   bool isCopyable() const { return !(Data & IsNonCopyable); }
   constexpr TargetValueWitnessFlags withCopyable(bool isCopyable) const {

lib/IRGen/GenEnum.cpp

@@ -3906,8 +3906,8 @@ namespace {
       } else if (allSingleRefcount
                  && ElementsWithNoPayload.size() <= 1) {
         CopyDestroyKind = TaggedRefcounted;
-      } else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakable &&
-                 Copyable == IsCopyable) {
+      } else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakableAndBorrowable
+                 && Copyable == IsCopyable) {
         CopyDestroyKind = BitwiseTakable;
       }
     }
@@ -6365,7 +6365,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
     ? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
   auto copyable = !theEnum->canBeCopyable()
     ? IsNotCopyable : IsCopyable;
-  auto bitwiseTakable = IsBitwiseTakable; // FIXME: will there be check here?
+  auto bitwiseTakable = IsBitwiseTakableAndBorrowable; // FIXME: will there be check here?
   bool allowFixedLayoutOptimizations = true;
   std::vector<Element> elementsWithPayload;
   std::vector<Element> elementsWithNoPayload;
@@ -6377,7 +6377,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
       payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
     copyable = copyable & payloadTI.isCopyable(ResilienceExpansion::Maximal);
     bitwiseTakable = bitwiseTakable &
-      payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
+      payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
   };
 
   if (TC.IGM.isResilient(theEnum, ResilienceExpansion::Minimal))
@@ -6825,7 +6825,8 @@ EnumImplStrategy::getFixedEnumTypeInfo(llvm::StructType *T, Size S,
                                       AlwaysFixedSize);
     break;
   case Loadable:
-    assert(isBT && "loadable enum not bitwise takable?!");
+    assert(isBT == IsBitwiseTakableAndBorrowable
+           && "loadable enum not bitwise takable?!");
     mutableTI = new LoadableEnumTypeInfo(*this, T, S, std::move(SB), A,
                                          isTriviallyDestroyable,
                                          isCopyable,
@@ -7038,7 +7039,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeFixedLayout(
   getFixedEnumTypeInfo(
       enumTy, Size(sizeWithTag), spareBits.build(), alignment,
       deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
-      payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
+      payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
       copyable);
   if (TIK >= Loadable && CopyDestroyKind == Normal) {
     computePayloadTypesAndTagType(TC.IGM, *TI, PayloadTypesAndTagType);
@@ -7073,7 +7074,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeDynamicLayout(
   return registerEnumTypeInfo(new NonFixedEnumTypeInfo(*this, enumTy,
          alignment,
          deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
-         payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
+         payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
          copyable,
          enumAccessible));
 }
@@ -7108,16 +7109,15 @@ MultiPayloadEnumImplStrategy::completeFixedLayout(TypeConverter &TC,
     ? IsNotCopyable : IsCopyable;
   auto isTriviallyDestroyable = theEnum->getValueTypeDestructor()
     ? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
-  IsBitwiseTakable_t isBT = IsBitwiseTakable;
+  IsBitwiseTakable_t isBT = IsBitwiseTakableAndBorrowable;
   PayloadSize = 0;
   for (auto &elt : ElementsWithPayload) {
     auto &fixedPayloadTI = cast<FixedTypeInfo>(*elt.ti);
     if (fixedPayloadTI.getFixedAlignment() > worstAlignment)
       worstAlignment = fixedPayloadTI.getFixedAlignment();
     if (!fixedPayloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal))
       isTriviallyDestroyable = IsNotTriviallyDestroyable;
-    if (!fixedPayloadTI.isBitwiseTakable(ResilienceExpansion::Maximal))
-      isBT = IsNotBitwiseTakable;
+    isBT &= fixedPayloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
 
     unsigned payloadBytes = fixedPayloadTI.getFixedSize().getValue();
     unsigned payloadBits = fixedPayloadTI.getFixedSize().getValueInBits();
@@ -7274,12 +7274,12 @@ TypeInfo *MultiPayloadEnumImplStrategy::completeDynamicLayout(
   Alignment alignment(1);
   auto td = theEnum->getValueTypeDestructor()
     ? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
-  auto bt = IsBitwiseTakable;
+  auto bt = IsBitwiseTakableAndBorrowable;
   for (auto &element : ElementsWithPayload) {
     auto &payloadTI = *element.ti;
     alignment = std::max(alignment, payloadTI.getBestKnownAlignment());
     td &= payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
-    bt &= payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
+    bt &= payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
   }
 
   applyLayoutAttributes(TC.IGM, theEnum, /*fixed*/false, alignment);

lib/IRGen/GenExistential.cpp

@@ -897,11 +897,16 @@ class OpaqueExistentialTypeInfo final :
 
   OpaqueExistentialTypeInfo(ArrayRef<const ProtocolDecl *> protocols,
                             llvm::Type *ty, Size size,
+                            IsCopyable_t copyable,
                             SpareBitVector &&spareBits,
                             Alignment align)
     : super(protocols, ty, size,
             std::move(spareBits), align,
-            IsNotTriviallyDestroyable, IsBitwiseTakable, IsCopyable,
+            IsNotTriviallyDestroyable,
+            // Copyable existentials are bitwise-takable and borrowable.
+            // Noncopyable existentials are bitwise-takable only.
+            copyable ? IsBitwiseTakableAndBorrowable : IsBitwiseTakableOnly,
+            copyable,
             IsFixedSize) {}
 
 public:
@@ -1645,14 +1650,19 @@ static const TypeInfo *createExistentialTypeInfo(IRGenModule &IGM, CanType T) {
   OpaqueExistentialLayout opaque(protosWithWitnessTables.size());
   Alignment align = opaque.getAlignment(IGM);
   Size size = opaque.getSize(IGM);
+  IsCopyable_t copyable = T->isNoncopyable() ? IsNotCopyable : IsCopyable;
+
   // There are spare bits in the metadata pointer and witness table pointers
   // consistent with a native object reference.
-  // TODO: There are spare bits we could theoretically use in the type metadata
-  // and witness table pointers, but opaque existentials are currently address-
+  // NB: There are spare bits we could theoretically use in the type metadata
+  // and witness table pointers, but opaque existentials are address-
   // only, and we can't soundly take advantage of spare bits for in-memory
   // representations.
+  // Maybe an ABI break that made opaque existentials loadable could use those
+  // bits, though.
   auto spareBits = SpareBitVector::getConstant(size.getValueInBits(), false);
   return OpaqueExistentialTypeInfo::create(protosWithWitnessTables, type, size,
+                                           copyable,
                                            std::move(spareBits),
                                            align);
 }

lib/IRGen/GenFunc.cpp

@@ -577,7 +577,7 @@ const TypeInfo *TypeConverter::convertBlockStorageType(SILBlockStorageType *T) {
 
     size = captureOffset + fixedCapture->getFixedSize();
     pod = fixedCapture->isTriviallyDestroyable(ResilienceExpansion::Maximal);
-    bt = fixedCapture->isBitwiseTakable(ResilienceExpansion::Maximal);
+    bt = fixedCapture->getBitwiseTakable(ResilienceExpansion::Maximal);
   }
 
   llvm::Type *storageElts[] = {

lib/IRGen/GenReflection.cpp

@@ -1137,7 +1137,7 @@ class FixedTypeMetadataBuilder : public ReflectionMetadataBuilder {
 
     auto alignment = ti->getFixedAlignment().getValue();
     unsigned bitwiseTakable =
-      (ti->isBitwiseTakable(ResilienceExpansion::Minimal) == IsBitwiseTakable
+      (ti->getBitwiseTakable(ResilienceExpansion::Minimal) >= IsBitwiseTakableOnly
        ? 1 : 0);
     B.addInt32(alignment | (bitwiseTakable << 16));
 

lib/IRGen/GenType.cpp

@@ -233,7 +233,7 @@ llvm::Value *FixedTypeInfo::getIsTriviallyDestroyable(IRGenFunction &IGF, SILTyp
 }
 llvm::Value *FixedTypeInfo::getIsBitwiseTakable(IRGenFunction &IGF, SILType T) const {
   return llvm::ConstantInt::get(IGF.IGM.Int1Ty,
-                                isBitwiseTakable(ResilienceExpansion::Maximal) == IsBitwiseTakable);
+      getBitwiseTakable(ResilienceExpansion::Maximal) >= IsBitwiseTakableOnly);
 }
 llvm::Constant *FixedTypeInfo::getStaticStride(IRGenModule &IGM) const {
   return IGM.getSize(getFixedStride());

lib/IRGen/GenValueWitness.cpp

@@ -847,12 +847,22 @@ ValueWitnessFlags getValueWitnessFlags(const TypeInfo *TI, SILType concreteType,
     bool isInline = packing == FixedPacking::OffsetZero;
     bool isBitwiseTakable =
         fixedTI->isBitwiseTakable(ResilienceExpansion::Maximal);
+    bool isBitwiseBorrowable =
+        fixedTI->isBitwiseBorrowable(ResilienceExpansion::Maximal);
     assert(isBitwiseTakable || !isInline);
     flags = flags.withAlignment(fixedTI->getFixedAlignment().getValue())
                 .withPOD(fixedTI->isTriviallyDestroyable(ResilienceExpansion::Maximal))
                 .withCopyable(fixedTI->isCopyable(ResilienceExpansion::Maximal))
                 .withInlineStorage(isInline)
-                .withBitwiseTakable(isBitwiseTakable);
+                .withBitwiseTakable(isBitwiseTakable)
+                // the IsNotBitwiseBorrowable bit only needs to be set if the
+                // type is bitwise-takable but not bitwise-borrowable, since
+                // a type must be bitwise-takable to be bitwise-borrowable.
+                //
+                // Swift prior to version 6 didn't have the
+                // IsNotBitwiseBorrowable bit, so to avoid unnecessary variation
+                // in metadata output, we only set the bit when needed.
+                .withBitwiseBorrowable(!isBitwiseTakable || isBitwiseBorrowable);
   } else {
     flags = flags.withIncomplete(true);
   }
@@ -1530,12 +1540,12 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
   unsigned align = ti.getFixedAlignment().getValue();
 
   bool pod = ti.isTriviallyDestroyable(ResilienceExpansion::Maximal);
-  bool bt = ti.isBitwiseTakable(ResilienceExpansion::Maximal);
+  IsBitwiseTakable_t bt = ti.getBitwiseTakable(ResilienceExpansion::Maximal);
   unsigned numExtraInhabitants = ti.getFixedExtraInhabitantCount(*this);
 
   // Try to use common type layouts exported by the runtime.
   llvm::Constant *commonValueWitnessTable = nullptr;
-  if (pod && bt && numExtraInhabitants == 0) {
+  if (pod && bt == IsBitwiseTakableAndBorrowable && numExtraInhabitants == 0) {
     if (size == 0)
       commonValueWitnessTable =
         getAddrOfValueWitnessTable(Context.TheEmptyTupleType);
@@ -1560,7 +1570,7 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
 
   // Otherwise, see if a layout has been emitted with these characteristics
   // already.
-  FixedLayoutKey key{size, numExtraInhabitants, align, pod, bt};
+  FixedLayoutKey key{size, numExtraInhabitants, align, pod, unsigned(bt)};
 
   auto found = PrivateFixedLayouts.find(key);
   if (found != PrivateFixedLayouts.end())
@@ -1576,16 +1586,29 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
     addValueWitness(*this, witnesses, witness, packing, t, silTy, ti);
   }
 
+  auto pod_bt_string = [](bool pod, IsBitwiseTakable_t bt) -> StringRef {
+    if (pod) {
+      return "_pod";
+    }
+    switch (bt) {
+    case IsNotBitwiseTakable:
+      return "";
+    case IsBitwiseTakableOnly:
+      return "_bt_nbb";
+    case IsBitwiseTakableAndBorrowable:
+      return "_bt";
+    }
+  };
+
   auto layoutVar
     = witnesses.finishAndCreateGlobal(
         "type_layout_" + llvm::Twine(size)
                        + "_" + llvm::Twine(align)
                        + "_" + llvm::Twine::utohexstr(numExtraInhabitants)
-                       + (pod ? "_pod" :
-                          bt  ? "_bt"  : ""),
-                                      getPointerAlignment(),
-                                      /*constant*/ true,
-                                      llvm::GlobalValue::PrivateLinkage);
+                       + pod_bt_string(pod, bt),
+        getPointerAlignment(),
+        /*constant*/ true,
+        llvm::GlobalValue::PrivateLinkage);
 
   // Cast to the standard currency type for type layouts.
   auto layout = llvm::ConstantExpr::getBitCast(layoutVar, Int8PtrPtrTy);

lib/IRGen/IRGen.h

@@ -80,9 +80,15 @@ inline IsLoadable_t &operator&=(IsLoadable_t &l, IsLoadable_t r) {
   return (l = (l & r));
 }
 
-enum IsBitwiseTakable_t : bool { IsNotBitwiseTakable, IsBitwiseTakable };
+enum IsBitwiseTakable_t : uint8_t {
+  IsNotBitwiseTakable = 0,
+  // The type is bitwise-takable, but borrows are pinned to memory.
+  IsBitwiseTakableOnly = 1,
+  // The type is bitwise-takable and -borrowable.
+  IsBitwiseTakableAndBorrowable = 3,
+};
 inline IsBitwiseTakable_t operator&(IsBitwiseTakable_t l, IsBitwiseTakable_t r) {
-  return IsBitwiseTakable_t(unsigned(l) & unsigned(r));
+  return IsBitwiseTakable_t(std::min(unsigned(l), unsigned(r)));
 }
 inline IsBitwiseTakable_t &operator&=(IsBitwiseTakable_t &l, IsBitwiseTakable_t r) {
   return (l = (l & r));

lib/IRGen/IRGenModule.h

@@ -1363,7 +1363,7 @@ class IRGenModule {
     unsigned numExtraInhabitants;
     unsigned align: 16;
     unsigned pod: 1;
-    unsigned bitwiseTakable: 1;
+    unsigned bitwiseTakable: 2;
   };
   friend struct ::llvm::DenseMapInfo<swift::irgen::IRGenModule::FixedLayoutKey>;
   llvm::DenseMap<FixedLayoutKey, llvm::Constant *> PrivateFixedLayouts;

lib/IRGen/LoadableTypeInfo.h

@@ -64,8 +64,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
                    IsFixedSize_t alwaysFixedSize,
                    SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
       : FixedTypeInfo(type, size, spareBits, align, pod,
-                      // All currently implemented loadable types are bitwise-takable.
-                      IsBitwiseTakable,
+                      // All currently implemented loadable types are
+                      // bitwise-takable and -borrowable.
+                      IsBitwiseTakableAndBorrowable,
                       copy, alwaysFixedSize, stik) {
     assert(isLoadable());
   }
@@ -78,8 +79,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
                    IsFixedSize_t alwaysFixedSize,
                    SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
       : FixedTypeInfo(type, size, std::move(spareBits), align, pod,
-                      // All currently implemented loadable types are bitwise-takable.
-                      IsBitwiseTakable,
+                      // All currently implemented loadable types are
+                      // bitwise-takable and borrowable.
+                      IsBitwiseTakableAndBorrowable,
                       copy, alwaysFixedSize, stik) {
     assert(isLoadable());
   }

lib/IRGen/StructLayout.cpp

@@ -80,7 +80,8 @@ StructLayout::StructLayout(IRGenModule &IGM, std::optional<CanType> type,
   if (rawLayout && type) {
     auto sd = cast<StructDecl>(decl);
     IsKnownTriviallyDestroyable = deinit;
-    IsKnownBitwiseTakable = IsBitwiseTakable;
+    // Raw layout types are never bitwise-borrowable.
+    IsKnownBitwiseTakable = IsBitwiseTakableOnly;
     SpareBits.clear();
     assert(!copyable);
     IsKnownCopyable = copyable;
@@ -125,7 +126,9 @@ StructLayout::StructLayout(IRGenModule &IGM, std::optional<CanType> type,
         // as the like its like.
         if (rawLayout->shouldMoveAsLikeType()) {
           IsKnownTriviallyDestroyable = likeFixedType->isTriviallyDestroyable(ResilienceExpansion::Maximal);
-          IsKnownBitwiseTakable = likeFixedType->isBitwiseTakable(ResilienceExpansion::Maximal);
+          // Raw layout types are still never bitwise-borrowable.
+          IsKnownBitwiseTakable = likeFixedType->getBitwiseTakable(ResilienceExpansion::Maximal)
+            & IsBitwiseTakableOnly;
         }
       } else {
         MinimumSize = Size(0);
@@ -397,7 +400,7 @@ bool StructLayoutBuilder::addField(ElementLayout &elt,
                                   LayoutStrategy strategy) {
   auto &eltTI = elt.getType();
   IsKnownTriviallyDestroyable &= eltTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
-  IsKnownBitwiseTakable &= eltTI.isBitwiseTakable(ResilienceExpansion::Maximal);
+  IsKnownBitwiseTakable &= eltTI.getBitwiseTakable(ResilienceExpansion::Maximal);
   IsKnownAlwaysFixedSize &= eltTI.isFixedSize(ResilienceExpansion::Minimal);
   IsLoadable &= eltTI.isLoadable();
   IsKnownCopyable &= eltTI.isCopyable(ResilienceExpansion::Maximal);

lib/IRGen/StructLayout.h

@@ -285,7 +285,7 @@ class StructLayoutBuilder {
   bool IsFixedLayout = true;
   bool IsLoadable = true;
   IsTriviallyDestroyable_t IsKnownTriviallyDestroyable = IsTriviallyDestroyable;
-  IsBitwiseTakable_t IsKnownBitwiseTakable = IsBitwiseTakable;
+  IsBitwiseTakable_t IsKnownBitwiseTakable = IsBitwiseTakableAndBorrowable;
   IsCopyable_t IsKnownCopyable = IsCopyable;
   IsFixedSize_t IsKnownAlwaysFixedSize = IsFixedSize;
 public: