Skip to content

Commit d7065f1

Browse files
aschwaighoferaschwaighofer
authored
Merge pull request swiftlang#8246 from aschwaighofer/cow_existentials_runtime_valuebuffer
Copy-on-write existential improvements
2 parents 505c25f + 5634280 commit d7065f1

File tree

3 files changed

+164
-37
lines changed

3 files changed

+164
-37
lines changed

‎lib/IRGen/GenOpaque.cpp

Lines changed: 153 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "llvm/Support/raw_ostream.h"
2525
#include "llvm/IR/DerivedTypes.h"
2626

27+
#include "FixedTypeInfo.h"
2728
#include "IRGenFunction.h"
2829
#include "IRGenModule.h"
2930
#include "ProtocolInfo.h"
@@ -948,41 +949,161 @@ void irgen::emitDestroyCall(IRGenFunction &IGF, llvm::Value *metadata,
948949

949950
Address irgen::emitAllocateValueInBuffer(IRGenFunction &IGF, SILType type,
950951
Address buffer) {
951-
auto *size = emitLoadOfSize(IGF, type);
952-
auto *alignMask = emitLoadOfAlignmentMask(IGF, type);
953-
// TODO: check whether we fit in the inline value buffer.
954-
auto valueAddr = IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
955-
IGF.Builder.CreateStore(
956-
valueAddr,
957-
Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
958-
valueAddr->getType()->getPointerTo()),
959-
Alignment(1)));
960-
valueAddr =
961-
IGF.Builder.CreateBitCast(valueAddr, IGF.IGM.getStoragePointerType(type));
962-
return Address(valueAddr, Alignment(1));
963-
964-
}
965-
966-
Address irgen::emitProjectValueInBuffer(IRGenFunction &IGF,
967-
SILType type,
968-
Address buffer) {
969-
// TODO: check whether we fit in the inline value buffer.
970-
auto ptr = IGF.Builder.CreateLoad(Address(
971-
IGF.Builder.CreateBitCast(buffer.getAddress(), IGF.IGM.Int8PtrPtrTy),
972-
Alignment(1)));
973-
auto valueAddr =
974-
IGF.Builder.CreateBitCast(ptr, IGF.IGM.getStoragePointerType(type));
975-
return Address(valueAddr, Alignment(1));
952+
// Handle FixedSize types.
953+
auto &IGM = IGF.IGM;
954+
auto storagePtrTy = IGM.getStoragePointerType(type);
955+
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
956+
auto packing = fixedTI->getFixedPacking(IGM);
957+
958+
// Inline representation.
959+
if (packing == FixedPacking::OffsetZero) {
960+
return Address(
961+
IGF.Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
962+
buffer.getAlignment());
963+
}
964+
965+
// Outline representation.
966+
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
967+
auto size = fixedTI->getStaticSize(IGM);
968+
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
969+
auto valueAddr =
970+
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
971+
IGF.Builder.CreateStore(
972+
valueAddr,
973+
Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
974+
valueAddr->getType()->getPointerTo()),
975+
buffer.getAlignment()));
976+
return Address(IGF.Builder.CreateBitCast(valueAddr, storagePtrTy),
977+
buffer.getAlignment());
978+
}
979+
980+
// Dynamic packing.
981+
llvm::Value *isInline = emitLoadOfIsInline(IGF, type);
982+
auto *outlineBB = IGF.createBasicBlock("outline.allocateValueInBuffer");
983+
auto *doneBB = IGF.createBasicBlock("done");
984+
llvm::Value *addressInline, *addressOutline;
985+
auto *origBB = IGF.Builder.GetInsertBlock();
986+
addressInline = IGF.Builder.CreateBitCast(buffer.getAddress(), storagePtrTy);
987+
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
988+
989+
IGF.Builder.emitBlock(outlineBB);
990+
{
991+
ConditionalDominanceScope scope(IGF);
992+
auto *size = emitLoadOfSize(IGF, type);
993+
auto *alignMask = emitLoadOfAlignmentMask(IGF, type);
994+
auto valueAddr =
995+
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
996+
IGF.Builder.CreateStore(
997+
valueAddr,
998+
Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
999+
valueAddr->getType()->getPointerTo()),
1000+
Alignment(1)));
1001+
addressOutline = IGF.Builder.CreateBitCast(valueAddr, storagePtrTy);
1002+
IGF.Builder.CreateBr(doneBB);
1003+
}
1004+
1005+
IGF.Builder.emitBlock(doneBB);
1006+
auto *addressOfValue = IGF.Builder.CreatePHI(storagePtrTy, 2);
1007+
addressOfValue->addIncoming(addressInline, origBB);
1008+
addressOfValue->addIncoming(addressOutline, outlineBB);
1009+
1010+
return Address(addressOfValue, Alignment(1));
1011+
}
1012+
1013+
Address irgen::emitProjectValueInBuffer(IRGenFunction &IGF, SILType type,
1014+
Address buffer) {
1015+
// Handle FixedSize types.
1016+
auto &IGM = IGF.IGM;
1017+
auto storagePtrTy = IGM.getStoragePointerType(type);
1018+
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
1019+
auto packing = fixedTI->getFixedPacking(IGM);
1020+
1021+
// Inline representation.
1022+
if (packing == FixedPacking::OffsetZero) {
1023+
return Address(
1024+
IGF.Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
1025+
buffer.getAlignment());
1026+
}
1027+
1028+
// Outline representation.
1029+
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
1030+
auto valueAddr = IGF.Builder.CreateLoad(
1031+
Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
1032+
storagePtrTy->getPointerTo()),
1033+
buffer.getAlignment()));
1034+
return Address(IGF.Builder.CreateBitCast(valueAddr, storagePtrTy),
1035+
buffer.getAlignment());
1036+
}
1037+
1038+
// Dynamic packing.
1039+
llvm::Value *isInline = emitLoadOfIsInline(IGF, type);
1040+
auto *outlineBB = IGF.createBasicBlock("outline.projectValueInBuffer");
1041+
auto *doneBB = IGF.createBasicBlock("done");
1042+
llvm::Value *addressInline, *addressOutline;
1043+
auto *origBB = IGF.Builder.GetInsertBlock();
1044+
addressInline = IGF.Builder.CreateBitCast(buffer.getAddress(), storagePtrTy);
1045+
1046+
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
1047+
1048+
IGF.Builder.emitBlock(outlineBB);
1049+
{
1050+
auto ptr = IGF.Builder.CreateLoad(
1051+
Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
1052+
storagePtrTy->getPointerTo()),
1053+
Alignment(1)));
1054+
addressOutline = IGF.Builder.CreateBitCast(ptr, storagePtrTy);
1055+
IGF.Builder.CreateBr(doneBB);
1056+
}
1057+
1058+
IGF.Builder.emitBlock(doneBB);
1059+
auto *addressOfValue = IGF.Builder.CreatePHI(storagePtrTy, 2);
1060+
addressOfValue->addIncoming(addressInline, origBB);
1061+
addressOfValue->addIncoming(addressOutline, outlineBB);
1062+
1063+
return Address(addressOfValue, Alignment(1));
9761064
}
9771065

9781066
void irgen::emitDeallocateValueInBuffer(IRGenFunction &IGF,
9791067
SILType type,
9801068
Address buffer) {
981-
auto *size = emitLoadOfSize(IGF, type);
982-
auto *alignMask = emitLoadOfAlignmentMask(IGF, type);
983-
auto *ptr = IGF.Builder.CreateLoad(Address(
984-
IGF.Builder.CreateBitCast(buffer.getAddress(), IGF.IGM.Int8PtrPtrTy),
985-
Alignment(1)));
986-
// TODO: check whether we fit in the inline value buffer.
987-
IGF.emitDeallocRawCall(ptr, size, alignMask);
1069+
// Handle FixedSize types.
1070+
auto &IGM = IGF.IGM;
1071+
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
1072+
auto packing = fixedTI->getFixedPacking(IGM);
1073+
1074+
// Inline representation.
1075+
if (packing == FixedPacking::OffsetZero)
1076+
return;
1077+
1078+
// Outline representation.
1079+
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
1080+
auto size = fixedTI->getStaticSize(IGM);
1081+
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
1082+
auto *ptr = IGF.Builder.CreateLoad(Address(
1083+
IGF.Builder.CreateBitCast(buffer.getAddress(), IGM.Int8PtrPtrTy),
1084+
buffer.getAlignment()));
1085+
IGF.emitDeallocRawCall(ptr, size, alignMask);
1086+
return;
1087+
}
1088+
1089+
// Dynamic packing.
1090+
llvm::Value *isInline = emitLoadOfIsInline(IGF, type);
1091+
auto *outlineBB = IGF.createBasicBlock("outline.projectValueInBuffer");
1092+
auto *doneBB = IGF.createBasicBlock("done");
1093+
1094+
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
1095+
1096+
IGF.Builder.emitBlock(outlineBB);
1097+
{
1098+
ConditionalDominanceScope scope(IGF);
1099+
auto *size = emitLoadOfSize(IGF, type);
1100+
auto *alignMask = emitLoadOfAlignmentMask(IGF, type);
1101+
auto *ptr = IGF.Builder.CreateLoad(Address(
1102+
IGF.Builder.CreateBitCast(buffer.getAddress(), IGM.Int8PtrPtrTy),
1103+
buffer.getAlignment()));
1104+
IGF.emitDeallocRawCall(ptr, size, alignMask);
1105+
IGF.Builder.CreateBr(doneBB);
1106+
}
1107+
1108+
IGF.Builder.emitBlock(doneBB);
9881109
}

‎stdlib/public/runtime/Casting.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1593,7 +1593,10 @@ static void unwrapExistential(OpaqueValue *src,
15931593
auto opaqueContainer = reinterpret_cast<OpaqueExistentialContainer*>(src);
15941594
srcCapturedType = opaqueContainer->Type;
15951595
srcValue = srcType->projectValue(src);
1596-
canTake = false;
1596+
// Can't take out of possibly shared existential boxes.
1597+
canTake = (src == srcValue);
1598+
assert(canTake == srcCapturedType->getValueWitnesses()->isValueInline() &&
1599+
"Only inline storage is take-able");
15971600
#else
15981601
auto opaqueContainer = reinterpret_cast<OpaqueExistentialContainer*>(src);
15991602
srcCapturedType = opaqueContainer->Type;
@@ -1653,9 +1656,7 @@ static bool _dynamicCastFromExistential(OpaqueValue *dest,
16531656
} else {
16541657
#ifdef SWIFT_RUNTIME_ENABLE_COW_EXISTENTIALS
16551658
assert(!isOutOfLine &&
1656-
srcType->getRepresentation() !=
1657-
ExistentialTypeRepresentation::Opaque &&
1658-
"Should only see inline represenations and no opaque existential");
1659+
"Should only see inline represenations of existentials");
16591660
#else
16601661
// swift_dynamicCast took or destroyed the value as per the original request
16611662
// We may still have an opaque existential container to deallocate.

‎stdlib/public/runtime/Metadata.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2228,8 +2228,13 @@ ExistentialTypeMetadata::mayTakeValue(const OpaqueValue *container) const {
22282228
// Opaque existential containers uniquely own their contained value.
22292229
case ExistentialTypeRepresentation::Opaque:
22302230
#ifdef SWIFT_RUNTIME_ENABLE_COW_EXISTENTIALS
2231+
{
22312232
// We can't take from a shared existential box without checking uniqueness.
2232-
return false;
2233+
auto *opaque =
2234+
reinterpret_cast<const OpaqueExistentialContainer *>(container);
2235+
auto *vwt = opaque->Type->getValueWitnesses();
2236+
return vwt->isValueInline();
2237+
}
22332238
#else
22342239
return true;
22352240
#endif

0 commit comments

Comments
 (0)