Skip to content

Commit 923cccf

Browse files
drexindrexin
committed
[Runtime] Add specialized CVW entry points for multi payload enums
rdar://143852239 Adding these specialized entry points reduces the overhead of the witness functions by removing the first indirection.
1 parent 109e208 commit 923cccf

File tree

8 files changed

+318
-35
lines changed

8 files changed

+318
-35
lines changed

‎include/swift/Runtime/RuntimeFunctions.def

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2747,6 +2747,68 @@ FUNCTION(GenericInitializeBufferWithCopyOfBuffer,
27472747
EFFECT(RefCounting),
27482748
UNKNOWN_MEMEFFECTS)
27492749

2750+
// void swift_cvw_destroyMultiPayloadEnumFN(opaque*, const Metadata* type);
2751+
FUNCTION(GenericDestroyMultiPayloadEnumFN,
2752+
Swift, swift_cvw_destroyMultiPayloadEnumFN,
2753+
C_CC, AlwaysAvailable,
2754+
RETURNS(VoidTy),
2755+
ARGS(Int8PtrTy, TypeMetadataPtrTy),
2756+
ATTRS(NoUnwind),
2757+
EFFECT(Deallocating),
2758+
UNKNOWN_MEMEFFECTS)
2759+
2760+
// void *swift_cvw_assignWithCopyMultiPayloadEnumFN(opaque* dest, opaque* src, const Metadata* type);
2761+
FUNCTION(GenericAssignWithCopyMultiPayloadEnumFN,
2762+
Swift, swift_cvw_assignWithCopyMultiPayloadEnumFN,
2763+
C_CC, AlwaysAvailable,
2764+
RETURNS(Int8PtrTy),
2765+
ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
2766+
ATTRS(NoUnwind),
2767+
EFFECT(RefCounting, Deallocating),
2768+
UNKNOWN_MEMEFFECTS)
2769+
2770+
// void *swift_cvw_assignWithTakeMultiPayloadEnumFN(opaque* dest, opaque* src, const Metadata* type);
2771+
FUNCTION(GenericAssignWithTakeMultiPayloadEnumFN,
2772+
Swift, swift_cvw_assignWithTakeMultiPayloadEnumFN,
2773+
C_CC, AlwaysAvailable,
2774+
RETURNS(Int8PtrTy),
2775+
ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
2776+
ATTRS(NoUnwind),
2777+
EFFECT(RefCounting, Deallocating),
2778+
UNKNOWN_MEMEFFECTS)
2779+
2780+
// void *swift_cvw_initWithCopyMultiPayloadEnumFN(opaque* dest, opaque* src, const Metadata* type);
2781+
FUNCTION(GenericInitWithCopyMultiPayloadEnumFN,
2782+
Swift, swift_cvw_initWithCopyMultiPayloadEnumFN,
2783+
C_CC, AlwaysAvailable,
2784+
RETURNS(Int8PtrTy),
2785+
ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
2786+
ATTRS(NoUnwind),
2787+
EFFECT(RefCounting),
2788+
UNKNOWN_MEMEFFECTS)
2789+
2790+
// void *swift_cvw_initWithTakeMultiPayloadEnumFN(opaque* dest, opaque* src, const Metadata* type);
2791+
FUNCTION(GenericInitWithTakeMultiPayloadEnumFN,
2792+
Swift, swift_cvw_initWithTakeMultiPayloadEnumFN,
2793+
C_CC, AlwaysAvailable,
2794+
RETURNS(Int8PtrTy),
2795+
ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
2796+
ATTRS(NoUnwind),
2797+
EFFECT(RefCounting),
2798+
UNKNOWN_MEMEFFECTS)
2799+
2800+
// void *swift_cvw_initializeBufferWithCopyOfBufferMultiPayloadEnumFN(ValueBuffer* dest, ValueBuffer* src, const Metadata* type);
2801+
FUNCTION(GenericInitializeBufferWithCopyOfBufferMultiPayloadEnumFN,
2802+
Swift, swift_cvw_initializeBufferWithCopyOfBufferMultiPayloadEnumFN,
2803+
C_CC, AlwaysAvailable,
2804+
RETURNS(Int8PtrTy),
2805+
ARGS(getFixedBufferTy()->getPointerTo(),
2806+
getFixedBufferTy()->getPointerTo(),
2807+
TypeMetadataPtrTy),
2808+
ATTRS(NoUnwind),
2809+
EFFECT(RefCounting),
2810+
UNKNOWN_MEMEFFECTS)
2811+
27502812
// unsigned swift_cvw_singletonEnum_getEnumTag(swift::OpaqueValue *address,
27512813
// const Metadata *metadata);
27522814
FUNCTION(SingletonEnumGetEnumTag,

‎lib/IRGen/GenValueWitness.cpp

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -933,6 +933,21 @@ static bool isRuntimeInstatiatedLayoutString(IRGenModule &IGM,
933933
return false;
934934
}
935935

936+
static bool
937+
useMultiPayloadEnumFNSpecialization(IRGenModule &IGM,
938+
const TypeLayoutEntry *typeLayoutEntry,
939+
GenericSignature genericSig) {
940+
// if (!typeLayoutEntry->layoutString(IGM, genericSig)) {
941+
// return false;
942+
// }
943+
// auto *enumTLE = typeLayoutEntry->getAsEnum();
944+
// return enumTLE && enumTLE->isFixedSize(IGM) &&
945+
// enumTLE->isMultiPayloadEnum();
946+
947+
// Disabled for now
948+
return false;
949+
}
950+
936951
static llvm::Constant *getEnumTagFunction(IRGenModule &IGM,
937952
const EnumTypeLayoutEntry *typeLayoutEntry,
938953
GenericSignature genericSig) {
@@ -953,19 +968,7 @@ static llvm::Constant *getEnumTagFunction(IRGenModule &IGM,
953968
} else if (typeLayoutEntry->isMultiPayloadEnum()) {
954969
return IGM.getEnumFnGetEnumTagFn();
955970
} else {
956-
auto &payloadTI = **(typeLayoutEntry->cases[0]->getFixedTypeInfo());
957-
auto mask = payloadTI.getFixedExtraInhabitantMask(IGM);
958-
auto tzCount = mask.countTrailingZeros();
959-
auto shiftedMask = mask.lshr(tzCount);
960-
// auto toCount = shiftedMask.countTrailingOnes();
961-
// if (payloadTI.mayHaveExtraInhabitants(IGM) &&
962-
// (mask.popcount() > 64 ||
963-
// toCount != mask.popcount() ||
964-
// (tzCount % toCount != 0))) {
965-
return IGM.getEnumFnGetEnumTagFn();
966-
// } else {
967-
// return IGM.getEnumSimpleGetEnumTagFn();
968-
// }
971+
return IGM.getEnumFnGetEnumTagFn();
969972
}
970973
}
971974

@@ -990,18 +993,7 @@ getDestructiveInjectEnumTagFunction(IRGenModule &IGM,
990993
} else if (typeLayoutEntry->isMultiPayloadEnum()) {
991994
return nullptr;
992995
} else {
993-
auto &payloadTI = **(typeLayoutEntry->cases[0]->getFixedTypeInfo());
994-
auto mask = payloadTI.getFixedExtraInhabitantMask(IGM);
995-
auto tzCount = mask.countTrailingZeros();
996-
auto shiftedMask = mask.lshr(tzCount);
997-
// auto toCount = shiftedMask.countTrailingOnes();
998-
// if (payloadTI.mayHaveExtraInhabitants(IGM) &&
999-
// (mask.popcount() > 64 || toCount != mask.popcount() ||
1000-
// (tzCount % toCount != 0))) {
1001-
return nullptr;
1002-
// } else {
1003-
// return IGM.getEnumSimpleDestructiveInjectEnumTagFn();
1004-
// }
996+
return nullptr;
1005997
}
1006998
}
1007999

@@ -1072,7 +1064,12 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
10721064
->getGenericSignature();
10731065
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
10741066
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1075-
return addFunction(IGM.getGenericDestroyFn());
1067+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1068+
genericSig)) {
1069+
return addFunction(IGM.getGenericDestroyMultiPayloadEnumFNFn());
1070+
} else {
1071+
return addFunction(IGM.getGenericDestroyFn());
1072+
}
10761073
}
10771074
}
10781075
}
@@ -1101,8 +1098,14 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
11011098
->getGenericSignature();
11021099
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
11031100
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1104-
return addFunction(
1105-
IGM.getGenericInitializeBufferWithCopyOfBufferFn());
1101+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1102+
genericSig)) {
1103+
return addFunction(
1104+
IGM.getGenericInitializeBufferWithCopyOfBufferMultiPayloadEnumFNFn());
1105+
} else {
1106+
return addFunction(
1107+
IGM.getGenericInitializeBufferWithCopyOfBufferFn());
1108+
}
11061109
}
11071110
}
11081111
}
@@ -1121,7 +1124,13 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
11211124
->getGenericSignature();
11221125
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
11231126
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1124-
return addFunction(IGM.getGenericInitWithTakeFn());
1127+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1128+
genericSig)) {
1129+
return addFunction(
1130+
IGM.getGenericInitWithTakeMultiPayloadEnumFNFn());
1131+
} else {
1132+
return addFunction(IGM.getGenericInitWithTakeFn());
1133+
}
11251134
}
11261135
}
11271136
}
@@ -1142,7 +1151,13 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
11421151
->getGenericSignature();
11431152
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
11441153
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1145-
return addFunction(IGM.getGenericAssignWithCopyFn());
1154+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1155+
genericSig)) {
1156+
return addFunction(
1157+
IGM.getGenericAssignWithCopyMultiPayloadEnumFNFn());
1158+
} else {
1159+
return addFunction(IGM.getGenericAssignWithCopyFn());
1160+
}
11461161
}
11471162
}
11481163
}
@@ -1163,7 +1178,13 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
11631178
->getGenericSignature();
11641179
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
11651180
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1166-
return addFunction(IGM.getGenericAssignWithTakeFn());
1181+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1182+
genericSig)) {
1183+
return addFunction(
1184+
IGM.getGenericAssignWithTakeMultiPayloadEnumFNFn());
1185+
} else {
1186+
return addFunction(IGM.getGenericAssignWithTakeFn());
1187+
}
11671188
}
11681189
}
11691190
}
@@ -1184,7 +1205,13 @@ static void addValueWitness(IRGenModule &IGM, ConstantStructBuilder &B,
11841205
->getGenericSignature();
11851206
if (typeLayoutEntry->layoutString(IGM, genericSig) ||
11861207
isRuntimeInstatiatedLayoutString(IGM, typeLayoutEntry)) {
1187-
return addFunction(IGM.getGenericInitWithCopyFn());
1208+
if (useMultiPayloadEnumFNSpecialization(IGM, typeLayoutEntry,
1209+
genericSig)) {
1210+
return addFunction(
1211+
IGM.getGenericInitWithCopyMultiPayloadEnumFNFn());
1212+
} else {
1213+
return addFunction(IGM.getGenericInitWithCopyFn());
1214+
}
11881215
}
11891216
}
11901217
}

‎stdlib/public/CompatibilityOverride/CompatibilityOverrideRuntime.def

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,42 @@ OVERRIDE_CVW(cvw_initializeBufferWithCopyOfBuffer, swift::OpaqueValue *,
310310
const Metadata *metadata),
311311
(dest, src, metadata))
312312

313+
OVERRIDE_CVW(cvw_destroyMultiPayloadEnumFN, void, SWIFT_RUNTIME_EXPORT,
314+
, swift::, (swift::OpaqueValue *address,
315+
const Metadata *metadata),
316+
(address, metadata))
317+
318+
OVERRIDE_CVW(cvw_assignWithCopyMultiPayloadEnumFN, swift::OpaqueValue *, SWIFT_RUNTIME_EXPORT,
319+
, swift::, (swift::OpaqueValue *dest,
320+
swift::OpaqueValue *src,
321+
const Metadata *metadata),
322+
(dest, src, metadata))
323+
324+
OVERRIDE_CVW(cvw_assignWithTakeMultiPayloadEnumFN, swift::OpaqueValue *, SWIFT_RUNTIME_EXPORT,
325+
, swift::, (swift::OpaqueValue *dest,
326+
swift::OpaqueValue *src,
327+
const Metadata *metadata),
328+
(dest, src, metadata))
329+
330+
OVERRIDE_CVW(cvw_initWithCopyMultiPayloadEnumFN, swift::OpaqueValue *, SWIFT_RUNTIME_EXPORT,
331+
, swift::, (swift::OpaqueValue *dest,
332+
swift::OpaqueValue *src,
333+
const Metadata *metadata),
334+
(dest, src, metadata))
335+
336+
OVERRIDE_CVW(cvw_initWithTakeMultiPayloadEnumFN, swift::OpaqueValue *, SWIFT_RUNTIME_EXPORT,
337+
, swift::, (swift::OpaqueValue *dest,
338+
swift::OpaqueValue *src,
339+
const Metadata *metadata),
340+
(dest, src, metadata))
341+
342+
OVERRIDE_CVW(cvw_initializeBufferWithCopyOfBufferMultiPayloadEnumFN, swift::OpaqueValue *,
343+
SWIFT_RUNTIME_EXPORT,
344+
, swift::, (swift::ValueBuffer *dest,
345+
swift::ValueBuffer *src,
346+
const Metadata *metadata),
347+
(dest, src, metadata))
348+
313349
OVERRIDE_CVW(cvw_enumSimple_getEnumTag, unsigned, SWIFT_RUNTIME_EXPORT,
314350
, swift::, (swift::OpaqueValue *address,
315351
const Metadata *metadata),

0 commit comments

Comments
 (0)