Skip to content

Commit 405e659

Browse files
rjmccallrjmccall
committed
Propagate the XI count into the get/store XI tag callbacks.
This allows callers to avoid needing to reload these tags in common cases.
1 parent 4b72c3a commit 405e659

File tree

12 files changed

+92
-54
lines changed

12 files changed

+92
-54
lines changed

‎include/swift/Runtime/Enum.h

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ void swift_initEnumMetadataSinglePayload(EnumMetadata *enumType,
5858
const TypeLayout *payload,
5959
unsigned emptyCases);
6060

61+
using getExtraInhabitantTag_t =
62+
SWIFT_CC(swift) unsigned (const OpaqueValue *vaue,
63+
unsigned numExtraInhabitants,
64+
const Metadata *payloadType);
65+
6166
/// Implement getEnumTagSinglePayload generically in terms of a
6267
/// payload type with a getExtraInhabitantIndex function.
6368
///
@@ -68,12 +73,17 @@ void swift_initEnumMetadataSinglePayload(EnumMetadata *enumType,
6873
/// \returns 0 if the payload case is inhabited. If an empty case is inhabited,
6974
/// returns a value greater than or equal to one and less than or equal
7075
/// emptyCases.
71-
SWIFT_RUNTIME_EXPORT
76+
SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
7277
unsigned swift_getEnumTagSinglePayloadGeneric(const OpaqueValue *value,
7378
unsigned emptyCases,
7479
const Metadata *payloadType,
75-
unsigned (*getExtraInhabitantTag)(const OpaqueValue *value,
76-
const Metadata *payloadType));
80+
getExtraInhabitantTag_t *getTag);
81+
82+
using storeExtraInhabitantTag_t =
83+
SWIFT_CC(swift) void (OpaqueValue *value,
84+
unsigned whichCase,
85+
unsigned numExtraInhabitants,
86+
const Metadata *payloadType);
7787

7888
/// Implement storeEnumTagSinglePayload generically in terms of a
7989
/// payload type with a storeExtraInhabitant function.
@@ -86,14 +96,12 @@ unsigned swift_getEnumTagSinglePayloadGeneric(const OpaqueValue *value,
8696
/// case, or a value greater than or equal to one and less
8797
/// than or equal emptyCases for an empty case.
8898
/// \param emptyCases - the number of empty cases in the enum.
89-
SWIFT_RUNTIME_EXPORT
99+
SWIFT_RUNTIME_EXPORT SWIFT_CC(swift)
90100
void swift_storeEnumTagSinglePayloadGeneric(OpaqueValue *value,
91101
unsigned whichCase,
92102
unsigned emptyCases,
93103
const Metadata *payloadType,
94-
void (*storeExtraInhabitantTag)(OpaqueValue *value,
95-
unsigned whichCase,
96-
const Metadata *payloadType));
104+
storeExtraInhabitantTag_t *storeTag);
97105

98106
/// \brief Initialize the type metadata for a generic, multi-payload
99107
/// enum instance.

‎include/swift/Runtime/RuntimeFunctions.def

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -874,14 +874,16 @@ FUNCTION(GetEnumCaseMultiPayload,
874874
// int swift_getEnumTagSinglePayloadGeneric(opaque_t *obj,
875875
// unsigned num_empty_cases,
876876
// Metadata *payloadType,
877-
// int (*getExtraInhabitantIndex)(opaque_t *obj,
878-
// Metadata *payload));
877+
// int (*getExtraInhabitantIndex)(opaque_t *obj,
878+
// unsigned numPayloadXI,
879+
// Metadata *payload));
879880
FUNCTION(GetEnumTagSinglePayloadGeneric,
880881
swift_getEnumTagSinglePayloadGeneric,
881-
C_CC,
882+
SwiftCC,
882883
RETURNS(Int32Ty),
883884
ARGS(OpaquePtrTy, Int32Ty, TypeMetadataPtrTy,
884-
llvm::FunctionType::get(Int32Ty, {OpaquePtrTy, TypeMetadataPtrTy},
885+
llvm::FunctionType::get(Int32Ty, {OpaquePtrTy, Int32Ty,
886+
TypeMetadataPtrTy},
885887
false)->getPointerTo()),
886888
ATTRS(NoUnwind, ReadOnly))
887889

@@ -892,13 +894,14 @@ FUNCTION(GetEnumTagSinglePayloadGeneric,
892894
// Metadata *payloadType,
893895
// void (*storeExtraInhabitant)(opaque_t *obj,
894896
// unsigned case_index,
897+
// unsigned numPayloadXI,
895898
// Metadata *payload));
896899
FUNCTION(StoreEnumTagSinglePayloadGeneric,
897900
swift_storeEnumTagSinglePayloadGeneric,
898-
C_CC,
901+
SwiftCC,
899902
RETURNS(VoidTy),
900903
ARGS(OpaquePtrTy, Int32Ty, Int32Ty, TypeMetadataPtrTy,
901-
llvm::FunctionType::get(VoidTy, {OpaquePtrTy, Int32Ty,
904+
llvm::FunctionType::get(VoidTy, {OpaquePtrTy, Int32Ty, Int32Ty,
902905
TypeMetadataPtrTy},
903906
false)->getPointerTo()),
904907
ATTRS(NoUnwind))

‎lib/IRGen/GenEnum.cpp

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3066,15 +3066,13 @@ namespace {
30663066
// double-optionals.
30673067
return emitGetEnumTagSinglePayloadGenericCall(IGF, T, *TI, numEmptyCases,
30683068
addr,
3069-
[this, T](IRGenFunction &IGF, Address addr) -> llvm::Value* {
3069+
[this, T](IRGenFunction &IGF, Address addr,
3070+
llvm::Value *numXI) -> llvm::Value* {
30703071
auto payloadType = getPayloadType(IGF.IGM, T);
30713072
auto payloadAddr = projectPayloadData(IGF, addr);
30723073

3073-
// For the case count, we just load the XI count from the metadata
3074-
// for the payload type. We're almost certainly about to call a value
3075-
// witness on that metadata anyway.
3076-
auto payloadNumExtraCases =
3077-
emitLoadOfExtraInhabitantCount(IGF, payloadType);
3074+
// For the case count, we just use the XI count from the payload type.
3075+
auto payloadNumExtraCases = numXI;
30783076

30793077
llvm::Value *tag
30803078
= getPayloadTypeInfo().getEnumTagSinglePayload(IGF,
@@ -3131,16 +3129,13 @@ namespace {
31313129
// double-optionals.
31323130
emitStoreEnumTagSinglePayloadGenericCall(IGF, T, *TI, tag,
31333131
numEmptyCases, dest,
3134-
[this, T](IRGenFunction &IGF, Address dest, llvm::Value *tag) {
3132+
[this, T](IRGenFunction &IGF, Address dest, llvm::Value *tag,
3133+
llvm::Value *payloadNumXI) {
31353134
auto payloadType = getPayloadType(IGF.IGM, T);
31363135
auto payloadDest = projectPayloadData(IGF, dest);
31373136
auto payloadTag = adjustExtraInhabitantTagForPayload(IGF, tag,
31383137
/*nonzero*/false);
31393138

3140-
// We know that tag <= numXI, and as long as that's true, storing an
3141-
// extra inhabitant is always a no-op.
3142-
auto payloadNumXI = payloadTag;
3143-
31443139
getPayloadTypeInfo().storeEnumTagSinglePayload(IGF, payloadTag,
31453140
payloadNumXI,
31463141
payloadDest,

‎lib/IRGen/GenOpaque.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1285,6 +1285,7 @@ irgen::emitGetEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
12851285
numExtraCases,
12861286
metadata,
12871287
getExtraInhabitantTagFn});
1288+
call->setCallingConv(IGF.IGM.SwiftCC);
12881289
return call;
12891290
}
12901291

@@ -1299,16 +1300,19 @@ irgen::getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
12991300

13001301
auto fnTy = llvm::FunctionType::get(IGM.Int32Ty,
13011302
{IGM.OpaquePtrTy,
1303+
IGM.Int32Ty,
13021304
IGM.TypeMetadataPtrTy},
13031305
false);
13041306

13051307
// TODO: use a meaningful mangled name and internal/shared linkage.
13061308
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
13071309
"__swift_get_extra_inhabitant_index",
13081310
&IGM.Module);
1311+
fn->setCallingConv(IGM.SwiftCC);
13091312
IRGenFunction IGF(IGM, fn);
13101313
auto parameters = IGF.collectParameters();
13111314
auto ptr = parameters.claimNext();
1315+
auto xiCount = parameters.claimNext();
13121316
auto metadata = parameters.claimNext();
13131317

13141318
// Bind the metadata to make any archetypes available.
@@ -1320,7 +1324,7 @@ irgen::getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
13201324
objectTI.getStorageType()->getPointerTo());
13211325
Address addr = objectTI.getAddressForPointer(ptr);
13221326

1323-
auto tag = emitter(IGF, addr);
1327+
auto tag = emitter(IGF, addr, xiCount);
13241328
IGF.Builder.CreateRet(tag);
13251329

13261330
return fn;
@@ -1354,7 +1358,7 @@ irgen::emitStoreEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
13541358
numExtraCases,
13551359
metadata,
13561360
storeExtraInhabitantTagFn});
1357-
(void) call;
1361+
call->setCallingConv(IGF.IGM.SwiftCC);
13581362
}
13591363

13601364
llvm::Constant *
@@ -1368,6 +1372,7 @@ irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
13681372

13691373
auto fnTy = llvm::FunctionType::get(IGM.VoidTy,
13701374
{IGM.OpaquePtrTy,
1375+
IGM.Int32Ty,
13711376
IGM.Int32Ty,
13721377
IGM.TypeMetadataPtrTy},
13731378
false);
@@ -1376,10 +1381,12 @@ irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
13761381
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
13771382
"__swift_get_extra_inhabitant_index",
13781383
&IGM.Module);
1384+
fn->setCallingConv(IGM.SwiftCC);
13791385
IRGenFunction IGF(IGM, fn);
13801386
auto parameters = IGF.collectParameters();
13811387
auto ptr = parameters.claimNext();
13821388
auto tag = parameters.claimNext();
1389+
auto xiCount = parameters.claimNext();
13831390
auto metadata = parameters.claimNext();
13841391

13851392
// Bind the metadata to make any archetypes available.
@@ -1391,7 +1398,7 @@ irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
13911398
objectTI.getStorageType()->getPointerTo());
13921399
Address addr = objectTI.getAddressForPointer(ptr);
13931400

1394-
emitter(IGF, addr, tag);
1401+
emitter(IGF, addr, tag, xiCount);
13951402
IGF.Builder.CreateRetVoid();
13961403

13971404
return fn;

‎lib/IRGen/GenOpaque.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@ namespace irgen {
239239

240240
using GetExtraInhabitantTagEmitter =
241241
llvm::function_ref<llvm::Value*(IRGenFunction &IGF,
242-
Address addr)>;
242+
Address addr,
243+
llvm::Value *xiCount)>;
243244

244245
llvm::Constant *
245246
getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
@@ -258,7 +259,8 @@ namespace irgen {
258259
using StoreExtraInhabitantTagEmitter =
259260
llvm::function_ref<void(IRGenFunction &IGF,
260261
Address addr,
261-
llvm::Value *tag)>;
262+
llvm::Value *tag,
263+
llvm::Value *xiCount)>;
262264

263265
llvm::Constant *
264266
getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,

‎lib/IRGen/GenRecord.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -303,13 +303,14 @@ class RecordTypeInfoImpl : public Base,
303303
llvm::Value *withExtraInhabitantProvidingField(IRGenFunction &IGF,
304304
Address structAddr,
305305
SILType structType,
306+
llvm::Value *knownStructNumXI,
306307
llvm::Type *resultTy,
307308
llvm::function_ref<llvm::Value* (const FieldImpl &field,
308309
llvm::Value *numXI)> body) const {
309310
// If we know one field consistently provides extra inhabitants, delegate
310311
// to that field.
311312
if (auto field = asImpl().getFixedExtraInhabitantProvidingField(IGF.IGM)){
312-
return body(*field, nullptr);
313+
return body(*field, knownStructNumXI);
313314
}
314315

315316
// Otherwise, we have to figure out which field at runtime.
@@ -346,7 +347,9 @@ class RecordTypeInfoImpl : public Base,
346347
// Loop through checking to see whether we picked the fixed candidate
347348
// (if any) or one of the unknown-layout fields.
348349
llvm::Value *instantiatedCount
349-
= emitLoadOfExtraInhabitantCount(IGF, structType);
350+
= (knownStructNumXI
351+
? knownStructNumXI
352+
: emitLoadOfExtraInhabitantCount(IGF, structType));
350353

351354
auto contBB = IGF.createBasicBlock("chose_field_for_xi");
352355
llvm::PHINode *contPhi = nullptr;

‎lib/IRGen/GenStruct.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -487,9 +487,10 @@ namespace {
487487

488488
return emitGetEnumTagSinglePayloadGenericCall(IGF, structType, *this,
489489
numEmptyCases, structAddr,
490-
[this,structType](IRGenFunction &IGF, Address structAddr) {
490+
[this,structType](IRGenFunction &IGF, Address structAddr,
491+
llvm::Value *structNumXI) {
491492
return withExtraInhabitantProvidingField(IGF, structAddr, structType,
492-
IGF.IGM.Int32Ty,
493+
structNumXI, IGF.IGM.Int32Ty,
493494
[&](const FieldImpl &field, llvm::Value *numXI) -> llvm::Value* {
494495
Address fieldAddr = asImpl().projectFieldAddress(
495496
IGF, structAddr, structType, field);
@@ -518,9 +519,9 @@ namespace {
518519
whichCase, numEmptyCases,
519520
structAddr,
520521
[this,structType](IRGenFunction &IGF, Address structAddr,
521-
llvm::Value *tag) {
522+
llvm::Value *tag, llvm::Value *structNumXI) {
522523
withExtraInhabitantProvidingField(IGF, structAddr, structType,
523-
IGF.IGM.VoidTy,
524+
structNumXI, IGF.IGM.VoidTy,
524525
[&](const FieldImpl &field, llvm::Value *numXI) -> llvm::Value* {
525526
Address fieldAddr = asImpl().projectFieldAddress(
526527
IGF, structAddr, structType, field);

‎stdlib/public/runtime/Enum.cpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,7 @@ unsigned
130130
swift::swift_getEnumTagSinglePayloadGeneric(const OpaqueValue *value,
131131
unsigned emptyCases,
132132
const Metadata *payloadType,
133-
unsigned (*getExtraInhabitantTag)(const OpaqueValue *value,
134-
const Metadata *payloadType)) {
133+
getExtraInhabitantTag_t *getExtraInhabitantTag) {
135134
auto size = payloadType->vw_size();
136135
auto numExtraInhabitants = payloadType->vw_getNumExtraInhabitants();
137136
return getEnumTagSinglePayloadImpl(value, emptyCases, payloadType, size,
@@ -143,9 +142,7 @@ void swift::swift_storeEnumTagSinglePayloadGeneric(OpaqueValue *value,
143142
unsigned whichCase,
144143
unsigned emptyCases,
145144
const Metadata *payloadType,
146-
void (*storeExtraInhabitantTag)(OpaqueValue *value,
147-
unsigned whichCase,
148-
const Metadata *payloadType)) {
145+
storeExtraInhabitantTag_t *storeExtraInhabitantTag) {
149146
auto size = payloadType->vw_size();
150147
auto numExtraInhabitants = payloadType->vw_getNumExtraInhabitants();
151148
storeEnumTagSinglePayloadImpl(value, whichCase, emptyCases, payloadType, size,
@@ -305,17 +302,22 @@ static unsigned loadMultiPayloadValue(const OpaqueValue *value,
305302
return payloadValue;
306303
}
307304

305+
SWIFT_CC(swift)
308306
static unsigned getMultiPayloadExtraInhabitantTag(const OpaqueValue *value,
309-
const Metadata *enumType) {
307+
unsigned enumNumXI,
308+
const Metadata *enumType) {
310309
auto layout = getMultiPayloadLayout(cast<EnumMetadata>(enumType));
311310
unsigned index = ~loadMultiPayloadTag(value, layout, ~0u);
312311

313312
if (index >= enumType->getValueWitnesses()->getNumExtraInhabitants())
314313
return 0;
315314
return index + 1;
316315
}
316+
317+
SWIFT_CC(swift)
317318
static void storeMultiPayloadExtraInhabitantTag(OpaqueValue *value,
318319
unsigned tag,
320+
unsigned enumNumXI,
319321
const Metadata *enumType) {
320322
auto layout = getMultiPayloadLayout(cast<EnumMetadata>(enumType));
321323
storeMultiPayloadTag(value, layout, ~(tag - 1));

‎stdlib/public/runtime/EnumImpl.h

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#define SWIFT_RUNTIME_ENUMIMPL_H
1818

1919
#include "swift/ABI/Enum.h"
20+
#include "swift/Runtime/Enum.h"
2021

2122
namespace swift {
2223

@@ -63,8 +64,8 @@ static inline void small_memset(void *dest, uint8_t value, unsigned count) {
6364

6465
inline unsigned getEnumTagSinglePayloadImpl(
6566
const OpaqueValue *enumAddr, unsigned emptyCases, const Metadata *payload,
66-
size_t payloadSize, size_t payloadNumExtraInhabitants,
67-
unsigned (*getExtraInhabitantTag)(const OpaqueValue *, const Metadata *)) {
67+
size_t payloadSize, unsigned payloadNumExtraInhabitants,
68+
getExtraInhabitantTag_t *getExtraInhabitantTag) {
6869

6970
// If there are extra tag bits, check them.
7071
if (emptyCases > payloadNumExtraInhabitants) {
@@ -116,7 +117,7 @@ inline unsigned getEnumTagSinglePayloadImpl(
116117

117118
// If there are extra inhabitants, see whether the payload is valid.
118119
if (payloadNumExtraInhabitants > 0) {
119-
return getExtraInhabitantTag(enumAddr, payload);
120+
return getExtraInhabitantTag(enumAddr, payloadNumExtraInhabitants, payload);
120121
}
121122

122123
// Otherwise, we have always have a valid payload.
@@ -126,9 +127,8 @@ inline unsigned getEnumTagSinglePayloadImpl(
126127
inline void storeEnumTagSinglePayloadImpl(
127128
OpaqueValue *value, unsigned whichCase, unsigned emptyCases,
128129
const Metadata *payload, size_t payloadSize,
129-
size_t payloadNumExtraInhabitants,
130-
void (*storeExtraInhabitantTag)(OpaqueValue *, unsigned whichCase,
131-
const Metadata *)) {
130+
unsigned payloadNumExtraInhabitants,
131+
storeExtraInhabitantTag_t *storeExtraInhabitantTag) {
132132

133133
auto *valueAddr = reinterpret_cast<uint8_t *>(value);
134134
auto *extraTagBitAddr = valueAddr + payloadSize;
@@ -150,7 +150,8 @@ inline void storeEnumTagSinglePayloadImpl(
150150
return;
151151

152152
// Store the extra inhabitant.
153-
storeExtraInhabitantTag(value, whichCase, payload);
153+
storeExtraInhabitantTag(value, whichCase, payloadNumExtraInhabitants,
154+
payload);
154155
return;
155156
}
156157

0 commit comments

Comments
 (0)