Skip to content

Commit 3844cb2

Browse files
mundaymmundaym
authored
Merge pull request #26144 from linux-on-ibm-z/runtime-enum-simplifications
Runtime: simplify loading and storing enum values
2 parents 419e97d + 71fa7ec commit 3844cb2

File tree

2 files changed

+99
-136
lines changed

2 files changed

+99
-136
lines changed

‎stdlib/public/runtime/Enum.cpp

Lines changed: 18 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -233,74 +233,39 @@ static MultiPayloadLayout getMultiPayloadLayout(const EnumMetadata *enumType) {
233233
static void storeMultiPayloadTag(OpaqueValue *value,
234234
MultiPayloadLayout layout,
235235
unsigned tag) {
236-
auto tagBytes = reinterpret_cast<char *>(value) + layout.payloadSize;
237-
#if defined(__BIG_ENDIAN__)
238-
small_memcpy(tagBytes,
239-
reinterpret_cast<char *>(&tag) + 4 - layout.numTagBytes,
240-
layout.numTagBytes);
241-
#else
242-
small_memcpy(tagBytes, &tag, layout.numTagBytes);
243-
#endif
236+
auto tagBytes = reinterpret_cast<uint8_t *>(value) + layout.payloadSize;
237+
storeEnumElement(tagBytes, tag, layout.numTagBytes);
244238
}
245239

246240
static void storeMultiPayloadValue(OpaqueValue *value,
247241
MultiPayloadLayout layout,
248242
unsigned payloadValue) {
249-
auto bytes = reinterpret_cast<char *>(value);
250-
#if defined(__BIG_ENDIAN__)
251-
unsigned numPayloadValueBytes =
252-
std::min(layout.payloadSize, sizeof(payloadValue));
253-
memcpy(bytes + sizeof(OpaqueValue *) - numPayloadValueBytes,
254-
reinterpret_cast<char *>(&payloadValue) + 4 - numPayloadValueBytes,
255-
numPayloadValueBytes);
256-
if (layout.payloadSize > sizeof(payloadValue) &&
257-
layout.payloadSize > sizeof(OpaqueValue *)) {
258-
memset(bytes, 0,
259-
sizeof(OpaqueValue *) - numPayloadValueBytes);
260-
memset(bytes + sizeof(OpaqueValue *), 0,
261-
layout.payloadSize - sizeof(OpaqueValue *));
262-
}
263-
#else
264-
memcpy(bytes, &payloadValue,
265-
std::min(layout.payloadSize, sizeof(payloadValue)));
266-
267-
// If the payload is larger than the value, zero out the rest.
268-
if (layout.payloadSize > sizeof(payloadValue))
269-
memset(bytes + sizeof(payloadValue), 0,
270-
layout.payloadSize - sizeof(payloadValue));
271-
#endif
243+
auto bytes = reinterpret_cast<uint8_t *>(value);
244+
storeEnumElement(bytes, payloadValue, layout.payloadSize);
272245
}
273246

274247
static unsigned loadMultiPayloadTag(const OpaqueValue *value,
275248
MultiPayloadLayout layout,
276249
unsigned baseValue = 0) {
277-
auto tagBytes = reinterpret_cast<const char *>(value) + layout.payloadSize;
278-
279-
unsigned tag = baseValue;
280-
#if defined(__BIG_ENDIAN__)
281-
small_memcpy(reinterpret_cast<char *>(&tag) + 4 - layout.numTagBytes,
282-
tagBytes, layout.numTagBytes);
283-
#else
284-
small_memcpy(&tag, tagBytes, layout.numTagBytes);
285-
#endif
250+
auto tagBytes = reinterpret_cast<const uint8_t *>(value) +
251+
layout.payloadSize;
252+
auto tag = loadEnumElement(tagBytes, layout.numTagBytes);
253+
254+
// The maximum number of extra tag bytes is 4.
255+
// Note: return early to avoid shifting baseValue by 32 which is
256+
// undefined behaviour.
257+
if (layout.numTagBytes == 4) {
258+
return tag;
259+
}
286260

287-
return tag;
261+
// Replace out-of-range bytes with the base value.
262+
return tag | (baseValue & (~0u << (layout.numTagBytes * 8)));
288263
}
289264

290265
static unsigned loadMultiPayloadValue(const OpaqueValue *value,
291266
MultiPayloadLayout layout) {
292-
auto bytes = reinterpret_cast<const char *>(value);
293-
unsigned payloadValue = 0;
294-
#if defined(__BIG_ENDIAN__)
295-
unsigned numPayloadValueBytes =
296-
std::min(layout.payloadSize, sizeof(payloadValue));
297-
memcpy(reinterpret_cast<char *>(&payloadValue) + 4 - numPayloadValueBytes,
298-
bytes + sizeof(OpaqueValue *) - numPayloadValueBytes, numPayloadValueBytes);
299-
#else
300-
memcpy(&payloadValue, bytes,
301-
std::min(layout.payloadSize, sizeof(payloadValue)));
302-
#endif
303-
return payloadValue;
267+
auto bytes = reinterpret_cast<const uint8_t *>(value);
268+
return loadEnumElement(bytes, layout.payloadSize);
304269
}
305270

306271
SWIFT_CC(swift)

‎stdlib/public/runtime/EnumImpl.h

Lines changed: 81 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -21,45 +21,85 @@
2121

2222
namespace swift {
2323

24-
/// This is a small and fast implementation of memcpy with a constant count. It
25-
/// should be a performance win for small constant values where the function
26-
/// can be inlined, the loop unrolled and the memory accesses merged.
27-
template <unsigned count> static void small_memcpy(void *dest, const void *src) {
28-
uint8_t *d8 = (uint8_t*)dest;
29-
const uint8_t *s8 = (const uint8_t*)src;
30-
for (unsigned i = 0; i < count; i++) {
31-
*d8++ = *s8++;
32-
}
33-
}
34-
35-
static inline void small_memcpy(void *dest, const void *src, unsigned count,
36-
bool countMaybeThree = false) {
37-
// This is specialization of the memcpy line below with
38-
// specialization for values of 1, 2 and 4.
39-
// memcpy(dst, src, count)
40-
if (count == 1) {
41-
small_memcpy<1>(dest, src);
42-
} else if (count == 2) {
43-
small_memcpy<2>(dest, src);
44-
} else if (countMaybeThree && count == 3) {
45-
small_memcpy<3>(dest, src);
46-
} else if (count == 4) {
47-
small_memcpy<4>(dest, src);
48-
} else {
49-
swift::crash("Tagbyte values should be 1, 2 or 4.");
24+
/// Store the given 4-byte unsigned integer value into the variable-
25+
/// length destination buffer. The value will be zero extended or
26+
/// truncated to fit in the buffer.
27+
static inline void storeEnumElement(uint8_t *dst,
28+
uint32_t value,
29+
size_t size) {
30+
// Note: we use fixed size memcpys to encourage the compiler to
31+
// optimize them into unaligned stores.
32+
switch (size) {
33+
case 0:
34+
return;
35+
case 1:
36+
dst[0] = uint8_t(value);
37+
return;
38+
case 2:
39+
#if defined(__BIG_ENDIAN__)
40+
value <<= 16;
41+
#endif
42+
memcpy(dst, &value, 2);
43+
return;
44+
case 3:
45+
#if defined(__BIG_ENDIAN__)
46+
value <<= 8;
47+
#endif
48+
memcpy(dst, &value, 3);
49+
return;
50+
case 4:
51+
memcpy(dst, &value, 4);
52+
return;
5053
}
54+
// Store value into first word of destination. Set the rest of the
55+
// destination to zero.
56+
// NOTE: this is likely to change on big-endian systems.
57+
#if defined(__BIG_ENDIAN__) && defined(__LP64__)
58+
memset(&dst[0], 0, size);
59+
memcpy(&dst[std::min(size - 4, size_t(4))], &value, 4);
60+
#else
61+
memcpy(&dst[0], &value, 4);
62+
memset(&dst[4], 0, size - 4);
63+
#endif
5164
}
5265

53-
static inline void small_memset(void *dest, uint8_t value, unsigned count) {
54-
if (count == 1) {
55-
memset(dest, value, 1);
56-
} else if (count == 2) {
57-
memset(dest, value, 2);
58-
} else if (count == 4) {
59-
memset(dest, value, 4);
60-
} else {
61-
swift::crash("Tagbyte values should be 1, 2 or 4.");
66+
/// Load a 4-byte unsigned integer value from the variable-length
67+
/// source buffer. The value will be zero-extended or truncated to fit
68+
/// into the returned value.
69+
static inline uint32_t loadEnumElement(const uint8_t *src,
70+
size_t size) {
71+
// Note: we use fixed size memcpys to encourage the compiler to
72+
// optimize them into unaligned loads.
73+
uint32_t result = 0;
74+
switch (size) {
75+
case 0:
76+
return 0;
77+
case 1:
78+
return uint32_t(src[0]);
79+
case 2:
80+
memcpy(&result, src, 2);
81+
#if defined(__BIG_ENDIAN__)
82+
result >>= 16;
83+
#endif
84+
return result;
85+
case 3:
86+
memcpy(&result, src, 3);
87+
#if defined(__BIG_ENDIAN__)
88+
result >>= 8;
89+
#endif
90+
return result;
91+
case 4:
92+
memcpy(&result, src, 4);
93+
return result;
6294
}
95+
// Load value from the first word of the source.
96+
// NOTE: this is likely to change on big-endian systems.
97+
#if defined(__BIG_ENDIAN__) && defined(__LP64__)
98+
memcpy(&result, &src[std::min(size - 4, size_t(4))], 4);
99+
#else
100+
memcpy(&result, &src[0], 4);
101+
#endif
102+
return result;
63103
}
64104

65105
inline unsigned getEnumTagSinglePayloadImpl(
@@ -71,43 +111,20 @@ inline unsigned getEnumTagSinglePayloadImpl(
71111
if (emptyCases > payloadNumExtraInhabitants) {
72112
auto *valueAddr = reinterpret_cast<const uint8_t *>(enumAddr);
73113
auto *extraTagBitAddr = valueAddr + payloadSize;
74-
unsigned extraTagBits = 0;
75114
unsigned numBytes =
76115
getEnumTagCounts(payloadSize,
77116
emptyCases - payloadNumExtraInhabitants,
78117
1 /*payload case*/).numTagBytes;
79118

80-
#if defined(__BIG_ENDIAN__)
81-
small_memcpy(reinterpret_cast<uint8_t *>(&extraTagBits) + 4 - numBytes,
82-
extraTagBitAddr, numBytes);
83-
#else
84-
small_memcpy(&extraTagBits, extraTagBitAddr, numBytes);
85-
#endif
119+
unsigned extraTagBits = loadEnumElement(extraTagBitAddr, numBytes);
86120

87121
// If the extra tag bits are zero, we have a valid payload or
88122
// extra inhabitant (checked below). If nonzero, form the case index from
89123
// the extra tag value and the value stored in the payload.
90124
if (extraTagBits > 0) {
91125
unsigned caseIndexFromExtraTagBits =
92126
payloadSize >= 4 ? 0 : (extraTagBits - 1U) << (payloadSize * 8U);
93-
94-
#if defined(__BIG_ENDIAN__)
95-
// On BE high order bytes contain the index
96-
unsigned long caseIndexFromValue = 0;
97-
unsigned numPayloadTagBytes = std::min(size_t(8), payloadSize);
98-
if (numPayloadTagBytes)
99-
memcpy(reinterpret_cast<uint8_t *>(&caseIndexFromValue) + 8 -
100-
numPayloadTagBytes,
101-
valueAddr, numPayloadTagBytes);
102-
#else
103-
// In practice we should need no more than four bytes from the payload
104-
// area.
105-
unsigned caseIndexFromValue = 0;
106-
unsigned numPayloadTagBytes = std::min(size_t(4), payloadSize);
107-
if (numPayloadTagBytes)
108-
small_memcpy(&caseIndexFromValue, valueAddr,
109-
numPayloadTagBytes, true);
110-
#endif
127+
unsigned caseIndexFromValue = loadEnumElement(valueAddr, payloadSize);
111128
unsigned noPayloadIndex =
112129
(caseIndexFromExtraTagBits | caseIndexFromValue) +
113130
payloadNumExtraInhabitants;
@@ -143,7 +160,7 @@ inline void storeEnumTagSinglePayloadImpl(
143160
// if any.
144161
if (whichCase <= payloadNumExtraInhabitants) {
145162
if (numExtraTagBytes != 0)
146-
small_memset(extraTagBitAddr, 0, numExtraTagBytes);
163+
storeEnumElement(extraTagBitAddr, 0, numExtraTagBytes);
147164

148165
// If this is the payload case, we're done.
149166
if (whichCase == 0)
@@ -169,29 +186,10 @@ inline void storeEnumTagSinglePayloadImpl(
169186
}
170187

171188
// Store into the value.
172-
#if defined(__BIG_ENDIAN__)
173-
uint64_t payloadIndexBuf = static_cast<uint64_t>(payloadIndex);
174-
if (payloadSize >= 8) {
175-
memcpy(valueAddr, &payloadIndexBuf, 8);
176-
memset(valueAddr + 8, 0, payloadSize - 8);
177-
} else if (payloadSize > 0) {
178-
payloadIndexBuf <<= (8 - payloadSize) * 8;
179-
memcpy(valueAddr, &payloadIndexBuf, payloadSize);
180-
}
181-
if (numExtraTagBytes)
182-
small_memcpy(extraTagBitAddr,
183-
reinterpret_cast<uint8_t *>(&extraTagIndex) + 4 -
184-
numExtraTagBytes,
185-
numExtraTagBytes);
186-
#else
187-
unsigned numPayloadTagBytes = std::min(size_t(4), payloadSize);
188-
if (numPayloadTagBytes)
189-
small_memcpy(valueAddr, &payloadIndex, numPayloadTagBytes, true);
190-
if (payloadSize > 4)
191-
memset(valueAddr + 4, 0, payloadSize - 4);
189+
if (payloadSize)
190+
storeEnumElement(valueAddr, payloadIndex, payloadSize);
192191
if (numExtraTagBytes)
193-
small_memcpy(extraTagBitAddr, &extraTagIndex, numExtraTagBytes);
194-
#endif
192+
storeEnumElement(extraTagBitAddr, extraTagIndex, numExtraTagBytes);
195193
}
196194

197195
} /* end namespace swift */

0 commit comments

Comments
 (0)