Skip to content

Commit 04976e1

Browse files
mundaymmundaym
committed
IRGen: fix get/store enum tag functions for big-endian platforms
Use sized loads and stores, rather than memcpy calls, to get and set tag values in the payload and extra tag bytes. The code works the same regardless of the endianness of the target platform.
1 parent c4626ba commit 04976e1

File tree

1 file changed

+145
-100
lines changed

1 file changed

+145
-100
lines changed

‎lib/IRGen/GenType.cpp

Lines changed: 145 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,110 @@ static llvm::Value *computeExtraTagBytes(IRGenFunction &IGF, IRBuilder &Builder,
426426
return phi;
427427
}
428428

429+
/// Emit a specialized memory operation for a \p size of 0, 1, 2 or
430+
/// 4 bytes.
431+
/// \p emitMemOpFn will be called in a basic block where \p size is
432+
/// a known constant value passed as a parameter to the function.
433+
static void emitSpecializedMemOperation(
434+
IRGenFunction &IGF,
435+
llvm::function_ref<void(IRBuilder &, Size)> emitMemOpFn,
436+
llvm::Value *size) {
437+
auto &Ctx = IGF.IGM.getLLVMContext();
438+
auto &Builder = IGF.Builder;
439+
440+
// Sizes to try. Tested in order.
441+
const auto sizes = std::array<Size, 4>{
442+
Size(0),
443+
Size(1),
444+
Size(2),
445+
Size(4),
446+
};
447+
448+
// Block to jump to after successful match.
449+
auto *returnBB = llvm::BasicBlock::Create(Ctx);
450+
451+
// Test all the sizes in turn, generating code similar to:
452+
//
453+
// if (size == 0) {
454+
// <op>
455+
// } else if (size == 1) {
456+
// <op>
457+
// } else if (size == 2) {
458+
// <op>
459+
// } else if (size == 4) {
460+
// <op>
461+
// } else {
462+
// <unreachable>
463+
// }
464+
//
465+
for (const Size &s : sizes) {
466+
auto *matchBB = llvm::BasicBlock::Create(Ctx);
467+
auto *nextBB = llvm::BasicBlock::Create(Ctx);
468+
469+
// Check if size matches.
470+
auto *imm = Builder.getInt32(s.getValue());
471+
auto *cmp = Builder.CreateICmpEQ(size, imm);
472+
Builder.CreateCondBr(cmp, matchBB, nextBB);
473+
474+
// Size matches: execute sized operation.
475+
Builder.emitBlock(matchBB);
476+
emitMemOpFn(Builder, s);
477+
Builder.CreateBr(returnBB);
478+
479+
// Size does not match: try next size.
480+
Builder.emitBlock(nextBB);
481+
}
482+
// No size matched. Should never happen.
483+
Builder.CreateUnreachable();
484+
485+
// Continue.
486+
Builder.emitBlock(returnBB);
487+
}
488+
489+
/// Emit a load of a \p size byte integer value zero extended to 4 bytes.
490+
/// \p size may be 0, 1, 2 or 4.
491+
static llvm::Value *emitGetTag(IRGenFunction &IGF,
492+
Address from,
493+
llvm::Value *size) {
494+
auto *phi = llvm::PHINode::Create(IGF.IGM.Int32Ty, 4);
495+
emitSpecializedMemOperation(IGF,
496+
[=](IRBuilder &B, Size s) {
497+
if (s == Size(0)) {
498+
// If the size is 0 bytes return 0.
499+
phi->addIncoming(B.getInt32(0), B.GetInsertBlock());
500+
return;
501+
}
502+
// Generate a load of size bytes and zero-extend it to 32-bits.
503+
auto *type = B.getIntNTy(s.getValueInBits());
504+
Address addr = B.CreateBitCast(from, type->getPointerTo());
505+
auto *val = B.CreateZExtOrTrunc(B.CreateLoad(addr), B.getInt32Ty());
506+
phi->addIncoming(val, B.GetInsertBlock());
507+
},
508+
size);
509+
IGF.Builder.Insert(phi);
510+
return phi;
511+
}
512+
513+
/// Emit a store of \p val truncated to \p size bytes.
514+
/// \p size may be 0, 1, 2 or 4.
515+
static void emitSetTag(IRGenFunction &IGF,
516+
Address to, llvm::Value *val,
517+
llvm::Value *size) {
518+
emitSpecializedMemOperation(IGF,
519+
[=](IRBuilder &B, Size s) {
520+
if (s == Size(0)) {
521+
// Nothing to store.
522+
return;
523+
}
524+
// Store value truncated to size bytes.
525+
auto *type = B.getIntNTy(s.getValueInBits());
526+
auto *trunc = B.CreateZExtOrTrunc(val, type);
527+
Address addr = B.CreateBitCast(to, type->getPointerTo());
528+
B.CreateStore(trunc, addr);
529+
},
530+
size);
531+
}
532+
429533
llvm::Value *FixedTypeInfo::getEnumTagSinglePayload(IRGenFunction &IGF,
430534
llvm::Value *numEmptyCases,
431535
Address enumAddr,
@@ -472,26 +576,21 @@ llvm::Value *irgen::getFixedTypeEnumTagSinglePayload(IRGenFunction &IGF,
472576

473577
// There are extra tag bits to check.
474578
Builder.emitBlock(extraTagBitsBB);
475-
Address extraTagBitsSlot = IGF.createAlloca(IGM.Int32Ty, Alignment(4));
476-
Builder.CreateStore(zero, extraTagBitsSlot);
477579

478580
// Compute the number of extra tag bytes.
479581
auto *emptyCases = Builder.CreateSub(numEmptyCases, numExtraInhabitants);
480582
auto *numExtraTagBytes =
481583
computeExtraTagBytes(IGF, Builder, fixedSize, emptyCases);
482584

483-
// Read the extra tag bytes.
585+
// Read the value stored in the extra tag bytes.
484586
auto *valueAddr =
485587
Builder.CreateBitOrPointerCast(enumAddr.getAddress(), IGM.Int8PtrTy);
486588
auto *extraTagBitsAddr =
487589
Builder.CreateConstInBoundsGEP1_32(IGM.Int8Ty, valueAddr,
488-
fixedSize.getValue());
489-
490-
// TODO: big endian.
491-
Builder.CreateMemCpy(
492-
Builder.CreateBitCast(extraTagBitsSlot, IGM.Int8PtrTy).getAddress(), 1,
493-
extraTagBitsAddr, 1, numExtraTagBytes);
494-
auto extraTagBits = Builder.CreateLoad(extraTagBitsSlot);
590+
fixedSize.getValue());
591+
auto *extraTagBits = emitGetTag(IGF,
592+
Address(extraTagBitsAddr, Alignment(1)),
593+
numExtraTagBytes);
495594

496595
extraTagBitsBB = llvm::BasicBlock::Create(Ctx);
497596
Builder.CreateCondBr(Builder.CreateICmpEQ(extraTagBits, zero),
@@ -502,20 +601,25 @@ llvm::Value *irgen::getFixedTypeEnumTagSinglePayload(IRGenFunction &IGF,
502601
Builder.emitBlock(extraTagBitsBB);
503602

504603
auto *truncSize = Builder.CreateZExtOrTrunc(size, IGM.Int32Ty);
505-
Address caseIndexFromValueSlot = IGF.createAlloca(IGM.Int32Ty, Alignment(4));
506-
Builder.CreateStore(zero, caseIndexFromValueSlot);
507604

508605
auto *caseIndexFromExtraTagBits = Builder.CreateSelect(
509606
Builder.CreateICmpUGE(truncSize, four), zero,
510607
Builder.CreateShl(Builder.CreateSub(extraTagBits, one),
511608
Builder.CreateMul(eight, truncSize)));
512609

513-
// TODO: big endian.
514-
Builder.CreateMemCpy(
515-
Builder.CreateBitCast(caseIndexFromValueSlot, IGM.Int8PtrTy),
516-
Address(valueAddr, Alignment(1)),
517-
std::min(Size(4U), fixedSize));
518-
auto caseIndexFromValue = Builder.CreateLoad(caseIndexFromValueSlot);
610+
llvm::Value *caseIndexFromValue = zero;
611+
if (fixedSize > Size(0)) {
612+
// Read up to one pointer-sized 'chunk' of the payload.
613+
// The size of the chunk does not have to be a power of 2.
614+
Size limit = IGM.getPointerSize();
615+
auto *caseIndexType = llvm::IntegerType::get(Ctx,
616+
std::min(limit, fixedSize).getValueInBits());
617+
auto *caseIndexAddr = Builder.CreateBitCast(valueAddr,
618+
caseIndexType->getPointerTo());
619+
caseIndexFromValue = Builder.CreateZExtOrTrunc(
620+
Builder.CreateLoad(Address(caseIndexAddr, Alignment(1))),
621+
IGM.Int32Ty);
622+
}
519623

520624
auto *result1 = Builder.CreateAdd(
521625
numExtraInhabitants,
@@ -549,72 +653,6 @@ llvm::Value *irgen::getFixedTypeEnumTagSinglePayload(IRGenFunction &IGF,
549653
return Builder.CreateAdd(result, llvm::ConstantInt::get(IGM.Int32Ty, 1));
550654
}
551655

552-
/// Emit a speciaize memory operation for a \p size of 0 to 4 bytes.
553-
static void emitSpecializedMemOperation(
554-
IRGenFunction &IGF,
555-
llvm::function_ref<void(IRBuilder &, Size)> emitMemOpFn,
556-
llvm::Value *size) {
557-
auto &IGM = IGF.IGM;
558-
auto &Ctx = IGF.IGM.getLLVMContext();
559-
auto &Builder = IGF.Builder;
560-
auto *returnBB = llvm::BasicBlock::Create(Ctx);
561-
auto *oneBB = llvm::BasicBlock::Create(Ctx);
562-
auto *twoBB = llvm::BasicBlock::Create(Ctx);
563-
auto *fourBB = llvm::BasicBlock::Create(Ctx);
564-
auto *int32Ty = IGM.Int32Ty;
565-
auto *zero = llvm::ConstantInt::get(int32Ty, 0U);
566-
auto *one = llvm::ConstantInt::get(int32Ty, 1U);
567-
auto *two = llvm::ConstantInt::get(int32Ty, 2U);
568-
569-
auto *continueBB = llvm::BasicBlock::Create(Ctx);
570-
auto *isZero = Builder.CreateICmpEQ(size, zero);
571-
Builder.CreateCondBr(isZero, returnBB, continueBB);
572-
573-
Builder.emitBlock(continueBB);
574-
continueBB = llvm::BasicBlock::Create(Ctx);
575-
auto *isOne = Builder.CreateICmpEQ(size, one);
576-
Builder.CreateCondBr(isOne, oneBB, continueBB);
577-
578-
Builder.emitBlock(continueBB);
579-
auto *isTwo = Builder.CreateICmpEQ(size, two);
580-
Builder.CreateCondBr(isTwo, twoBB, fourBB);
581-
582-
Builder.emitBlock(oneBB);
583-
emitMemOpFn(Builder, Size(1));
584-
Builder.CreateBr(returnBB);
585-
586-
Builder.emitBlock(twoBB);
587-
emitMemOpFn(Builder, Size(2));
588-
Builder.CreateBr(returnBB);
589-
590-
Builder.emitBlock(fourBB);
591-
emitMemOpFn(Builder, Size(4));
592-
Builder.CreateBr(returnBB);
593-
594-
Builder.emitBlock(returnBB);
595-
}
596-
597-
/// Emit a memset of zero operation for a \p size of 0 to 4 bytes.
598-
static void emitMemZero(IRGenFunction &IGF, Address addr,
599-
llvm::Value *size) {
600-
auto *zeroByte = llvm::ConstantInt::get(IGF.IGM.Int8Ty, 0U);
601-
emitSpecializedMemOperation(IGF,
602-
[=](IRBuilder &B, Size numBytes) {
603-
B.CreateMemSet(addr, zeroByte, numBytes);
604-
},
605-
size);
606-
}
607-
608-
/// Emit a memcpy operation for a \p size of 0 to 4 bytes.
609-
static void emitMemCpy(IRGenFunction &IGF, Address to, Address from,
610-
llvm::Value *size) {
611-
emitSpecializedMemOperation(IGF,
612-
[=](IRBuilder &B, Size numBytes) {
613-
B.CreateMemCpy(to, from, numBytes);
614-
},
615-
size);
616-
}
617-
618656
void FixedTypeInfo::storeEnumTagSinglePayload(IRGenFunction &IGF,
619657
llvm::Value *whichCase,
620658
llvm::Value *numEmptyCases,
@@ -684,7 +722,7 @@ void irgen::storeFixedTypeEnumTagSinglePayload(IRGenFunction &IGF,
684722
// We are the payload or fit within the extra inhabitants.
685723
Builder.emitBlock(isPayloadOrInhabitantCaseBB);
686724
// Zero the tag bits.
687-
emitMemZero(IGF, extraTagBitsAddr, numExtraTagBytes);
725+
emitSetTag(IGF, extraTagBitsAddr, zero, numExtraTagBytes);
688726
isPayloadOrInhabitantCaseBB = Builder.GetInsertBlock();
689727
auto *storeInhabitantBB = llvm::BasicBlock::Create(Ctx);
690728
auto *returnBB = llvm::BasicBlock::Create(Ctx);
@@ -734,21 +772,28 @@ void irgen::storeFixedTypeEnumTagSinglePayload(IRGenFunction &IGF,
734772
payloadIndex->addIncoming(caseIndex, payloadGE4BB);
735773
payloadIndex->addIncoming(payloadIndex0, payloadLT4BB);
736774

737-
Address payloadIndexAddr = IGF.createAlloca(int32Ty, Alignment(4));
738-
Builder.CreateStore(payloadIndex, payloadIndexAddr);
739-
Address extraTagIndexAddr = IGF.createAlloca(int32Ty, Alignment(4));
740-
Builder.CreateStore(extraTagIndex, extraTagIndexAddr);
741-
// TODO: big endian
742-
Builder.CreateMemCpy(
743-
valueAddr,
744-
Builder.CreateBitCast(payloadIndexAddr, IGM.Int8PtrTy),
745-
std::min(Size(4U), fixedSize));
746-
Address extraZeroAddr = Builder.CreateConstByteArrayGEP(valueAddr, Size(4));
747-
if (fixedSize > Size(4))
748-
Builder.CreateMemSet(
749-
extraZeroAddr, llvm::ConstantInt::get(IGM.Int8Ty, 0),
750-
Builder.CreateSub(size, llvm::ConstantInt::get(size->getType(), 4)));
751-
emitMemCpy(IGF, extraTagBitsAddr, extraTagIndexAddr, numExtraTagBytes);
775+
if (fixedSize > Size(0)) {
776+
// Write the value into the first pointer-sized (or smaller)
777+
// 'chunk' of the payload.
778+
// The size of the chunk does not have to be a power of 2.
779+
Size limit = IGM.getPointerSize();
780+
auto *chunkType = Builder.getIntNTy(
781+
std::min(fixedSize, limit).getValueInBits());
782+
Builder.CreateStore(
783+
Builder.CreateZExtOrTrunc(payloadIndex, chunkType),
784+
Builder.CreateBitCast(valueAddr, chunkType->getPointerTo()));
785+
786+
// Zero the remainder of the payload.
787+
if (fixedSize > limit) {
788+
auto zeroAddr = Builder.CreateConstByteArrayGEP(valueAddr, limit);
789+
auto zeroSize = Builder.CreateSub(
790+
size,
791+
llvm::ConstantInt::get(size->getType(), limit.getValue()));
792+
Builder.CreateMemSet(zeroAddr, Builder.getInt8(0), zeroSize);
793+
}
794+
}
795+
// Write to the extra tag bytes, if any.
796+
emitSetTag(IGF, extraTagBitsAddr, extraTagIndex, numExtraTagBytes);
752797
Builder.CreateBr(returnBB);
753798

754799
Builder.emitBlock(returnBB);

0 commit comments

Comments
 (0)