@@ -19380,43 +19380,59 @@ class HorizontalReduction {
19380
19380
/// Creates reduction operation with the current opcode.
19381
19381
static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
19382
19382
Value *RHS, const Twine &Name, bool UseSelect) {
19383
- if (UseSelect) {
19384
- if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind)) {
19385
- CmpInst::Predicate Pred = llvm::getMinMaxReductionPredicate(Kind);
19386
- Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS, Name);
19387
- return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19388
- }
19389
- if ((Kind == RecurKind::Or || Kind == RecurKind::And) &&
19390
- LHS->getType() == CmpInst::makeCmpResultType(LHS->getType())) {
19391
- Value *TrueVal = Kind == RecurKind::Or ? Builder.getTrue() : RHS;
19392
- Value *FalseVal = Kind == RecurKind::Or ? RHS : Builder.getFalse();
19393
- return Builder.CreateSelect(LHS, TrueVal, FalseVal, Name);
19394
- }
19395
- }
19396
-
19383
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19397
19384
switch (Kind) {
19398
19385
case RecurKind::Or:
19386
+ if (UseSelect &&
19387
+ LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
19388
+ return Builder.CreateSelect(LHS, Builder.getTrue(), RHS, Name);
19389
+ return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
19390
+ Name);
19399
19391
case RecurKind::And:
19392
+ if (UseSelect &&
19393
+ LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
19394
+ return Builder.CreateSelect(LHS, RHS, Builder.getFalse(), Name);
19395
+ return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
19396
+ Name);
19400
19397
case RecurKind::Add:
19401
19398
case RecurKind::Mul:
19402
19399
case RecurKind::Xor:
19403
19400
case RecurKind::FAdd:
19404
- case RecurKind::FMul: {
19405
- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19401
+ case RecurKind::FMul:
19406
19402
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
19407
19403
Name);
19408
- }
19409
19404
case RecurKind::FMax:
19405
+ return Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS);
19410
19406
case RecurKind::FMin:
19407
+ return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
19411
19408
case RecurKind::FMaximum:
19409
+ return Builder.CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS);
19412
19410
case RecurKind::FMinimum:
19411
+ return Builder.CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS);
19413
19412
case RecurKind::SMax:
19413
+ if (UseSelect) {
19414
+ Value *Cmp = Builder.CreateICmpSGT(LHS, RHS, Name);
19415
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19416
+ }
19417
+ return Builder.CreateBinaryIntrinsic(Intrinsic::smax, LHS, RHS);
19414
19418
case RecurKind::SMin:
19419
+ if (UseSelect) {
19420
+ Value *Cmp = Builder.CreateICmpSLT(LHS, RHS, Name);
19421
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19422
+ }
19423
+ return Builder.CreateBinaryIntrinsic(Intrinsic::smin, LHS, RHS);
19415
19424
case RecurKind::UMax:
19416
- case RecurKind::UMin: {
19417
- Intrinsic::ID Id = llvm::getMinMaxReductionIntrinsicOp(Kind);
19418
- return Builder.CreateBinaryIntrinsic(Id, LHS, RHS);
19419
- }
19425
+ if (UseSelect) {
19426
+ Value *Cmp = Builder.CreateICmpUGT(LHS, RHS, Name);
19427
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19428
+ }
19429
+ return Builder.CreateBinaryIntrinsic(Intrinsic::umax, LHS, RHS);
19430
+ case RecurKind::UMin:
19431
+ if (UseSelect) {
19432
+ Value *Cmp = Builder.CreateICmpULT(LHS, RHS, Name);
19433
+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19434
+ }
19435
+ return Builder.CreateBinaryIntrinsic(Intrinsic::umin, LHS, RHS);
19420
19436
default:
19421
19437
llvm_unreachable("Unknown reduction operation.");
19422
19438
}
0 commit comments