|
26 | 26 | #include "llvm/Support/CommandLine.h"
|
27 | 27 | #include "llvm/Support/Compiler.h"
|
28 | 28 | #include "llvm/Support/Debug.h"
|
| 29 | +#include "llvm/Transforms/Utils/LoopUtils.h" |
29 | 30 | #include <optional>
|
30 | 31 |
|
31 | 32 | using namespace llvm;
|
@@ -437,69 +438,33 @@ CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
|
437 | 438 | default:
|
438 | 439 | llvm_unreachable("Impossible reduction kind");
|
439 | 440 | case Intrinsic::vp_reduce_add:
|
440 |
| - Reduction = Builder.CreateAddReduce(RedOp); |
441 |
| - Reduction = Builder.CreateAdd(Reduction, Start); |
442 |
| - break; |
443 | 441 | case Intrinsic::vp_reduce_mul:
|
444 |
| - Reduction = Builder.CreateMulReduce(RedOp); |
445 |
| - Reduction = Builder.CreateMul(Reduction, Start); |
446 |
| - break; |
447 | 442 | case Intrinsic::vp_reduce_and:
|
448 |
| - Reduction = Builder.CreateAndReduce(RedOp); |
449 |
| - Reduction = Builder.CreateAnd(Reduction, Start); |
450 |
| - break; |
451 | 443 | case Intrinsic::vp_reduce_or:
|
452 |
| - Reduction = Builder.CreateOrReduce(RedOp); |
453 |
| - Reduction = Builder.CreateOr(Reduction, Start); |
454 |
| - break; |
455 |
| - case Intrinsic::vp_reduce_xor: |
456 |
| - Reduction = Builder.CreateXorReduce(RedOp); |
457 |
| - Reduction = Builder.CreateXor(Reduction, Start); |
458 |
| - break; |
459 |
| - case Intrinsic::vp_reduce_smax: |
460 |
| - Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true); |
| 444 | + case Intrinsic::vp_reduce_xor: { |
| 445 | + Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID(); |
| 446 | + unsigned Opc = getArithmeticReductionInstruction(RedID); |
| 447 | + assert(Instruction::isBinaryOp(Opc)); |
| 448 | + Reduction = Builder.CreateUnaryIntrinsic(RedID, RedOp); |
461 | 449 | Reduction =
|
462 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start); |
| 450 | + Builder.CreateBinOp((Instruction::BinaryOps)Opc, Reduction, Start); |
463 | 451 | break;
|
| 452 | + } |
| 453 | + case Intrinsic::vp_reduce_smax: |
464 | 454 | case Intrinsic::vp_reduce_smin:
|
465 |
| - Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true); |
466 |
| - Reduction = |
467 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start); |
468 |
| - break; |
469 | 455 | case Intrinsic::vp_reduce_umax:
|
470 |
| - Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false); |
471 |
| - Reduction = |
472 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start); |
473 |
| - break; |
474 | 456 | case Intrinsic::vp_reduce_umin:
|
475 |
| - Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false); |
476 |
| - Reduction = |
477 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start); |
478 |
| - break; |
479 | 457 | case Intrinsic::vp_reduce_fmax:
|
480 |
| - Reduction = Builder.CreateFPMaxReduce(RedOp); |
481 |
| - transferDecorations(*Reduction, VPI); |
482 |
| - Reduction = |
483 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start); |
484 |
| - break; |
485 | 458 | case Intrinsic::vp_reduce_fmin:
|
486 |
| - Reduction = Builder.CreateFPMinReduce(RedOp); |
487 |
| - transferDecorations(*Reduction, VPI); |
488 |
| - Reduction = |
489 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start); |
490 |
| - break; |
491 | 459 | case Intrinsic::vp_reduce_fmaximum:
|
492 |
| - Reduction = Builder.CreateFPMaximumReduce(RedOp); |
493 |
| - transferDecorations(*Reduction, VPI); |
494 |
| - Reduction = |
495 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::maximum, Reduction, Start); |
496 |
| - break; |
497 |
| - case Intrinsic::vp_reduce_fminimum: |
498 |
| - Reduction = Builder.CreateFPMinimumReduce(RedOp); |
| 460 | + case Intrinsic::vp_reduce_fminimum: { |
| 461 | + Intrinsic::ID RedID = *VPI.getFunctionalIntrinsicID(); |
| 462 | + Intrinsic::ID ScalarID = getMinMaxReductionIntrinsicOp(RedID); |
| 463 | + Reduction = Builder.CreateUnaryIntrinsic(RedID, RedOp); |
499 | 464 | transferDecorations(*Reduction, VPI);
|
500 |
| - Reduction = |
501 |
| - Builder.CreateBinaryIntrinsic(Intrinsic::minimum, Reduction, Start); |
| 465 | + Reduction = Builder.CreateBinaryIntrinsic(ScalarID, Reduction, Start); |
502 | 466 | break;
|
| 467 | + } |
503 | 468 | case Intrinsic::vp_reduce_fadd:
|
504 | 469 | Reduction = Builder.CreateFAddReduce(Start, RedOp);
|
505 | 470 | break;
|
|
0 commit comments