@@ -554,7 +554,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
554
554
return IndexCost +
555
555
getRISCVInstructionCost (RISCV::VRGATHER_VV, LT.second , CostKind);
556
556
}
557
- [[fallthrough]] ;
557
+ break ;
558
558
}
559
559
case TTI::SK_Transpose:
560
560
case TTI::SK_PermuteTwoSrc: {
@@ -574,55 +574,65 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
574
574
LT.second , CostKind) +
575
575
MaskCost;
576
576
}
577
- [[fallthrough]];
578
- }
579
- case TTI::SK_Select: {
580
- // We are going to permute multiple sources and the result will be in
581
- // multiple destinations. Providing an accurate cost only for splits where
582
- // the element type remains the same.
583
- if (!Mask.empty () && LT.first .isValid () && LT.first != 1 &&
584
- LT.second .getVectorElementType ().getSizeInBits () ==
585
- Tp->getElementType ()->getPrimitiveSizeInBits () &&
586
- LT.second .getVectorNumElements () <
587
- cast<FixedVectorType>(Tp)->getNumElements () &&
588
- divideCeil (Mask.size (),
589
- cast<FixedVectorType>(Tp)->getNumElements ()) ==
590
- static_cast <unsigned >(*LT.first .getValue ())) {
591
- unsigned NumRegs = *LT.first .getValue ();
592
- unsigned VF = cast<FixedVectorType>(Tp)->getNumElements ();
593
- unsigned SubVF = PowerOf2Ceil (VF / NumRegs);
594
- auto *SubVecTy = FixedVectorType::get (Tp->getElementType (), SubVF);
595
-
596
- InstructionCost Cost = 0 ;
597
- for (unsigned I = 0 , NumSrcRegs = divideCeil (Mask.size (), SubVF);
598
- I < NumSrcRegs; ++I) {
599
- bool IsSingleVector = true ;
600
- SmallVector<int > SubMask (SubVF, PoisonMaskElem);
601
- transform (
602
- Mask.slice (I * SubVF,
603
- I == NumSrcRegs - 1 ? Mask.size () % SubVF : SubVF),
604
- SubMask.begin (), [&](int I) -> int {
605
- if (I == PoisonMaskElem)
606
- return PoisonMaskElem;
607
- bool SingleSubVector = I / VF == 0 ;
608
- IsSingleVector &= SingleSubVector;
609
- return (SingleSubVector ? 0 : 1 ) * SubVF + (I % VF) % SubVF;
610
- });
611
- if (all_of (enumerate(SubMask), [](auto &&P) {
612
- return P.value () == PoisonMaskElem ||
613
- static_cast <unsigned >(P.value ()) == P.index ();
614
- }))
615
- continue ;
616
- Cost += getShuffleCost (IsSingleVector ? TTI::SK_PermuteSingleSrc
617
- : TTI::SK_PermuteTwoSrc,
618
- SubVecTy, SubMask, CostKind, 0 , nullptr );
619
- }
620
- return Cost;
621
- }
622
577
break ;
623
578
}
624
579
}
625
- };
580
+
581
+ auto shouldSplit = [](TTI::ShuffleKind Kind) {
582
+ switch (Kind) {
583
+ default :
584
+ return false ;
585
+ case TTI::SK_PermuteSingleSrc:
586
+ case TTI::SK_Transpose:
587
+ case TTI::SK_PermuteTwoSrc:
588
+ case TTI::SK_Select:
589
+ return true ;
590
+ }
591
+ };
592
+ // We are going to permute multiple sources and the result will be in
593
+ // multiple destinations. Providing an accurate cost only for splits where
594
+ // the element type remains the same.
595
+ if (!Mask.empty () && LT.first .isValid () && LT.first != 1 &&
596
+ shouldSplit (Kind) &&
597
+ LT.second .getVectorElementType ().getSizeInBits () ==
598
+ Tp->getElementType ()->getPrimitiveSizeInBits () &&
599
+ LT.second .getVectorNumElements () <
600
+ cast<FixedVectorType>(Tp)->getNumElements () &&
601
+ divideCeil (Mask.size (),
602
+ cast<FixedVectorType>(Tp)->getNumElements ()) ==
603
+ static_cast <unsigned >(*LT.first .getValue ())) {
604
+ unsigned NumRegs = *LT.first .getValue ();
605
+ unsigned VF = cast<FixedVectorType>(Tp)->getNumElements ();
606
+ unsigned SubVF = PowerOf2Ceil (VF / NumRegs);
607
+ auto *SubVecTy = FixedVectorType::get (Tp->getElementType (), SubVF);
608
+
609
+ InstructionCost Cost = 0 ;
610
+ for (unsigned I = 0 , NumSrcRegs = divideCeil (Mask.size (), SubVF);
611
+ I < NumSrcRegs; ++I) {
612
+ bool IsSingleVector = true ;
613
+ SmallVector<int > SubMask (SubVF, PoisonMaskElem);
614
+ transform (
615
+ Mask.slice (I * SubVF,
616
+ I == NumSrcRegs - 1 ? Mask.size () % SubVF : SubVF),
617
+ SubMask.begin (), [&](int I) -> int {
618
+ if (I == PoisonMaskElem)
619
+ return PoisonMaskElem;
620
+ bool SingleSubVector = I / VF == 0 ;
621
+ IsSingleVector &= SingleSubVector;
622
+ return (SingleSubVector ? 0 : 1 ) * SubVF + (I % VF) % SubVF;
623
+ });
624
+ if (all_of (enumerate(SubMask), [](auto &&P) {
625
+ return P.value () == PoisonMaskElem ||
626
+ static_cast <unsigned >(P.value ()) == P.index ();
627
+ }))
628
+ continue ;
629
+ Cost += getShuffleCost (IsSingleVector ? TTI::SK_PermuteSingleSrc
630
+ : TTI::SK_PermuteTwoSrc,
631
+ SubVecTy, SubMask, CostKind, 0 , nullptr );
632
+ }
633
+ return Cost;
634
+ }
635
+ }
626
636
627
637
// Handle scalable vectors (and fixed vectors legalized to scalable vectors).
628
638
switch (Kind) {
0 commit comments