Skip to content

Commit 21b0f34

Browse files
mshockwavemshockwave
committed
[RISCV][LoopIdiomVectorize] Support VP intrinsics in LoopIdiomVectorize
Teach LoopIdiomVectorize to use VP intrinsics to replace the byte compare loops. Right now only RISC-V uses LoopIdiomVectorize of this style.
1 parent c0b7092 commit 21b0f34

File tree

6 files changed

+1948
-14
lines changed

6 files changed

+1948
-14
lines changed

‎llvm/include/llvm/Transforms/Vectorize/LoopIdiomVectorize.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,22 @@
1313
#include "llvm/Transforms/Scalar/LoopPassManager.h"
1414

1515
namespace llvm {
16-
struct LoopIdiomVectorizePass : PassInfoMixin<LoopIdiomVectorizePass> {
16+
enum class LoopIdiomVectorizeStyle { Masked, Predicated };
17+
18+
class LoopIdiomVectorizePass : public PassInfoMixin<LoopIdiomVectorizePass> {
19+
LoopIdiomVectorizeStyle VectorizeStyle = LoopIdiomVectorizeStyle::Masked;
20+
21+
// The VF used in vectorizing the byte compare pattern.
22+
unsigned ByteCompareVF = 16;
23+
24+
public:
25+
LoopIdiomVectorizePass() = default;
26+
explicit LoopIdiomVectorizePass(LoopIdiomVectorizeStyle S)
27+
: VectorizeStyle(S) {}
28+
29+
LoopIdiomVectorizePass(LoopIdiomVectorizeStyle S, unsigned BCVF)
30+
: VectorizeStyle(S), ByteCompareVF(BCVF) {}
31+
1732
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
1833
LoopStandardAnalysisResults &AR, LPMUpdater &U);
1934
};

‎llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@
3333
#include "llvm/CodeGen/TargetPassConfig.h"
3434
#include "llvm/InitializePasses.h"
3535
#include "llvm/MC/TargetRegistry.h"
36+
#include "llvm/Passes/PassBuilder.h"
3637
#include "llvm/Support/FormattedStream.h"
3738
#include "llvm/Target/TargetOptions.h"
3839
#include "llvm/Transforms/IPO.h"
3940
#include "llvm/Transforms/Scalar.h"
41+
#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
4042
#include <optional>
4143
using namespace llvm;
4244

@@ -573,6 +575,14 @@ void RISCVPassConfig::addPostRegAlloc() {
573575
addPass(createRISCVRedundantCopyEliminationPass());
574576
}
575577

578+
void RISCVTargetMachine::registerPassBuilderCallbacks(
579+
PassBuilder &PB, bool PopulateClassToPassNames) {
580+
PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,
581+
OptimizationLevel Level) {
582+
LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
583+
});
584+
}
585+
576586
yaml::MachineFunctionInfo *
577587
RISCVTargetMachine::createDefaultFuncInfoYAML() const {
578588
return new yaml::RISCVMachineFunctionInfo();

‎llvm/lib/Target/RISCV/RISCVTargetMachine.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ class RISCVTargetMachine : public LLVMTargetMachine {
5959
PerFunctionMIParsingState &PFS,
6060
SMDiagnostic &Error,
6161
SMRange &SourceRange) const override;
62+
void registerPassBuilderCallbacks(PassBuilder &PB,
63+
bool PopulateClassToPassNames) override;
6264
};
6365
} // namespace llvm
6466

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
397397
bool shouldFoldTerminatingConditionAfterLSR() const {
398398
return true;
399399
}
400+
401+
std::optional<unsigned> getMinPageSize() const { return 4096; }
400402
};
401403

402404
} // end namespace llvm

‎llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp

Lines changed: 167 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -59,19 +59,34 @@ static cl::opt<bool> DisableAll("disable-loop-idiom-vectorize-all", cl::Hidden,
5959
cl::init(false),
6060
cl::desc("Disable Loop Idiom Vectorize Pass."));
6161

62+
static cl::opt<LoopIdiomVectorizeStyle>
63+
LITVecStyle("loop-idiom-vectorize-style", cl::Hidden,
64+
cl::desc("The vectorization style for loop idiom transform."),
65+
cl::values(clEnumValN(LoopIdiomVectorizeStyle::Masked, "masked",
66+
"Use masked vector intrinsics"),
67+
clEnumValN(LoopIdiomVectorizeStyle::Predicated,
68+
"predicated", "Use VP intrinsics")),
69+
cl::init(LoopIdiomVectorizeStyle::Masked));
70+
6271
static cl::opt<bool>
6372
DisableByteCmp("disable-loop-idiom-vectorize-bytecmp", cl::Hidden,
6473
cl::init(false),
6574
cl::desc("Proceed with Loop Idiom Vectorize Pass, but do "
6675
"not convert byte-compare loop(s)."));
6776

77+
static cl::opt<unsigned>
78+
ByteCmpVF("loop-idiom-vectorize-bytecmp-vf", cl::Hidden,
79+
cl::desc("The vectorization factor for byte-compare patterns."),
80+
cl::init(16));
81+
6882
static cl::opt<bool>
6983
VerifyLoops("loop-idiom-vectorize-verify", cl::Hidden, cl::init(false),
7084
cl::desc("Verify loops generated Loop Idiom Vectorize Pass."));
7185

7286
namespace {
73-
7487
class LoopIdiomVectorize {
88+
LoopIdiomVectorizeStyle VectorizeStyle;
89+
unsigned ByteCompareVF;
7590
Loop *CurLoop = nullptr;
7691
DominatorTree *DT;
7792
LoopInfo *LI;
@@ -86,10 +101,11 @@ class LoopIdiomVectorize {
86101
BasicBlock *VectorLoopIncBlock = nullptr;
87102

88103
public:
89-
explicit LoopIdiomVectorize(DominatorTree *DT, LoopInfo *LI,
90-
const TargetTransformInfo *TTI,
91-
const DataLayout *DL)
92-
: DT(DT), LI(LI), TTI(TTI), DL(DL) {}
104+
LoopIdiomVectorize(LoopIdiomVectorizeStyle S, unsigned VF, DominatorTree *DT,
105+
LoopInfo *LI, const TargetTransformInfo *TTI,
106+
const DataLayout *DL)
107+
: VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL) {
108+
}
93109

94110
bool run(Loop *L);
95111

@@ -111,6 +127,10 @@ class LoopIdiomVectorize {
111127
GetElementPtrInst *GEPA,
112128
GetElementPtrInst *GEPB, Value *ExtStart,
113129
Value *ExtEnd);
130+
Value *createPredicatedFindMismatch(IRBuilder<> &Builder, DomTreeUpdater &DTU,
131+
GetElementPtrInst *GEPA,
132+
GetElementPtrInst *GEPB, Value *ExtStart,
133+
Value *ExtEnd);
114134

115135
void transformByteCompare(GetElementPtrInst *GEPA, GetElementPtrInst *GEPB,
116136
PHINode *IndPhi, Value *MaxLen, Instruction *Index,
@@ -128,8 +148,16 @@ PreservedAnalyses LoopIdiomVectorizePass::run(Loop &L, LoopAnalysisManager &AM,
128148

129149
const auto *DL = &L.getHeader()->getModule()->getDataLayout();
130150

131-
LoopIdiomVectorize LIT(&AR.DT, &AR.LI, &AR.TTI, DL);
132-
if (!LIT.run(&L))
151+
LoopIdiomVectorizeStyle VecStyle = VectorizeStyle;
152+
if (LITVecStyle.getNumOccurrences())
153+
VecStyle = LITVecStyle;
154+
155+
unsigned BCVF = ByteCompareVF;
156+
if (ByteCmpVF.getNumOccurrences())
157+
BCVF = ByteCmpVF;
158+
159+
LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL);
160+
if (!LIV.run(&L))
133161
return PreservedAnalyses::all();
134162

135163
return PreservedAnalyses::none();
@@ -360,14 +388,15 @@ Value *LoopIdiomVectorize::createMaskedFindMismatch(
360388
// Therefore, we know that we can use a 64-bit induction variable that
361389
// starts from 0 -> ExtMaxLen and it will not overflow.
362390
ScalableVectorType *PredVTy =
363-
ScalableVectorType::get(Builder.getInt1Ty(), 16);
391+
ScalableVectorType::get(Builder.getInt1Ty(), ByteCompareVF);
364392

365393
Value *InitialPred = Builder.CreateIntrinsic(
366394
Intrinsic::get_active_lane_mask, {PredVTy, I64Type}, {ExtStart, ExtEnd});
367395

368396
Value *VecLen = Builder.CreateIntrinsic(Intrinsic::vscale, {I64Type}, {});
369-
VecLen = Builder.CreateMul(VecLen, ConstantInt::get(I64Type, 16), "",
370-
/*HasNUW=*/true, /*HasNSW=*/true);
397+
VecLen =
398+
Builder.CreateMul(VecLen, ConstantInt::get(I64Type, ByteCompareVF), "",
399+
/*HasNUW=*/true, /*HasNSW=*/true);
371400

372401
Value *PFalse = Builder.CreateVectorSplat(PredVTy->getElementCount(),
373402
Builder.getInt1(false));
@@ -385,7 +414,8 @@ Value *LoopIdiomVectorize::createMaskedFindMismatch(
385414
LoopPred->addIncoming(InitialPred, VectorLoopPreheaderBlock);
386415
PHINode *VectorIndexPhi = Builder.CreatePHI(I64Type, 2, "mismatch_vec_index");
387416
VectorIndexPhi->addIncoming(ExtStart, VectorLoopPreheaderBlock);
388-
Type *VectorLoadType = ScalableVectorType::get(Builder.getInt8Ty(), 16);
417+
Type *VectorLoadType =
418+
ScalableVectorType::get(Builder.getInt8Ty(), ByteCompareVF);
389419
Value *Passthru = ConstantInt::getNullValue(VectorLoadType);
390420

391421
Value *VectorLhsGep =
@@ -454,6 +484,121 @@ Value *LoopIdiomVectorize::createMaskedFindMismatch(
454484
return Builder.CreateTrunc(VectorLoopRes64, ResType);
455485
}
456486

487+
Value *LoopIdiomVectorize::createPredicatedFindMismatch(
488+
IRBuilder<> &Builder, DomTreeUpdater &DTU, GetElementPtrInst *GEPA,
489+
GetElementPtrInst *GEPB, Value *ExtStart, Value *ExtEnd) {
490+
Type *I64Type = Builder.getInt64Ty();
491+
Type *I32Type = Builder.getInt32Ty();
492+
Type *ResType = I32Type;
493+
Type *LoadType = Builder.getInt8Ty();
494+
Value *PtrA = GEPA->getPointerOperand();
495+
Value *PtrB = GEPB->getPointerOperand();
496+
497+
// At this point we know two things must be true:
498+
// 1. Start <= End
499+
// 2. ExtMaxLen <= 4096 due to the page checks.
500+
// Therefore, we know that we can use a 64-bit induction variable that
501+
// starts from 0 -> ExtMaxLen and it will not overflow.
502+
auto *JumpToVectorLoop = BranchInst::Create(VectorLoopStartBlock);
503+
Builder.Insert(JumpToVectorLoop);
504+
505+
DTU.applyUpdates({{DominatorTree::Insert, VectorLoopPreheaderBlock,
506+
VectorLoopStartBlock}});
507+
508+
// Set up the first Vector loop block by creating the PHIs, doing the vector
509+
// loads and comparing the vectors.
510+
Builder.SetInsertPoint(VectorLoopStartBlock);
511+
auto *VectorIndexPhi = Builder.CreatePHI(I64Type, 2, "mismatch_vector_index");
512+
VectorIndexPhi->addIncoming(ExtStart, VectorLoopPreheaderBlock);
513+
514+
// Calculate AVL by subtracting the vector loop index from the trip count
515+
Value *AVL = Builder.CreateSub(ExtEnd, VectorIndexPhi, "avl", /*HasNUW=*/true,
516+
/*HasNSW=*/true);
517+
518+
auto *VectorLoadType = ScalableVectorType::get(LoadType, ByteCompareVF);
519+
auto *VF = ConstantInt::get(
520+
I32Type, VectorLoadType->getElementCount().getKnownMinValue());
521+
auto *IsScalable = ConstantInt::getBool(
522+
Builder.getContext(), VectorLoadType->getElementCount().isScalable());
523+
524+
Value *VL = Builder.CreateIntrinsic(Intrinsic::experimental_get_vector_length,
525+
{I64Type}, {AVL, VF, IsScalable});
526+
Value *GepOffset = VectorIndexPhi;
527+
528+
Value *VectorLhsGep = Builder.CreateGEP(LoadType, PtrA, GepOffset);
529+
if (GEPA->isInBounds())
530+
cast<GetElementPtrInst>(VectorLhsGep)->setIsInBounds(true);
531+
VectorType *TrueMaskTy =
532+
VectorType::get(Builder.getInt1Ty(), VectorLoadType->getElementCount());
533+
Value *AllTrueMask = Constant::getAllOnesValue(TrueMaskTy);
534+
Value *VectorLhsLoad = Builder.CreateIntrinsic(
535+
Intrinsic::vp_load, {VectorLoadType, VectorLhsGep->getType()},
536+
{VectorLhsGep, AllTrueMask, VL}, nullptr, "lhs.load");
537+
538+
Value *VectorRhsGep = Builder.CreateGEP(LoadType, PtrB, GepOffset);
539+
if (GEPB->isInBounds())
540+
cast<GetElementPtrInst>(VectorRhsGep)->setIsInBounds(true);
541+
Value *VectorRhsLoad = Builder.CreateIntrinsic(
542+
Intrinsic::vp_load, {VectorLoadType, VectorLhsGep->getType()},
543+
{VectorRhsGep, AllTrueMask, VL}, nullptr, "rhs.load");
544+
545+
StringRef PredicateStr = CmpInst::getPredicateName(CmpInst::ICMP_NE);
546+
auto *PredicateMDS = MDString::get(VectorLhsLoad->getContext(), PredicateStr);
547+
Value *Pred = MetadataAsValue::get(VectorLhsLoad->getContext(), PredicateMDS);
548+
Value *VectorMatchCmp = Builder.CreateIntrinsic(
549+
Intrinsic::vp_icmp, {VectorLhsLoad->getType()},
550+
{VectorLhsLoad, VectorRhsLoad, Pred, AllTrueMask, VL}, nullptr,
551+
"mismatch.cmp");
552+
Value *CTZ = Builder.CreateIntrinsic(
553+
Intrinsic::vp_cttz_elts, {ResType, VectorMatchCmp->getType()},
554+
{VectorMatchCmp, /*ZeroIsPoison=*/Builder.getInt1(true), AllTrueMask,
555+
VL});
556+
// RISC-V refines/lowers the poison returned by vp.cttz.elts to -1.
557+
Value *MismatchFound =
558+
Builder.CreateICmpSGE(CTZ, ConstantInt::get(ResType, 0));
559+
auto *VectorEarlyExit = BranchInst::Create(VectorLoopMismatchBlock,
560+
VectorLoopIncBlock, MismatchFound);
561+
Builder.Insert(VectorEarlyExit);
562+
563+
DTU.applyUpdates(
564+
{{DominatorTree::Insert, VectorLoopStartBlock, VectorLoopMismatchBlock},
565+
{DominatorTree::Insert, VectorLoopStartBlock, VectorLoopIncBlock}});
566+
567+
// Increment the index counter and calculate the predicate for the next
568+
// iteration of the loop. We branch back to the start of the loop if there
569+
// is at least one active lane.
570+
Builder.SetInsertPoint(VectorLoopIncBlock);
571+
Value *VL64 = Builder.CreateZExt(VL, I64Type);
572+
Value *NewVectorIndexPhi =
573+
Builder.CreateAdd(VectorIndexPhi, VL64, "",
574+
/*HasNUW=*/true, /*HasNSW=*/true);
575+
VectorIndexPhi->addIncoming(NewVectorIndexPhi, VectorLoopIncBlock);
576+
Value *ExitCond = Builder.CreateICmpNE(NewVectorIndexPhi, ExtEnd);
577+
auto *VectorLoopBranchBack =
578+
BranchInst::Create(VectorLoopStartBlock, EndBlock, ExitCond);
579+
Builder.Insert(VectorLoopBranchBack);
580+
581+
DTU.applyUpdates(
582+
{{DominatorTree::Insert, VectorLoopIncBlock, VectorLoopStartBlock},
583+
{DominatorTree::Insert, VectorLoopIncBlock, EndBlock}});
584+
585+
// If we found a mismatch then we need to calculate which lane in the vector
586+
// had a mismatch and add that on to the current loop index.
587+
Builder.SetInsertPoint(VectorLoopMismatchBlock);
588+
589+
// Add LCSSA phis for CTZ and VectorIndexPhi.
590+
auto *CTZLCSSAPhi = Builder.CreatePHI(CTZ->getType(), 1, "ctz");
591+
CTZLCSSAPhi->addIncoming(CTZ, VectorLoopStartBlock);
592+
auto *VectorIndexLCSSAPhi =
593+
Builder.CreatePHI(VectorIndexPhi->getType(), 1, "mismatch_vector_index");
594+
VectorIndexLCSSAPhi->addIncoming(VectorIndexPhi, VectorLoopStartBlock);
595+
596+
Value *CTZI64 = Builder.CreateZExt(CTZLCSSAPhi, I64Type);
597+
Value *VectorLoopRes64 = Builder.CreateAdd(VectorIndexLCSSAPhi, CTZI64, "",
598+
/*HasNUW=*/true, /*HasNSW=*/true);
599+
return Builder.CreateTrunc(VectorLoopRes64, ResType);
600+
}
601+
457602
Value *LoopIdiomVectorize::expandFindMismatch(
458603
IRBuilder<> &Builder, DomTreeUpdater &DTU, GetElementPtrInst *GEPA,
459604
GetElementPtrInst *GEPB, Instruction *Index, Value *Start, Value *MaxLen) {
@@ -613,8 +758,17 @@ Value *LoopIdiomVectorize::expandFindMismatch(
613758
// processed in each iteration, etc.
614759
Builder.SetInsertPoint(VectorLoopPreheaderBlock);
615760

616-
Value *VectorLoopRes =
617-
createMaskedFindMismatch(Builder, DTU, GEPA, GEPB, ExtStart, ExtEnd);
761+
Value *VectorLoopRes = nullptr;
762+
switch (VectorizeStyle) {
763+
case LoopIdiomVectorizeStyle::Masked:
764+
VectorLoopRes =
765+
createMaskedFindMismatch(Builder, DTU, GEPA, GEPB, ExtStart, ExtEnd);
766+
break;
767+
case LoopIdiomVectorizeStyle::Predicated:
768+
VectorLoopRes = createPredicatedFindMismatch(Builder, DTU, GEPA, GEPB,
769+
ExtStart, ExtEnd);
770+
break;
771+
}
618772

619773
Builder.Insert(BranchInst::Create(EndBlock));
620774

0 commit comments

Comments
 (0)