Skip to content

[CodeGen] Port InterleavedAccess to new pass manager #74904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/CallBrPrepare.h"
#include "llvm/CodeGen/ExpandReductions.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/ReplaceWithVeclib.h"
Expand Down
34 changes: 34 additions & 0 deletions llvm/include/llvm/CodeGen/InterleavedAccess.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//===---- llvm/CodeGen/InterleavedAccess.h ----------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file contains the declaration of the InterleavedAccessPass class,
/// its corresponding pass name is `interleaved-access`.
///
//===----------------------------------------------------------------------===//

#ifndef LLVM_CODEGEN_INTERLEAVEDACCESS_H
#define LLVM_CODEGEN_INTERLEAVEDACCESS_H

#include "llvm/IR/PassManager.h"

namespace llvm {

class TargetMachine;

class InterleavedAccessPass : public PassInfoMixin<InterleavedAccessPass> {
const TargetMachine *TM;

public:
explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {}
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
};

} // namespace llvm

#endif // LLVM_CODEGEN_INTERLEAVEDACCESS_H
2 changes: 1 addition & 1 deletion llvm/include/llvm/CodeGen/MachinePassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ())
FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ())
FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ())
FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM))
FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ())
FUNCTION_PASS("lowerinvoke", LowerInvokePass, ())
FUNCTION_PASS("mergeicmps", MergeICmpsPass, ())
Expand Down Expand Up @@ -127,7 +128,6 @@ DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ())
DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ())
DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ())
DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ())
DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ())
Expand Down
98 changes: 65 additions & 33 deletions llvm/lib/CodeGen/InterleavedAccessPass.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
Expand Down Expand Up @@ -82,22 +83,14 @@ static cl::opt<bool> LowerInterleavedAccesses(

namespace {

class InterleavedAccess : public FunctionPass {
public:
static char ID;

InterleavedAccess() : FunctionPass(ID) {
initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
}
class InterleavedAccessImpl {
friend class InterleavedAccess;

StringRef getPassName() const override { return "Interleaved Access Pass"; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<DominatorTreeWrapperPass>();
AU.setPreservesCFG();
}
public:
InterleavedAccessImpl() = default;
InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI)
: DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {}
bool runOnFunction(Function &F);

private:
DominatorTree *DT = nullptr;
Expand Down Expand Up @@ -141,10 +134,60 @@ class InterleavedAccess : public FunctionPass {
LoadInst *LI);
};

class InterleavedAccess : public FunctionPass {
InterleavedAccessImpl Impl;

public:
static char ID;

InterleavedAccess() : FunctionPass(ID) {
initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
}

StringRef getPassName() const override { return "Interleaved Access Pass"; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<DominatorTreeWrapperPass>();
AU.setPreservesCFG();
}
};

} // end anonymous namespace.

PreservedAnalyses InterleavedAccessPass::run(Function &F,
FunctionAnalysisManager &FAM) {
auto *DT = &FAM.getResult<DominatorTreeAnalysis>(F);
auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering();
InterleavedAccessImpl Impl(DT, TLI);
bool Changed = Impl.runOnFunction(F);

if (!Changed)
return PreservedAnalyses::all();

PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
return PA;
}

char InterleavedAccess::ID = 0;

bool InterleavedAccess::runOnFunction(Function &F) {
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC || !LowerInterleavedAccesses)
return false;

LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This kind of printing I think is redundant with regular pass debugging statements

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just copy the old code here.


Impl.DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
auto &TM = TPC->getTM<TargetMachine>();
Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering();
Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor();

return Impl.runOnFunction(F);
}

INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE,
"Lower interleaved memory accesses to target specific intrinsics", false,
false)
Expand Down Expand Up @@ -228,7 +271,7 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor,
return false;
}

bool InterleavedAccess::lowerInterleavedLoad(
bool InterleavedAccessImpl::lowerInterleavedLoad(
LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
return false;
Expand Down Expand Up @@ -334,7 +377,7 @@ bool InterleavedAccess::lowerInterleavedLoad(
return true;
}

bool InterleavedAccess::replaceBinOpShuffles(
bool InterleavedAccessImpl::replaceBinOpShuffles(
ArrayRef<ShuffleVectorInst *> BinOpShuffles,
SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) {
for (auto *SVI : BinOpShuffles) {
Expand Down Expand Up @@ -367,7 +410,7 @@ bool InterleavedAccess::replaceBinOpShuffles(
return !BinOpShuffles.empty();
}

bool InterleavedAccess::tryReplaceExtracts(
bool InterleavedAccessImpl::tryReplaceExtracts(
ArrayRef<ExtractElementInst *> Extracts,
ArrayRef<ShuffleVectorInst *> Shuffles) {
// If there aren't any extractelement instructions to modify, there's nothing
Expand Down Expand Up @@ -431,7 +474,7 @@ bool InterleavedAccess::tryReplaceExtracts(
return true;
}

bool InterleavedAccess::lowerInterleavedStore(
bool InterleavedAccessImpl::lowerInterleavedStore(
StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) {
if (!SI->isSimple())
return false;
Expand All @@ -457,7 +500,7 @@ bool InterleavedAccess::lowerInterleavedStore(
return true;
}

bool InterleavedAccess::lowerDeinterleaveIntrinsic(
bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
IntrinsicInst *DI, SmallVector<Instruction *, 32> &DeadInsts) {
LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));

Expand All @@ -476,7 +519,7 @@ bool InterleavedAccess::lowerDeinterleaveIntrinsic(
return true;
}

bool InterleavedAccess::lowerInterleaveIntrinsic(
bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
IntrinsicInst *II, SmallVector<Instruction *, 32> &DeadInsts) {
if (!II->hasOneUse())
return false;
Expand All @@ -498,18 +541,7 @@ bool InterleavedAccess::lowerInterleaveIntrinsic(
return true;
}

bool InterleavedAccess::runOnFunction(Function &F) {
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC || !LowerInterleavedAccesses)
return false;

LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");

DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
auto &TM = TPC->getTM<TargetMachine>();
TLI = TM.getSubtargetImpl(F)->getTargetLowering();
MaxFactor = TLI->getMaxSupportedInterleaveFactor();

bool InterleavedAccessImpl::runOnFunction(Function &F) {
// Holds dead instructions that will be erased later.
SmallVector<Instruction *, 32> DeadInsts;
bool Changed = false;
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
#include "llvm/CodeGen/ExpandLargeDivRem.h"
#include "llvm/CodeGen/ExpandLargeFpConvert.h"
#include "llvm/CodeGen/HardwareLoops.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/SafeStack.h"
#include "llvm/CodeGen/TypePromotion.h"
#include "llvm/CodeGen/WasmEHPrepare.h"
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings())
FUNCTION_PASS("instcount", InstCountPass())
FUNCTION_PASS("instnamer", InstructionNamerPass())
FUNCTION_PASS("instsimplify", InstSimplifyPass())
FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM))
FUNCTION_PASS("invalidate<all>", InvalidateAllAnalysesPass())
FUNCTION_PASS("irce", IRCEPass())
FUNCTION_PASS("jump-threading", JumpThreadingPass())
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON
; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON
; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED

target triple = "aarch64-linux-gnu"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target triple = "aarch64-linux-gnu"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target triple = "aarch64-linux-gnu"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv32 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV32
; RUN: opt < %s -mtriple=riscv64 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV64
; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV32
; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV64

define void @load_factor2(ptr %ptr) {
; RV32-LABEL: @load_factor2(
Expand Down
2 changes: 2 additions & 0 deletions llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE32X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE64X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE32X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE64X

define <4 x i1> @load_large_vector(ptr %p) {
; ZVE32X-LABEL: @load_large_vector(
Expand Down
2 changes: 2 additions & 0 deletions llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -interleaved-access -S | FileCheck %s -check-prefix=ZVL32B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVL128B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL32B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL128B

; Make sure that we don't lower interleaved loads that won't fit into the minimum vlen

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -interleaved-access -S %s | FileCheck %s
; RUN: opt -passes=interleaved-access -S %s | FileCheck %s

target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.15.0"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s

; This file tests the function `llvm::lowerInterleavedLoad/Store`.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s

; This file tests the function `llvm::lowerInterleavedLoad/Store`.

Expand Down
Loading