Skip to content

[RISCV] Support isel for Zacas for 2*XLen types. #77814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[RISCV] Support isel for Zacas for 2*XLen types. #77814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 124 additions & 2 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
}

if (Subtarget.hasStdExtA()) {
setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
unsigned MaxAtomicSize = Subtarget.getXLen();
if (Subtarget.hasStdExtZacas())
MaxAtomicSize = 2 * Subtarget.getXLen();
setMaxAtomicSizeInBitsSupported(MaxAtomicSize);
setMinCmpXchgSizeInBits(32);
} else if (Subtarget.hasForcedAtomics()) {
setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
Expand Down Expand Up @@ -1338,6 +1341,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
if (RV64LegalI32 && Subtarget.is64Bit())
setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
if (Subtarget.hasStdExtZacas())
setOperationAction(ISD::ATOMIC_CMP_SWAP,
Subtarget.is64Bit() ? MVT::i128 : MVT::i64, Custom);
}

if (Subtarget.hasForcedAtomics()) {
Expand Down Expand Up @@ -11237,13 +11243,86 @@ static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
}

// Create an even/odd pair of X registers holding integer value V.
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V, MVT VT,
MVT SubRegVT) {
SDLoc DL(V.getNode());
auto [VLo, VHi] = DAG.SplitScalar(V, DL, SubRegVT, SubRegVT);
SDValue RegClass =
DAG.getTargetConstant(RISCV::GPRPairRegClassID, DL, MVT::i32);
SDValue SubReg0 = DAG.getTargetConstant(RISCV::sub_gpr_even, DL, MVT::i32);
SDValue SubReg1 = DAG.getTargetConstant(RISCV::sub_gpr_odd, DL, MVT::i32);
const SDValue Ops[] = {RegClass, VLo, SubReg0, VHi, SubReg1};
return SDValue(
DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops), 0);
}

static void ReplaceCMP_SWAP_2XLenResults(SDNode *N,
SmallVectorImpl<SDValue> &Results,
SelectionDAG &DAG,
const RISCVSubtarget &Subtarget) {
MVT VT = N->getSimpleValueType(0);
assert(N->getValueType(0) == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) &&
"AtomicCmpSwap on types less than 2*XLen should be legal");
assert(Subtarget.hasStdExtZacas());
MVT XLenVT = Subtarget.getXLenVT();

SDValue Ops[] = {
createGPRPairNode(DAG, N->getOperand(2), VT, XLenVT), // Compare value
N->getOperand(1), // Ptr
createGPRPairNode(DAG, N->getOperand(3), VT, XLenVT), // Store value
N->getOperand(0), // Chain in
};

MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();

bool Is64Bit = Subtarget.is64Bit();
unsigned Opcode;
if (Subtarget.hasStdExtZtso()) {
Opcode = Subtarget.is64Bit() ? RISCV::AMOCAS_Q : RISCV::AMOCAS_D_RV32;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Opcode = Subtarget.is64Bit() ? RISCV::AMOCAS_Q : RISCV::AMOCAS_D_RV32;
Opcode = Is64Bit ? RISCV::AMOCAS_Q : RISCV::AMOCAS_D_RV32;

} else {
switch (MemOp->getMergedOrdering()) {
default:
llvm_unreachable("Unexpected ordering!");
case AtomicOrdering::Monotonic:
Opcode = Is64Bit ? RISCV::AMOCAS_Q : RISCV::AMOCAS_D_RV32;
break;
case AtomicOrdering::Acquire:
Opcode = Is64Bit ? RISCV::AMOCAS_Q_AQ : RISCV::AMOCAS_D_RV32_AQ;
break;
case AtomicOrdering::Release:
Opcode = Is64Bit ? RISCV::AMOCAS_Q_RL : RISCV::AMOCAS_D_RV32_RL;
break;
case AtomicOrdering::AcquireRelease:
case AtomicOrdering::SequentiallyConsistent:
Opcode = Is64Bit ? RISCV::AMOCAS_Q_AQ_RL : RISCV::AMOCAS_D_RV32_AQ_RL;
break;
}
}

SDLoc DL(N);
MachineSDNode *CmpSwap = DAG.getMachineNode(
Opcode, DL, DAG.getVTList(MVT::Untyped, MVT::Other), Ops);
DAG.setNodeMemRefs(CmpSwap, {MemOp});

SDValue Lo = DAG.getTargetExtractSubreg(RISCV::sub_gpr_even, DL, XLenVT,
SDValue(CmpSwap, 0));
SDValue Hi = DAG.getTargetExtractSubreg(RISCV::sub_gpr_odd, DL, XLenVT,
SDValue(CmpSwap, 0));
Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, VT, Lo, Hi));
Results.push_back(SDValue(CmpSwap, 1));
}

void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SmallVectorImpl<SDValue> &Results,
SelectionDAG &DAG) const {
SDLoc DL(N);
switch (N->getOpcode()) {
default:
llvm_unreachable("Don't know how to custom type legalize this operation!");
case ISD::ATOMIC_CMP_SWAP:
ReplaceCMP_SWAP_2XLenResults(N, Results, DAG, Subtarget);
break;
case ISD::STRICT_FP_TO_SINT:
case ISD::STRICT_FP_TO_UINT:
case ISD::FP_TO_SINT:
Expand Down Expand Up @@ -19003,6 +19082,20 @@ void RISCVTargetLowering::LowerAsmOperandForConstraint(
TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
}

bool RISCVTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
// We don't need a fence for 2*Xlen. We can use Zacas.
if (auto *LI = dyn_cast<LoadInst>(I))
return LI->getType()->getPrimitiveSizeInBits() != 2 * Subtarget.getXLen();

// We don't need a fence for 2*Xlen. We can use Zacas.
if (auto *SI = dyn_cast<StoreInst>(I))
return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() !=
2 * Subtarget.getXLen();

return false;
}

Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
Expand Down Expand Up @@ -19036,6 +19129,30 @@ Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
return nullptr;
}

TargetLowering::AtomicExpansionKind
RISCVTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
unsigned Size = LI->getType()->getPrimitiveSizeInBits();

if (Size != 2 * Subtarget.getXLen())
return AtomicExpansionKind::None;

// With Zacas we can use amocas for 2*XLen types.
assert(Subtarget.hasStdExtZacas() && "Unexpected extension");
return AtomicExpansionKind::CmpXChg;
}

TargetLowering::AtomicExpansionKind
RISCVTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();

if (Size != 2 * Subtarget.getXLen())
return AtomicExpansionKind::None;

// With Zacas we can use amocas for 2*XLen types.
assert(Subtarget.hasStdExtZacas() && "Unexpected extension");
return AtomicExpansionKind::Expand;
}

TargetLowering::AtomicExpansionKind
RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
Expand All @@ -19053,7 +19170,12 @@ RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
unsigned Size = AI->getType()->getPrimitiveSizeInBits();
if (Size == 8 || Size == 16)
return AtomicExpansionKind::MaskedIntrinsic;
return AtomicExpansionKind::None;
if (Size != 2 * Subtarget.getXLen())
return AtomicExpansionKind::None;

// With Zacas we can use amocas for 2*XLen types.
assert(Subtarget.hasStdExtZacas() && "Unexpected extension");
return AtomicExpansionKind::CmpXChg;
}

static Intrinsic::ID
Expand Down
8 changes: 5 additions & 3 deletions llvm/lib/Target/RISCV/RISCVISelLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,9 +618,7 @@ class RISCVTargetLowering : public TargetLowering {

bool preferZeroCompareBranch() const override { return true; }

bool shouldInsertFencesForAtomic(const Instruction *I) const override {
return isa<LoadInst>(I) || isa<StoreInst>(I);
}
bool shouldInsertFencesForAtomic(const Instruction *I) const override;
Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
AtomicOrdering Ord) const override;
Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
Expand Down Expand Up @@ -699,6 +697,10 @@ class RISCVTargetLowering : public TargetLowering {
bool isMulAddWithConstProfitable(SDValue AddNode,
SDValue ConstNode) const override;

TargetLoweringBase::AtomicExpansionKind
shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
TargetLoweringBase::AtomicExpansionKind
shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
TargetLowering::AtomicExpansionKind
shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
Value *emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI,
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,8 +754,9 @@ bool RISCVRegisterInfo::getRegAllocationHints(
bool NeedGPRC) -> void {
Register Reg = MO.getReg();
Register PhysReg = Reg.isPhysical() ? Reg : Register(VRM->getPhys(Reg));
if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg))) {
assert(!MO.getSubReg() && !VRRegMO.getSubReg() && "Unexpected subreg!");
// TODO: Add hints when there are GPRPair subregs?
if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg)) &&
!MO.getSubReg() && !VRRegMO.getSubReg()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to see test changes caused by this.

if (!MRI->isReserved(PhysReg) && !is_contained(Hints, PhysReg))
TwoAddrHints.insert(PhysReg);
}
Expand Down
Loading