Skip to content

Commit d093339

Browse files
SamTebbs33SamTebbs33
committed
Rework lowering location
1 parent ec8f183 commit d093339

File tree

12 files changed

+358
-238
lines changed

12 files changed

+358
-238
lines changed

‎llvm/include/llvm/CodeGen/ISDOpcodes.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1484,6 +1484,11 @@ enum NodeType {
14841484
// Operands: Mask
14851485
VECTOR_FIND_LAST_ACTIVE,
14861486

1487+
// The `llvm.experimental.get.alias.lane.mask.*` intrinsics
1488+
// Operands: Load pointer, Store pointer, Element size, Write after read
1489+
// Output: Mask
1490+
EXPERIMENTAL_ALIAS_LANE_MASK,
1491+
14871492
// llvm.clear_cache intrinsic
14881493
// Operands: Input Chain, Start Addres, End Address
14891494
// Outputs: Output Chain

‎llvm/include/llvm/IR/IntrinsicsAArch64.td

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2862,14 +2862,8 @@ def int_aarch64_sve_stnt1_pn_x4 : SVE2p1_Store_PN_X4_Intrinsic;
28622862
// SVE2 - Contiguous conflict detection
28632863
//
28642864

2865-
def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2866-
def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2867-
def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2868-
def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2869-
def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2870-
def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2871-
def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2872-
def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
2865+
def int_aarch64_sve_whilerw : SVE2_CONFLICT_DETECT_Intrinsic;
2866+
def int_aarch64_sve_whilewr : SVE2_CONFLICT_DETECT_Intrinsic;
28732867

28742868
// Scalable Matrix Extension (SME) Intrinsics
28752869
let TargetPrefix = "aarch64" in {

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
5555
N->dump(&DAG); dbgs() << "\n";
5656
#endif
5757
report_fatal_error("Do not know how to promote this operator!");
58+
case ISD::EXPERIMENTAL_ALIAS_LANE_MASK:
59+
Res = PromoteIntRes_EXPERIMENTAL_ALIAS_LANE_MASK(N);
60+
break;
5861
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
5962
case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
6063
case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
@@ -359,6 +362,14 @@ SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
359362
return GetPromotedInteger(Op);
360363
}
361364

365+
SDValue
366+
DAGTypeLegalizer::PromoteIntRes_EXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N) {
367+
EVT VT = N->getValueType(0);
368+
EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
369+
return DAG.getNode(ISD::EXPERIMENTAL_ALIAS_LANE_MASK, SDLoc(N), NewVT,
370+
N->ops());
371+
}
372+
362373
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
363374
// Sign-extend the new bits, and continue the assertion.
364375
SDValue Op = SExtPromotedInteger(N->getOperand(0));
@@ -2076,6 +2087,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
20762087
case ISD::VECTOR_FIND_LAST_ACTIVE:
20772088
Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(N, OpNo);
20782089
break;
2090+
case ISD::EXPERIMENTAL_ALIAS_LANE_MASK:
2091+
Res = DAGTypeLegalizer::PromoteIntOp_EXPERIMENTAL_ALIAS_LANE_MASK(N, OpNo);
2092+
break;
20792093
}
20802094

20812095
// If the result is null, the sub-method took care of registering results etc.
@@ -2824,6 +2838,14 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N,
28242838
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
28252839
}
28262840

2841+
SDValue
2842+
DAGTypeLegalizer::PromoteIntOp_EXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N,
2843+
unsigned OpNo) {
2844+
SmallVector<SDValue, 4> NewOps(N->ops());
2845+
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2846+
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2847+
}
2848+
28272849
//===----------------------------------------------------------------------===//
28282850
// Integer Result Expansion
28292851
//===----------------------------------------------------------------------===//

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
379379
SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
380380
SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
381381
SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
382+
SDValue PromoteIntRes_EXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N);
382383

383384
// Integer Operand Promotion.
384385
bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
@@ -429,7 +430,11 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
429430
SDValue PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo);
430431
SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo);
431432
SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo);
433+
<<<<<<< HEAD
432434
SDValue PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N, unsigned OpNo);
435+
=======
436+
SDValue PromoteIntOp_EXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N, unsigned OpNo);
437+
>>>>>>> 24d33ca82b86 (Rework lowering location)
433438

434439
void SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS);
435440
void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ class VectorLegalizer {
138138
SDValue ExpandVP_FNEG(SDNode *Node);
139139
SDValue ExpandVP_FABS(SDNode *Node);
140140
SDValue ExpandVP_FCOPYSIGN(SDNode *Node);
141+
SDValue ExpandEXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N);
141142
SDValue ExpandSELECT(SDNode *Node);
142143
std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
143144
SDValue ExpandStore(SDNode *N);
@@ -467,6 +468,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
467468
case ISD::VECTOR_COMPRESS:
468469
case ISD::SCMP:
469470
case ISD::UCMP:
471+
case ISD::EXPERIMENTAL_ALIAS_LANE_MASK:
470472
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
471473
break;
472474
case ISD::SMULFIX:
@@ -1233,6 +1235,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
12331235
case ISD::UCMP:
12341236
Results.push_back(TLI.expandCMP(Node, DAG));
12351237
return;
1238+
case ISD::EXPERIMENTAL_ALIAS_LANE_MASK:
1239+
Results.push_back(ExpandEXPERIMENTAL_ALIAS_LANE_MASK(Node));
1240+
return;
12361241

12371242
case ISD::FADD:
12381243
case ISD::FMUL:
@@ -1740,6 +1745,42 @@ SDValue VectorLegalizer::ExpandVP_FCOPYSIGN(SDNode *Node) {
17401745
return DAG.getNode(ISD::BITCAST, DL, VT, CopiedSign);
17411746
}
17421747

1748+
SDValue VectorLegalizer::ExpandEXPERIMENTAL_ALIAS_LANE_MASK(SDNode *N) {
1749+
SDLoc DL(N);
1750+
SDValue SourceValue = N->getOperand(0);
1751+
SDValue SinkValue = N->getOperand(1);
1752+
SDValue EltSize = N->getOperand(2);
1753+
1754+
bool IsWriteAfterRead =
1755+
cast<ConstantSDNode>(N->getOperand(3))->getZExtValue() != 0;
1756+
auto VT = N->getValueType(0);
1757+
auto PtrVT = SourceValue->getValueType(0);
1758+
1759+
SDValue Diff = DAG.getNode(ISD::SUB, DL, PtrVT, SinkValue, SourceValue);
1760+
if (!IsWriteAfterRead)
1761+
Diff = DAG.getNode(ISD::ABS, DL, PtrVT, Diff);
1762+
1763+
Diff = DAG.getNode(ISD::SDIV, DL, PtrVT, Diff, EltSize);
1764+
SDValue Zero = DAG.getTargetConstant(0, DL, PtrVT);
1765+
1766+
// If the difference is positive then some elements may alias
1767+
auto CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1768+
Diff.getValueType());
1769+
SDValue Cmp = DAG.getSetCC(DL, CmpVT, Diff, Zero,
1770+
IsWriteAfterRead ? ISD::SETLE : ISD::SETEQ);
1771+
1772+
EVT SplatTY =
1773+
EVT::getVectorVT(*DAG.getContext(), PtrVT, VT.getVectorElementCount());
1774+
SDValue DiffSplat = DAG.getSplat(SplatTY, DL, Diff);
1775+
SDValue VectorStep = DAG.getStepVector(DL, SplatTY);
1776+
SDValue DiffMask =
1777+
DAG.getSetCC(DL, VT, VectorStep, DiffSplat, ISD::CondCode::SETULT);
1778+
1779+
// Splat the compare result then OR it with a lane mask
1780+
SDValue Splat = DAG.getSplat(VT, DL, Cmp);
1781+
return DAG.getNode(ISD::OR, DL, VT, DiffMask, Splat);
1782+
}
1783+
17431784
void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
17441785
SmallVectorImpl<SDValue> &Results) {
17451786
// Attempt to expand using TargetLowering.

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 6 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8277,54 +8277,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
82778277
return;
82788278
}
82798279
case Intrinsic::experimental_get_alias_lane_mask: {
8280-
SDValue SourceValue = getValue(I.getOperand(0));
8281-
SDValue SinkValue = getValue(I.getOperand(1));
8282-
SDValue EltSize = getValue(I.getOperand(2));
8283-
bool IsWriteAfterRead =
8284-
cast<ConstantSDNode>(getValue(I.getOperand(3)))->getZExtValue() != 0;
82858280
auto IntrinsicVT = EVT::getEVT(I.getType());
8286-
auto PtrVT = SourceValue->getValueType(0);
8287-
8288-
if (!TLI.shouldExpandGetAliasLaneMask(
8289-
IntrinsicVT, PtrVT,
8290-
cast<ConstantSDNode>(EltSize)->getSExtValue())) {
8291-
visitTargetIntrinsic(I, Intrinsic);
8292-
return;
8293-
}
8294-
8295-
SDValue Diff = DAG.getNode(ISD::SUB, sdl, PtrVT, SinkValue, SourceValue);
8296-
if (!IsWriteAfterRead)
8297-
Diff = DAG.getNode(ISD::ABS, sdl, PtrVT, Diff);
8298-
8299-
Diff = DAG.getNode(ISD::SDIV, sdl, PtrVT, Diff, EltSize);
8300-
SDValue Zero = DAG.getTargetConstant(0, sdl, PtrVT);
8301-
8302-
// If the difference is positive then some elements may alias
8303-
auto CmpVT =
8304-
TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), PtrVT);
8305-
SDValue Cmp = DAG.getSetCC(sdl, CmpVT, Diff, Zero,
8306-
IsWriteAfterRead ? ISD::SETLE : ISD::SETEQ);
8307-
8308-
// Splat the compare result then OR it with a lane mask
8309-
SDValue Splat = DAG.getSplat(IntrinsicVT, sdl, Cmp);
8310-
8311-
SDValue DiffMask;
8312-
// Don't emit an active lane mask if the target doesn't support it
8313-
if (TLI.shouldExpandGetActiveLaneMask(IntrinsicVT, PtrVT)) {
8314-
EVT VecTy = EVT::getVectorVT(*DAG.getContext(), PtrVT,
8315-
IntrinsicVT.getVectorElementCount());
8316-
SDValue DiffSplat = DAG.getSplat(VecTy, sdl, Diff);
8317-
SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8318-
DiffMask = DAG.getSetCC(sdl, IntrinsicVT, VectorStep, DiffSplat,
8319-
ISD::CondCode::SETULT);
8320-
} else {
8321-
DiffMask = DAG.getNode(
8322-
ISD::INTRINSIC_WO_CHAIN, sdl, IntrinsicVT,
8323-
DAG.getTargetConstant(Intrinsic::get_active_lane_mask, sdl, MVT::i64),
8324-
Zero, Diff);
8325-
}
8326-
SDValue Or = DAG.getNode(ISD::OR, sdl, IntrinsicVT, DiffMask, Splat);
8327-
setValue(&I, Or);
8281+
SmallVector<SDValue, 4> Ops;
8282+
for (auto &Op : I.operands())
8283+
Ops.push_back(getValue(Op));
8284+
SDValue Mask =
8285+
DAG.getNode(ISD::EXPERIMENTAL_ALIAS_LANE_MASK, sdl, IntrinsicVT, Ops);
8286+
setValue(&I, Mask);
83288287
}
83298288
}
83308289
}

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
570570
case ISD::VECTOR_FIND_LAST_ACTIVE:
571571
return "find_last_active";
572572

573+
case ISD::EXPERIMENTAL_ALIAS_LANE_MASK:
574+
return "alias_mask";
575+
573576
// Vector Predication
574577
#define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...) \
575578
case ISD::SDID: \

‎llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -821,6 +821,9 @@ void TargetLoweringBase::initActions() {
821821
// Masked vector extracts default to expand.
822822
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Expand);
823823

824+
// Aliasing lanes mask default to expand
825+
setOperationAction(ISD::EXPERIMENTAL_ALIAS_LANE_MASK, VT, Expand);
826+
824827
// FP environment operations default to expand.
825828
setOperationAction(ISD::GET_FPENV, VT, Expand);
826829
setOperationAction(ISD::SET_FPENV, VT, Expand);

0 commit comments

Comments
 (0)