Skip to content

Commit 7648629

Browse files
RKSimonRKSimon
committed
[DAG] isKnownNeverNaN - add DemandedElts element mask to isKnownNeverZero calls.
Matches what we've done for computeKnownBits etc. to improve vector handling
1 parent 3d7e56f commit 7648629

File tree

8 files changed

+75
-35
lines changed

8 files changed

+75
-35
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2142,11 +2142,25 @@ class SelectionDAG {
21422142
/// X|Cst == X+Cst iff X&Cst = 0.
21432143
bool isBaseWithConstantOffset(SDValue Op) const;
21442144

2145+
/// Test whether the given SDValue (or all elements of it, if it is a
2146+
/// vector) is known to never be NaN in \p DemandedElts. If \p SNaN is true,
2147+
/// returns if \p Op is known to never be a signaling NaN (it may still be a
2148+
/// qNaN).
2149+
bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN = false,
2150+
unsigned Depth = 0) const;
2151+
21452152
/// Test whether the given SDValue (or all elements of it, if it is a
21462153
/// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is
21472154
/// known to never be a signaling NaN (it may still be a qNaN).
21482155
bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
21492156

2157+
/// \returns true if \p Op is known to never be a signaling NaN in \p
2158+
/// DemandedElts.
2159+
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts,
2160+
unsigned Depth = 0) const {
2161+
return isKnownNeverNaN(Op, DemandedElts, true, Depth);
2162+
}
2163+
21502164
/// \returns true if \p Op is known to never be a signaling NaN.
21512165
bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
21522166
return isKnownNeverNaN(Op, true, Depth);

‎llvm/include/llvm/CodeGen/TargetLowering.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4283,6 +4283,7 @@ class TargetLowering : public TargetLoweringBase {
42834283
/// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
42844284
/// NaN.
42854285
virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
4286+
const APInt &DemandedElts,
42864287
const SelectionDAG &DAG,
42874288
bool SNaN = false,
42884289
unsigned Depth = 0) const;

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5599,7 +5599,22 @@ bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
55995599
(Op.getOpcode() == ISD::ADD || isADDLike(Op));
56005600
}
56015601

5602-
bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5602+
bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
5603+
unsigned Depth) const {
5604+
EVT VT = Op.getValueType();
5605+
5606+
// Since the number of lanes in a scalable vector is unknown at compile time,
5607+
// we track one bit which is implicitly broadcast to all lanes. This means
5608+
// that all lanes in a scalable vector are considered demanded.
5609+
APInt DemandedElts = VT.isFixedLengthVector()
5610+
? APInt::getAllOnes(VT.getVectorNumElements())
5611+
: APInt(1, 1);
5612+
5613+
return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
5614+
}
5615+
5616+
bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
5617+
bool SNaN, unsigned Depth) const {
56035618
// If we're told that NaNs won't happen, assume they won't.
56045619
if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
56055620
return true;
@@ -5613,6 +5628,9 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
56135628
(SNaN && !C->getValueAPF().isSignaling());
56145629
}
56155630

5631+
if (!DemandedElts)
5632+
return false; // No demanded elts, better to assume we don't know anything.
5633+
56165634
unsigned Opcode = Op.getOpcode();
56175635
switch (Opcode) {
56185636
case ISD::FADD:
@@ -5655,21 +5673,21 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
56555673
case ISD::FLDEXP: {
56565674
if (SNaN)
56575675
return true;
5658-
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5676+
return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
56595677
}
56605678
case ISD::FABS:
56615679
case ISD::FNEG:
56625680
case ISD::FCOPYSIGN: {
5663-
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5681+
return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
56645682
}
56655683
case ISD::SELECT:
5666-
return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5667-
isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5684+
return isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1) &&
5685+
isKnownNeverNaN(Op.getOperand(2), DemandedElts, SNaN, Depth + 1);
56685686
case ISD::FP_EXTEND:
56695687
case ISD::FP_ROUND: {
56705688
if (SNaN)
56715689
return true;
5672-
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5690+
return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
56735691
}
56745692
case ISD::SINT_TO_FP:
56755693
case ISD::UINT_TO_FP:
@@ -5691,42 +5709,50 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
56915709
case ISD::FMAXIMUMNUM: {
56925710
// Only one needs to be known not-nan, since it will be returned if the
56935711
// other ends up being one.
5694-
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5695-
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5712+
return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||
5713+
isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
56965714
}
56975715
case ISD::FMINNUM_IEEE:
56985716
case ISD::FMAXNUM_IEEE: {
56995717
if (SNaN)
57005718
return true;
57015719
// This can return a NaN if either operand is an sNaN, or if both operands
57025720
// are NaN.
5703-
return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5704-
isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5705-
(isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5706-
isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5721+
return (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) &&
5722+
isKnownNeverSNaN(Op.getOperand(1), DemandedElts, Depth + 1)) ||
5723+
(isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1) &&
5724+
isKnownNeverSNaN(Op.getOperand(0), DemandedElts, Depth + 1));
57075725
}
57085726
case ISD::FMINIMUM:
57095727
case ISD::FMAXIMUM: {
57105728
// TODO: Does this quiet or return the origina NaN as-is?
5711-
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5712-
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5729+
return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) &&
5730+
isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
57135731
}
5714-
case ISD::EXTRACT_VECTOR_ELT:
5715-
case ISD::EXTRACT_SUBVECTOR: {
5732+
case ISD::EXTRACT_VECTOR_ELT: {
5733+
// TODO: Set DemandedSrcElts for constant index.
57165734
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
57175735
}
5736+
case ISD::EXTRACT_SUBVECTOR: {
5737+
SDValue Src = Op.getOperand(0);
5738+
unsigned Idx = Op.getConstantOperandVal(1);
5739+
unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5740+
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
5741+
return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
5742+
}
57185743
case ISD::BUILD_VECTOR: {
5719-
for (const SDValue &Opnd : Op->ops())
5720-
if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5744+
unsigned NumElts = Op.getNumOperands();
5745+
for (unsigned I = 0; I != NumElts; ++I)
5746+
if (DemandedElts[I] &&
5747+
!isKnownNeverNaN(Op.getOperand(I), SNaN, Depth + 1))
57215748
return false;
57225749
return true;
57235750
}
57245751
default:
5725-
if (Opcode >= ISD::BUILTIN_OP_END ||
5726-
Opcode == ISD::INTRINSIC_WO_CHAIN ||
5727-
Opcode == ISD::INTRINSIC_W_CHAIN ||
5728-
Opcode == ISD::INTRINSIC_VOID) {
5729-
return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5752+
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5753+
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
5754+
return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, *this, SNaN,
5755+
Depth);
57305756
}
57315757

57325758
return false;

‎llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3928,6 +3928,7 @@ bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
39283928
}
39293929

39303930
bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3931+
const APInt &DemandedElts,
39313932
const SelectionDAG &DAG,
39323933
bool SNaN,
39333934
unsigned Depth) const {

‎llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5971,10 +5971,9 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
59715971
}
59725972
}
59735973

5974-
bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
5975-
const SelectionDAG &DAG,
5976-
bool SNaN,
5977-
unsigned Depth) const {
5974+
bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
5975+
SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
5976+
unsigned Depth) const {
59785977
unsigned Opcode = Op.getOpcode();
59795978
switch (Opcode) {
59805979
case AMDGPUISD::FMIN_LEGACY:

‎llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,8 @@ class AMDGPUTargetLowering : public TargetLowering {
321321
const MachineRegisterInfo &MRI,
322322
unsigned Depth = 0) const override;
323323

324-
bool isKnownNeverNaNForTargetNode(SDValue Op,
325-
const SelectionDAG &DAG,
326-
bool SNaN = false,
324+
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
325+
const SelectionDAG &DAG, bool SNaN = false,
327326
unsigned Depth = 0) const override;
328327

329328
bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0,

‎llvm/lib/Target/AMDGPU/SIISelLowering.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16687,6 +16687,7 @@ bool SITargetLowering::denormalsEnabledForType(
1668716687
}
1668816688

1668916689
bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
16690+
const APInt &DemandedElts,
1669016691
const SelectionDAG &DAG,
1669116692
bool SNaN,
1669216693
unsigned Depth) const {
@@ -16699,8 +16700,8 @@ bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
1669916700
return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
1670016701
}
1670116702

16702-
return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG, SNaN,
16703-
Depth);
16703+
return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
16704+
DAG, SNaN, Depth);
1670416705
}
1670516706

1670616707
// On older subtargets, global FP atomic instructions have a hardcoded FP mode

‎llvm/lib/Target/AMDGPU/SIISelLowering.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -546,9 +546,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
546546

547547
bool isProfitableToHoist(Instruction *I) const override;
548548

549-
bool isKnownNeverNaNForTargetNode(SDValue Op,
550-
const SelectionDAG &DAG,
551-
bool SNaN = false,
549+
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
550+
const SelectionDAG &DAG, bool SNaN = false,
552551
unsigned Depth = 0) const override;
553552
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
554553
AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override;

0 commit comments

Comments
 (0)