Skip to content

Commit 072a9c0

Browse files
mskampmskamp
committed
[ValueTracking][X86] Compute KnownBits for phadd/phsub
Add KnownBits computations to ValueTracking and X86 DAG lowering. These instructions add/subtract adjacent vector elements in their operands. Example: phadd [X1, X2] [Y1, Y2] = [X1 + X2, Y1 + Y2]. This means that, in this example, we can compute the KnownBits of the operation by computing the KnownBits of [X1, X2] + [X1, X2] and [Y1, Y2] + [Y1, Y2] and intersecting the results. This approach also generalizes to all x86 vector types. There are also the operations phadd.sw and phsub.sw, which perform saturating addition/subtraction. Use sadd_sat and ssub_sat to compute the KnownBits of these operations. Also adjust the existing test case pr53247.ll because it can be transformed to a constant using the new KnownBits computation. Fixes #82516.
1 parent 3caccd8 commit 072a9c0

File tree

8 files changed

+282
-153
lines changed

8 files changed

+282
-153
lines changed

‎llvm/include/llvm/Analysis/VectorUtils.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,23 @@ void processShuffleMasks(
246246
function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction,
247247
function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction);
248248

249+
/// Compute the demanded elements mask of horizontal binary operations. A
250+
/// horizontal operation combines two adjacent elements in a vector operand.
251+
/// This function returns a mask for the elements that correspond to the first
252+
/// operand of this horizontal combination. For example, for two vectors
253+
/// [X1, X2, X3, X4] and [Y1, Y2, Y3, Y4], the resulting mask can include the
254+
/// elements X1, X3, Y1, and Y3. To get the other operands, simply shift the
255+
/// result of this function to the left by 1.
256+
///
257+
/// \param VectorBitWidth the total bit width of the vector
258+
/// \param DemandedElts the demanded elements mask for the operation
259+
/// \param DemandedLHS the demanded elements mask for the left operand
260+
/// \param DemandedRHS the demanded elements mask for the right operand
261+
void getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth,
262+
const APInt &DemandedElts,
263+
APInt &DemandedLHS,
264+
APInt &DemandedRHS);
265+
249266
/// Compute a map of integer instructions to their minimum legal type
250267
/// size.
251268
///

‎llvm/lib/Analysis/ValueTracking.cpp

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -950,6 +950,39 @@ getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
950950
return KnownOut;
951951
}
952952

953+
static KnownBits computeKnownBitsForHorizontalOperation(
954+
const Operator *I, const APInt &DemandedElts, unsigned Depth,
955+
const SimplifyQuery &Q,
956+
const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
957+
KnownBitsFunc) {
958+
APInt DemandedEltsLHS, DemandedEltsRHS;
959+
getHorizDemandedEltsForFirstOperand(Q.DL.getTypeSizeInBits(I->getType()),
960+
DemandedElts, DemandedEltsLHS,
961+
DemandedEltsRHS);
962+
963+
const auto ComputeForSingleOpFunc =
964+
[Depth, &Q](const Value *Op, APInt &DemandedEltsOp,
965+
std::array<KnownBits, 2> &Known) {
966+
for (unsigned Index = 0; Index < Known.size(); ++Index) {
967+
if (!DemandedEltsOp.isZero()) {
968+
Known[Index] = computeKnownBits(Op, DemandedEltsOp, Depth + 1, Q);
969+
} else {
970+
Known[Index] = KnownBits(getBitWidth(Op->getType(), Q.DL));
971+
Known[Index].setAllZero();
972+
}
973+
DemandedEltsOp <<= 1;
974+
}
975+
};
976+
977+
std::array<KnownBits, 2> KnownLHS;
978+
ComputeForSingleOpFunc(I->getOperand(0), DemandedEltsLHS, KnownLHS);
979+
std::array<KnownBits, 2> KnownRHS;
980+
ComputeForSingleOpFunc(I->getOperand(1), DemandedEltsRHS, KnownRHS);
981+
982+
return KnownBitsFunc(KnownLHS[0], KnownLHS[1])
983+
.intersectWith(KnownBitsFunc(KnownRHS[0], KnownRHS[1]));
984+
}
985+
953986
// Public so this can be used in `SimplifyDemandedUseBits`.
954987
KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
955988
const KnownBits &KnownLHS,
@@ -1725,6 +1758,56 @@ static void computeKnownBitsFromOperator(const Operator *I,
17251758
case Intrinsic::x86_sse42_crc32_64_64:
17261759
Known.Zero.setBitsFrom(32);
17271760
break;
1761+
case Intrinsic::x86_ssse3_phadd_d:
1762+
case Intrinsic::x86_ssse3_phadd_w:
1763+
case Intrinsic::x86_ssse3_phadd_d_128:
1764+
case Intrinsic::x86_ssse3_phadd_w_128:
1765+
case Intrinsic::x86_avx2_phadd_d:
1766+
case Intrinsic::x86_avx2_phadd_w: {
1767+
Known = computeKnownBitsForHorizontalOperation(
1768+
I, DemandedElts, Depth, Q,
1769+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1770+
return KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/false,
1771+
/*NUW=*/false, KnownLHS,
1772+
KnownRHS);
1773+
});
1774+
break;
1775+
}
1776+
case Intrinsic::x86_ssse3_phadd_sw:
1777+
case Intrinsic::x86_ssse3_phadd_sw_128:
1778+
case Intrinsic::x86_avx2_phadd_sw: {
1779+
Known = computeKnownBitsForHorizontalOperation(
1780+
I, DemandedElts, Depth, Q,
1781+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1782+
return KnownBits::sadd_sat(KnownLHS, KnownRHS);
1783+
});
1784+
break;
1785+
}
1786+
case Intrinsic::x86_ssse3_phsub_d:
1787+
case Intrinsic::x86_ssse3_phsub_w:
1788+
case Intrinsic::x86_ssse3_phsub_d_128:
1789+
case Intrinsic::x86_ssse3_phsub_w_128:
1790+
case Intrinsic::x86_avx2_phsub_d:
1791+
case Intrinsic::x86_avx2_phsub_w: {
1792+
Known = computeKnownBitsForHorizontalOperation(
1793+
I, DemandedElts, Depth, Q,
1794+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1795+
return KnownBits::computeForAddSub(/*Add=*/false, /*NSW=*/false,
1796+
/*NUW=*/false, KnownLHS,
1797+
KnownRHS);
1798+
});
1799+
break;
1800+
}
1801+
case Intrinsic::x86_ssse3_phsub_sw:
1802+
case Intrinsic::x86_ssse3_phsub_sw_128:
1803+
case Intrinsic::x86_avx2_phsub_sw: {
1804+
Known = computeKnownBitsForHorizontalOperation(
1805+
I, DemandedElts, Depth, Q,
1806+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1807+
return KnownBits::ssub_sat(KnownLHS, KnownRHS);
1808+
});
1809+
break;
1810+
}
17281811
case Intrinsic::riscv_vsetvli:
17291812
case Intrinsic::riscv_vsetvlimax: {
17301813
bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;

‎llvm/lib/Analysis/VectorUtils.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,33 @@ void llvm::processShuffleMasks(
541541
}
542542
}
543543

544+
void llvm::getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth,
545+
const APInt &DemandedElts,
546+
APInt &DemandedLHS,
547+
APInt &DemandedRHS) {
548+
int NumLanes = std::max<int>(1, VectorBitWidth / 128);
549+
int NumElts = DemandedElts.getBitWidth();
550+
int NumEltsPerLane = NumElts / NumLanes;
551+
int HalfEltsPerLane = NumEltsPerLane / 2;
552+
553+
DemandedLHS = APInt::getZero(NumElts);
554+
DemandedRHS = APInt::getZero(NumElts);
555+
556+
// Map DemandedElts to the horizontal operands.
557+
for (int Idx = 0; Idx != NumElts; ++Idx) {
558+
if (!DemandedElts[Idx])
559+
continue;
560+
int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
561+
int LocalIdx = Idx % NumEltsPerLane;
562+
if (LocalIdx < HalfEltsPerLane) {
563+
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx);
564+
} else {
565+
LocalIdx -= HalfEltsPerLane;
566+
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx);
567+
}
568+
}
569+
}
570+
544571
MapVector<Instruction *, uint64_t>
545572
llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
546573
const TargetTransformInfo *TTI) {

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 99 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5180,29 +5180,10 @@ static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
51805180
// Split the demanded elts of a HADD/HSUB node between its operands.
51815181
static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
51825182
APInt &DemandedLHS, APInt &DemandedRHS) {
5183-
int NumLanes = VT.getSizeInBits() / 128;
5184-
int NumElts = DemandedElts.getBitWidth();
5185-
int NumEltsPerLane = NumElts / NumLanes;
5186-
int HalfEltsPerLane = NumEltsPerLane / 2;
5187-
5188-
DemandedLHS = APInt::getZero(NumElts);
5189-
DemandedRHS = APInt::getZero(NumElts);
5190-
5191-
// Map DemandedElts to the horizontal operands.
5192-
for (int Idx = 0; Idx != NumElts; ++Idx) {
5193-
if (!DemandedElts[Idx])
5194-
continue;
5195-
int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
5196-
int LocalIdx = Idx % NumEltsPerLane;
5197-
if (LocalIdx < HalfEltsPerLane) {
5198-
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5199-
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5200-
} else {
5201-
LocalIdx -= HalfEltsPerLane;
5202-
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5203-
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5204-
}
5205-
}
5183+
getHorizDemandedEltsForFirstOperand(VT.getSizeInBits(), DemandedElts,
5184+
DemandedLHS, DemandedRHS);
5185+
DemandedLHS |= DemandedLHS << 1;
5186+
DemandedRHS |= DemandedRHS << 1;
52065187
}
52075188

52085189
/// Calculates the shuffle mask corresponding to the target-specific opcode.
@@ -36953,6 +36934,41 @@ static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS,
3695336934
Known = Known.zext(64);
3695436935
}
3695536936

36937+
static KnownBits computeKnownBitsForHorizontalOperation(
36938+
const SDValue Op, const APInt &DemandedElts, unsigned Depth,
36939+
unsigned OpIndexStart, const SelectionDAG &DAG,
36940+
const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
36941+
KnownBitsFunc) {
36942+
APInt DemandedEltsLHS, DemandedEltsRHS;
36943+
getHorizDemandedEltsForFirstOperand(Op.getValueType().getSizeInBits(),
36944+
DemandedElts, DemandedEltsLHS,
36945+
DemandedEltsRHS);
36946+
36947+
const auto ComputeForSingleOpFunc =
36948+
[&DAG, Depth](const SDValue &Op, APInt &DemandedEltsOp,
36949+
std::array<KnownBits, 2> &Known) {
36950+
for (unsigned Index = 0; Index < Known.size(); ++Index) {
36951+
if (!DemandedEltsOp.isZero()) {
36952+
Known[Index] = DAG.computeKnownBits(Op, DemandedEltsOp, Depth + 1);
36953+
} else {
36954+
Known[Index] = KnownBits(Op.getScalarValueSizeInBits());
36955+
Known[Index].setAllZero();
36956+
}
36957+
DemandedEltsOp <<= 1;
36958+
}
36959+
};
36960+
36961+
std::array<KnownBits, 2> KnownLHS;
36962+
ComputeForSingleOpFunc(Op.getOperand(OpIndexStart), DemandedEltsLHS,
36963+
KnownLHS);
36964+
std::array<KnownBits, 2> KnownRHS;
36965+
ComputeForSingleOpFunc(Op.getOperand(OpIndexStart + 1), DemandedEltsRHS,
36966+
KnownRHS);
36967+
36968+
return KnownBitsFunc(KnownLHS[0], KnownLHS[1])
36969+
.intersectWith(KnownBitsFunc(KnownRHS[0], KnownRHS[1]));
36970+
}
36971+
3695636972
void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3695736973
KnownBits &Known,
3695836974
const APInt &DemandedElts,
@@ -37262,6 +37278,17 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3726237278
}
3726337279
break;
3726437280
}
37281+
case X86ISD::HADD:
37282+
case X86ISD::HSUB: {
37283+
Known = computeKnownBitsForHorizontalOperation(
37284+
Op, DemandedElts, Depth, /*OpIndexStart=*/0, DAG,
37285+
[Opc](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37286+
return KnownBits::computeForAddSub(
37287+
/*Add=*/Opc == X86ISD::HADD, /*NSW=*/false, /*NUW=*/false,
37288+
KnownLHS, KnownRHS);
37289+
});
37290+
break;
37291+
}
3726537292
case ISD::INTRINSIC_WO_CHAIN: {
3726637293
switch (Op->getConstantOperandVal(0)) {
3726737294
case Intrinsic::x86_sse2_psad_bw:
@@ -37276,6 +37303,55 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3727637303
computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
3727737304
break;
3727837305
}
37306+
case Intrinsic::x86_ssse3_phadd_d:
37307+
case Intrinsic::x86_ssse3_phadd_w:
37308+
case Intrinsic::x86_ssse3_phadd_d_128:
37309+
case Intrinsic::x86_ssse3_phadd_w_128:
37310+
case Intrinsic::x86_avx2_phadd_d:
37311+
case Intrinsic::x86_avx2_phadd_w: {
37312+
Known = computeKnownBitsForHorizontalOperation(
37313+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37314+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37315+
return KnownBits::computeForAddSub(
37316+
/*Add=*/true, /*NSW=*/false, /*NUW=*/false, KnownLHS, KnownRHS);
37317+
});
37318+
break;
37319+
}
37320+
case Intrinsic::x86_ssse3_phadd_sw:
37321+
case Intrinsic::x86_ssse3_phadd_sw_128:
37322+
case Intrinsic::x86_avx2_phadd_sw: {
37323+
Known = computeKnownBitsForHorizontalOperation(
37324+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37325+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37326+
return KnownBits::sadd_sat(KnownLHS, KnownRHS);
37327+
});
37328+
break;
37329+
}
37330+
case Intrinsic::x86_ssse3_phsub_d:
37331+
case Intrinsic::x86_ssse3_phsub_w:
37332+
case Intrinsic::x86_ssse3_phsub_d_128:
37333+
case Intrinsic::x86_ssse3_phsub_w_128:
37334+
case Intrinsic::x86_avx2_phsub_d:
37335+
case Intrinsic::x86_avx2_phsub_w: {
37336+
Known = computeKnownBitsForHorizontalOperation(
37337+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37338+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37339+
return KnownBits::computeForAddSub(/*Add=*/false, /*NSW=*/false,
37340+
/*NUW=*/false, KnownLHS,
37341+
KnownRHS);
37342+
});
37343+
break;
37344+
}
37345+
case Intrinsic::x86_ssse3_phsub_sw:
37346+
case Intrinsic::x86_ssse3_phsub_sw_128:
37347+
case Intrinsic::x86_avx2_phsub_sw: {
37348+
Known = computeKnownBitsForHorizontalOperation(
37349+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37350+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37351+
return KnownBits::ssub_sat(KnownLHS, KnownRHS);
37352+
});
37353+
break;
37354+
}
3727937355
}
3728037356
break;
3728137357
}

0 commit comments

Comments
 (0)