Skip to content

Commit d680eab

Browse files
mskampmskamp
committed
[ValueTracking][X86] Compute KnownBits for phadd/phsub
Add KnownBits computations to ValueTracking and X86 DAG lowering. These instructions add/subtract adjacent vector elements in their operands. Example: phadd [X1, X2] [Y1, Y2] = [X1 + X2, Y1 + Y2]. This means that, in this example, we can compute the KnownBits of the operation by computing the KnownBits of [X1, X2] + [X1, X2] and [Y1, Y2] + [Y1, Y2] and intersecting the results. This approach also generalizes to all x86 vector types. There are also the operations phadd.sw and phsub.sw, which perform saturating addition/subtraction. Use sadd_sat and ssub_sat to compute the KnownBits of these operations. Also adjust the existing test case pr53247.ll because it can be transformed to a constant using the new KnownBits computation. Fixes #82516.
1 parent 3caccd8 commit d680eab

File tree

8 files changed

+275
-153
lines changed

8 files changed

+275
-153
lines changed

‎llvm/include/llvm/Analysis/VectorUtils.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,23 @@ void processShuffleMasks(
246246
function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction,
247247
function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction);
248248

249+
/// Compute the demanded elements mask of horizontal binary operations. A
250+
/// horizontal operation combines two adjacent elements in a vector operand.
251+
/// This function returns a mask for the elements that correspond to the first
252+
/// operand of this horizontal combination. For example, for two vectors
253+
/// [X1, X2, X3, X4] and [Y1, Y2, Y3, Y4], the resulting mask can include the
254+
/// elements X1, X3, Y1, and Y3. To get the other operands, simply shift the
255+
/// result of this function to the left by 1.
256+
///
257+
/// \param VectorBitWidth the total bit width of the vector
258+
/// \param DemandedElts the demanded elements mask for the operation
259+
/// \param DemandedLHS the demanded elements mask for the left operand
260+
/// \param DemandedRHS the demanded elements mask for the right operand
261+
void getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth,
262+
const APInt &DemandedElts,
263+
APInt &DemandedLHS,
264+
APInt &DemandedRHS);
265+
249266
/// Compute a map of integer instructions to their minimum legal type
250267
/// size.
251268
///

‎llvm/lib/Analysis/ValueTracking.cpp

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -950,6 +950,36 @@ getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
950950
return KnownOut;
951951
}
952952

953+
static KnownBits computeKnownBitsForHorizontalOperation(
954+
const Operator *I, const APInt &DemandedElts, unsigned Depth,
955+
const SimplifyQuery &Q,
956+
const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
957+
KnownBitsFunc) {
958+
APInt DemandedEltsLHS, DemandedEltsRHS;
959+
getHorizDemandedEltsForFirstOperand(Q.DL.getTypeSizeInBits(I->getType()),
960+
DemandedElts, DemandedEltsLHS,
961+
DemandedEltsRHS);
962+
963+
const auto ComputeForSingleOpFunc =
964+
[Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
965+
std::array<KnownBits, 2> Known;
966+
for (unsigned Index = 0; Index < Known.size(); ++Index) {
967+
Known[Index] = computeKnownBits(Op, DemandedEltsOp, Depth + 1, Q);
968+
DemandedEltsOp <<= 1;
969+
}
970+
return KnownBitsFunc(Known[0], Known[1]);
971+
};
972+
973+
if (!DemandedEltsLHS.isZero() && !DemandedEltsRHS.isZero()) {
974+
return ComputeForSingleOpFunc(I->getOperand(0), DemandedEltsLHS)
975+
.intersectWith(ComputeForSingleOpFunc(I->getOperand(1), DemandedEltsRHS));
976+
}
977+
if (!DemandedEltsLHS.isZero()) {
978+
return ComputeForSingleOpFunc(I->getOperand(0), DemandedEltsLHS);
979+
}
980+
return ComputeForSingleOpFunc(I->getOperand(1), DemandedEltsRHS);
981+
}
982+
953983
// Public so this can be used in `SimplifyDemandedUseBits`.
954984
KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
955985
const KnownBits &KnownLHS,
@@ -1725,6 +1755,56 @@ static void computeKnownBitsFromOperator(const Operator *I,
17251755
case Intrinsic::x86_sse42_crc32_64_64:
17261756
Known.Zero.setBitsFrom(32);
17271757
break;
1758+
case Intrinsic::x86_ssse3_phadd_d:
1759+
case Intrinsic::x86_ssse3_phadd_w:
1760+
case Intrinsic::x86_ssse3_phadd_d_128:
1761+
case Intrinsic::x86_ssse3_phadd_w_128:
1762+
case Intrinsic::x86_avx2_phadd_d:
1763+
case Intrinsic::x86_avx2_phadd_w: {
1764+
Known = computeKnownBitsForHorizontalOperation(
1765+
I, DemandedElts, Depth, Q,
1766+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1767+
return KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/false,
1768+
/*NUW=*/false, KnownLHS,
1769+
KnownRHS);
1770+
});
1771+
break;
1772+
}
1773+
case Intrinsic::x86_ssse3_phadd_sw:
1774+
case Intrinsic::x86_ssse3_phadd_sw_128:
1775+
case Intrinsic::x86_avx2_phadd_sw: {
1776+
Known = computeKnownBitsForHorizontalOperation(
1777+
I, DemandedElts, Depth, Q,
1778+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1779+
return KnownBits::sadd_sat(KnownLHS, KnownRHS);
1780+
});
1781+
break;
1782+
}
1783+
case Intrinsic::x86_ssse3_phsub_d:
1784+
case Intrinsic::x86_ssse3_phsub_w:
1785+
case Intrinsic::x86_ssse3_phsub_d_128:
1786+
case Intrinsic::x86_ssse3_phsub_w_128:
1787+
case Intrinsic::x86_avx2_phsub_d:
1788+
case Intrinsic::x86_avx2_phsub_w: {
1789+
Known = computeKnownBitsForHorizontalOperation(
1790+
I, DemandedElts, Depth, Q,
1791+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1792+
return KnownBits::computeForAddSub(/*Add=*/false, /*NSW=*/false,
1793+
/*NUW=*/false, KnownLHS,
1794+
KnownRHS);
1795+
});
1796+
break;
1797+
}
1798+
case Intrinsic::x86_ssse3_phsub_sw:
1799+
case Intrinsic::x86_ssse3_phsub_sw_128:
1800+
case Intrinsic::x86_avx2_phsub_sw: {
1801+
Known = computeKnownBitsForHorizontalOperation(
1802+
I, DemandedElts, Depth, Q,
1803+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
1804+
return KnownBits::ssub_sat(KnownLHS, KnownRHS);
1805+
});
1806+
break;
1807+
}
17281808
case Intrinsic::riscv_vsetvli:
17291809
case Intrinsic::riscv_vsetvlimax: {
17301810
bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;

‎llvm/lib/Analysis/VectorUtils.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,33 @@ void llvm::processShuffleMasks(
541541
}
542542
}
543543

544+
void llvm::getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth,
545+
const APInt &DemandedElts,
546+
APInt &DemandedLHS,
547+
APInt &DemandedRHS) {
548+
int NumLanes = std::max<int>(1, VectorBitWidth / 128);
549+
int NumElts = DemandedElts.getBitWidth();
550+
int NumEltsPerLane = NumElts / NumLanes;
551+
int HalfEltsPerLane = NumEltsPerLane / 2;
552+
553+
DemandedLHS = APInt::getZero(NumElts);
554+
DemandedRHS = APInt::getZero(NumElts);
555+
556+
// Map DemandedElts to the horizontal operands.
557+
for (int Idx = 0; Idx != NumElts; ++Idx) {
558+
if (!DemandedElts[Idx])
559+
continue;
560+
int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
561+
int LocalIdx = Idx % NumEltsPerLane;
562+
if (LocalIdx < HalfEltsPerLane) {
563+
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx);
564+
} else {
565+
LocalIdx -= HalfEltsPerLane;
566+
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx);
567+
}
568+
}
569+
}
570+
544571
MapVector<Instruction *, uint64_t>
545572
llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
546573
const TargetTransformInfo *TTI) {

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5180,29 +5180,10 @@ static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
51805180
// Split the demanded elts of a HADD/HSUB node between its operands.
51815181
static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
51825182
APInt &DemandedLHS, APInt &DemandedRHS) {
5183-
int NumLanes = VT.getSizeInBits() / 128;
5184-
int NumElts = DemandedElts.getBitWidth();
5185-
int NumEltsPerLane = NumElts / NumLanes;
5186-
int HalfEltsPerLane = NumEltsPerLane / 2;
5187-
5188-
DemandedLHS = APInt::getZero(NumElts);
5189-
DemandedRHS = APInt::getZero(NumElts);
5190-
5191-
// Map DemandedElts to the horizontal operands.
5192-
for (int Idx = 0; Idx != NumElts; ++Idx) {
5193-
if (!DemandedElts[Idx])
5194-
continue;
5195-
int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
5196-
int LocalIdx = Idx % NumEltsPerLane;
5197-
if (LocalIdx < HalfEltsPerLane) {
5198-
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5199-
DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5200-
} else {
5201-
LocalIdx -= HalfEltsPerLane;
5202-
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5203-
DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5204-
}
5205-
}
5183+
getHorizDemandedEltsForFirstOperand(VT.getSizeInBits(), DemandedElts,
5184+
DemandedLHS, DemandedRHS);
5185+
DemandedLHS |= DemandedLHS << 1;
5186+
DemandedRHS |= DemandedRHS << 1;
52065187
}
52075188

52085189
/// Calculates the shuffle mask corresponding to the target-specific opcode.
@@ -36953,6 +36934,37 @@ static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS,
3695336934
Known = Known.zext(64);
3695436935
}
3695536936

36937+
static KnownBits computeKnownBitsForHorizontalOperation(
36938+
const SDValue Op, const APInt &DemandedElts, unsigned Depth,
36939+
unsigned OpIndexStart, const SelectionDAG &DAG,
36940+
const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
36941+
KnownBitsFunc) {
36942+
APInt DemandedEltsLHS, DemandedEltsRHS;
36943+
getHorizDemandedEltsForFirstOperand(Op.getValueType().getSizeInBits(),
36944+
DemandedElts, DemandedEltsLHS,
36945+
DemandedEltsRHS);
36946+
36947+
const auto ComputeForSingleOpFunc =
36948+
[&DAG, Depth, KnownBitsFunc](const SDValue &Op, APInt &DemandedEltsOp) {
36949+
std::array<KnownBits, 2> Known;
36950+
for (unsigned Index = 0; Index < Known.size(); ++Index) {
36951+
Known[Index] = DAG.computeKnownBits(Op, DemandedEltsOp, Depth + 1);
36952+
DemandedEltsOp <<= 1;
36953+
}
36954+
return KnownBitsFunc(Known[0], Known[1]);
36955+
};
36956+
36957+
if (!DemandedEltsLHS.isZero() && !DemandedEltsRHS.isZero()) {
36958+
return ComputeForSingleOpFunc(Op.getOperand(OpIndexStart), DemandedEltsLHS)
36959+
.intersectWith(ComputeForSingleOpFunc(Op.getOperand(OpIndexStart + 1),
36960+
DemandedEltsRHS));
36961+
}
36962+
if (!DemandedEltsLHS.isZero()) {
36963+
return ComputeForSingleOpFunc(Op.getOperand(OpIndexStart), DemandedEltsLHS);
36964+
}
36965+
return ComputeForSingleOpFunc(Op.getOperand(OpIndexStart + 1), DemandedEltsRHS);
36966+
}
36967+
3695636968
void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3695736969
KnownBits &Known,
3695836970
const APInt &DemandedElts,
@@ -37262,6 +37274,17 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3726237274
}
3726337275
break;
3726437276
}
37277+
case X86ISD::HADD:
37278+
case X86ISD::HSUB: {
37279+
Known = computeKnownBitsForHorizontalOperation(
37280+
Op, DemandedElts, Depth, /*OpIndexStart=*/0, DAG,
37281+
[Opc](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37282+
return KnownBits::computeForAddSub(
37283+
/*Add=*/Opc == X86ISD::HADD, /*NSW=*/false, /*NUW=*/false,
37284+
KnownLHS, KnownRHS);
37285+
});
37286+
break;
37287+
}
3726537288
case ISD::INTRINSIC_WO_CHAIN: {
3726637289
switch (Op->getConstantOperandVal(0)) {
3726737290
case Intrinsic::x86_sse2_psad_bw:
@@ -37276,6 +37299,55 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3727637299
computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
3727737300
break;
3727837301
}
37302+
case Intrinsic::x86_ssse3_phadd_d:
37303+
case Intrinsic::x86_ssse3_phadd_w:
37304+
case Intrinsic::x86_ssse3_phadd_d_128:
37305+
case Intrinsic::x86_ssse3_phadd_w_128:
37306+
case Intrinsic::x86_avx2_phadd_d:
37307+
case Intrinsic::x86_avx2_phadd_w: {
37308+
Known = computeKnownBitsForHorizontalOperation(
37309+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37310+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37311+
return KnownBits::computeForAddSub(
37312+
/*Add=*/true, /*NSW=*/false, /*NUW=*/false, KnownLHS, KnownRHS);
37313+
});
37314+
break;
37315+
}
37316+
case Intrinsic::x86_ssse3_phadd_sw:
37317+
case Intrinsic::x86_ssse3_phadd_sw_128:
37318+
case Intrinsic::x86_avx2_phadd_sw: {
37319+
Known = computeKnownBitsForHorizontalOperation(
37320+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37321+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37322+
return KnownBits::sadd_sat(KnownLHS, KnownRHS);
37323+
});
37324+
break;
37325+
}
37326+
case Intrinsic::x86_ssse3_phsub_d:
37327+
case Intrinsic::x86_ssse3_phsub_w:
37328+
case Intrinsic::x86_ssse3_phsub_d_128:
37329+
case Intrinsic::x86_ssse3_phsub_w_128:
37330+
case Intrinsic::x86_avx2_phsub_d:
37331+
case Intrinsic::x86_avx2_phsub_w: {
37332+
Known = computeKnownBitsForHorizontalOperation(
37333+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37334+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37335+
return KnownBits::computeForAddSub(/*Add=*/false, /*NSW=*/false,
37336+
/*NUW=*/false, KnownLHS,
37337+
KnownRHS);
37338+
});
37339+
break;
37340+
}
37341+
case Intrinsic::x86_ssse3_phsub_sw:
37342+
case Intrinsic::x86_ssse3_phsub_sw_128:
37343+
case Intrinsic::x86_avx2_phsub_sw: {
37344+
Known = computeKnownBitsForHorizontalOperation(
37345+
Op, DemandedElts, Depth, /*OpIndexStart=*/1, DAG,
37346+
[](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37347+
return KnownBits::ssub_sat(KnownLHS, KnownRHS);
37348+
});
37349+
break;
37350+
}
3727937351
}
3728037352
break;
3728137353
}

0 commit comments

Comments
 (0)