Skip to content

[X86] combineX86ShufflesRecursively - replace Root node argument with opcode/valuetype/ismaskedshuffle data. NFC. #132437

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 25, 2025
Merged

[X86] combineX86ShufflesRecursively - replace Root node argument with opcode/valuetype/ismaskedshuffle data. NFC. #132437

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a8b4fa4
[X86] Remove Root node from combineX86ShuffleChain/combineX86ShuffleC…
RKSimon Mar 12, 2025
0735235
[X86] combineX86ShufflesRecursively - pull out repeated Root.getOpcod…
RKSimon Mar 12, 2025
33d4cb3
[X86] combineX86ShufflesRecursively - replace Root node argument with…
RKSimon Mar 12, 2025
15c7424
Update combineX86ShufflesRecursively argument comments to use /*...=*…
RKSimon Mar 25, 2025
682a88d
Merge branch 'upstream' into x86-shuffle-chain-root
RKSimon Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 49 additions & 49 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39642,9 +39642,10 @@ static bool matchBinaryPermuteShuffle(
}

static SDValue combineX86ShuffleChainWithExtract(
ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
ArrayRef<SDValue> Inputs, unsigned RootOpcode, MVT RootVT,
ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget);

/// Combine an arbitrary chain of shuffles into a single instruction if
Expand All @@ -39657,16 +39658,14 @@ static SDValue combineX86ShuffleChainWithExtract(
/// for this operation, or into a PSHUFB instruction which is a fully general
/// instruction but should only be used to replace chains over a certain depth.
static SDValue combineX86ShuffleChain(
ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
const SDLoc &DL, const X86Subtarget &Subtarget) {
ArrayRef<SDValue> Inputs, unsigned RootOpc, MVT RootVT,
ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget) {
assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
assert((Inputs.size() == 1 || Inputs.size() == 2) &&
"Unexpected number of shuffle inputs!");

unsigned RootOpc = Root.getOpcode();
MVT RootVT = Root.getSimpleValueType();
unsigned RootSizeInBits = RootVT.getSizeInBits();
unsigned NumRootElts = RootVT.getVectorNumElements();

Expand Down Expand Up @@ -40194,8 +40193,9 @@ static SDValue combineX86ShuffleChain(
// If that failed and either input is extracted then try to combine as a
// shuffle with the larger type.
if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
Inputs, Root, BaseMask, Depth, SrcNodes, AllowVariableCrossLaneMask,
AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget))
Inputs, RootOpc, RootVT, BaseMask, Depth, SrcNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
IsMaskedShuffle, DAG, DL, Subtarget))
return WideShuffle;

// If we have a dual input lane-crossing shuffle then lower to VPERMV3,
Expand Down Expand Up @@ -40366,8 +40366,9 @@ static SDValue combineX86ShuffleChain(
// If that failed and either input is extracted then try to combine as a
// shuffle with the larger type.
if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
Inputs, Root, BaseMask, Depth, SrcNodes, AllowVariableCrossLaneMask,
AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget))
Inputs, RootOpc, RootVT, BaseMask, Depth, SrcNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
DAG, DL, Subtarget))
return WideShuffle;

// If we have a dual input shuffle then lower to VPERMV3,
Expand Down Expand Up @@ -40404,16 +40405,16 @@ static SDValue combineX86ShuffleChain(
// -->
// extract_subvector(shuffle(x,y,m2),0)
static SDValue combineX86ShuffleChainWithExtract(
ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
ArrayRef<SDValue> Inputs, unsigned RootOpcode, MVT RootVT,
ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget) {
unsigned NumMaskElts = BaseMask.size();
unsigned NumInputs = Inputs.size();
if (NumInputs == 0)
return SDValue();

EVT RootVT = Root.getValueType();
unsigned RootSizeInBits = RootVT.getSizeInBits();
unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
Expand Down Expand Up @@ -40533,11 +40534,10 @@ static SDValue combineX86ShuffleChainWithExtract(
"WideRootSize mismatch");

if (SDValue WideShuffle = combineX86ShuffleChain(
WideInputs, WideRoot, WideMask, Depth, SrcNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
DAG, SDLoc(WideRoot), Subtarget)) {
WideShuffle =
extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
WideInputs, RootOpcode, WideRoot.getSimpleValueType(), WideMask,
Depth, SrcNodes, AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
IsMaskedShuffle, DAG, SDLoc(WideRoot), Subtarget)) {
WideShuffle = extractSubVector(WideShuffle, 0, DAG, DL, RootSizeInBits);
return DAG.getBitcast(RootVT, WideShuffle);
}

Expand Down Expand Up @@ -40881,15 +40881,14 @@ namespace llvm {
/// combine-ordering. To fix this, we should do the redundant instruction
/// combining in this recursive walk.
static SDValue combineX86ShufflesRecursively(
ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
ArrayRef<SDValue> SrcOps, int SrcOpIndex, unsigned RootOpc, MVT RootVT,
ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
unsigned MaxDepth, bool AllowVariableCrossLaneMask,
bool AllowVariablePerLaneMask, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget) {
bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
const SDLoc &DL, const X86Subtarget &Subtarget) {
assert(!RootMask.empty() &&
(RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
"Illegal shuffle root mask");
MVT RootVT = Root.getSimpleValueType();
assert(RootVT.isVector() && "Shuffles operate on vector types!");
unsigned RootSizeInBits = RootVT.getSizeInBits();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Expand Down Expand Up @@ -41185,8 +41184,9 @@ static SDValue combineX86ShufflesRecursively(
AllowPerLaneVar = AllowVariablePerLaneMask;
}
if (SDValue Res = combineX86ShufflesRecursively(
Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
AllowCrossLaneVar, AllowPerLaneVar, DAG, DL, Subtarget))
Ops, i, RootOpc, RootVT, ResolvedMask, CombinedNodes, Depth + 1,
MaxDepth, AllowCrossLaneVar, AllowPerLaneVar, IsMaskedShuffle,
DAG, DL, Subtarget))
return Res;
}
}
Expand Down Expand Up @@ -41271,10 +41271,6 @@ static SDValue combineX86ShufflesRecursively(
resolveTargetShuffleInputsAndMask(Ops, Mask);
}

// If we are a AVX512/EVEX target the mask element size should match the root
// element size to allow writemasks to be reused.
bool IsMaskedShuffle = isMaskableNode(Root, Subtarget);

// We can only combine unary and binary shuffle mask cases.
if (Ops.size() <= 2) {
// Minor canonicalization of the accumulated shuffle mask to make it easier
Expand All @@ -41298,8 +41294,9 @@ static SDValue combineX86ShufflesRecursively(

// Try to combine into a single shuffle instruction.
if (SDValue Shuffle = combineX86ShuffleChain(
Ops, Root, Mask, Depth, CombinedNodes, AllowVariableCrossLaneMask,
AllowVariablePerLaneMask, IsMaskedShuffle, DAG, DL, Subtarget))
Ops, RootOpc, RootVT, Mask, Depth, CombinedNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
IsMaskedShuffle, DAG, DL, Subtarget))
return Shuffle;

// If all the operands come from the same larger vector, fallthrough and try
Expand All @@ -41317,16 +41314,18 @@ static SDValue combineX86ShufflesRecursively(
// If that failed and any input is extracted then try to combine as a
// shuffle with the larger type.
return combineX86ShuffleChainWithExtract(
Ops, Root, Mask, Depth, CombinedNodes, AllowVariableCrossLaneMask,
AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget);
Ops, RootOpc, RootVT, Mask, Depth, CombinedNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
DAG, DL, Subtarget);
}

/// Helper entry wrapper to combineX86ShufflesRecursively.
static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
return combineX86ShufflesRecursively(
{Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
/*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
{Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), {0}, {}, /*Depth=*/0,
X86::MaxShuffleCombineDepth, /*AllowCrossLaneVarMask=*/true,
/*AllowPerLaneVarMask=*/true, isMaskableNode(Op, Subtarget), DAG,
SDLoc(Op), Subtarget);
}

Expand Down Expand Up @@ -41977,10 +41976,10 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
for (unsigned i = 0; i != Scale; ++i)
DemandedMask[i] = i;
if (SDValue Res = combineX86ShufflesRecursively(
{BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
X86::MaxShuffleCombineDepth,
/*AllowCrossLaneVarMask*/ true,
/*AllowPerLaneVarMask*/ true, DAG, DL, Subtarget))
{BC}, 0, BC.getOpcode(), BC.getSimpleValueType(), DemandedMask,
{}, /*Depth=*/0, X86::MaxShuffleCombineDepth,
/*AllowCrossLaneVarMask=*/true, /*AllowPerLaneVarMask=*/true,
/*IsMaskedShuffle=*/false, DAG, DL, Subtarget))
return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
DAG.getBitcast(SrcVT, Res));
}
Expand Down Expand Up @@ -43981,8 +43980,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
DemandedMask[i] = i;

SDValue NewShuffle = combineX86ShufflesRecursively(
{Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
/*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
{Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), DemandedMask, {}, 0,
X86::MaxShuffleCombineDepth - Depth, /*AllowCrossLaneVarMask=*/true,
/*AllowPerLaneVarMask=*/true, isMaskableNode(Op, Subtarget), TLO.DAG,
SDLoc(Op), Subtarget);
if (NewShuffle)
return TLO.CombineTo(Op, NewShuffle);
Expand Down Expand Up @@ -51617,10 +51617,10 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
}

if (SDValue Shuffle = combineX86ShufflesRecursively(
{SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
X86::MaxShuffleCombineDepth,
/*AllowVarCrossLaneMask*/ true,
/*AllowVarPerLaneMask*/ true, DAG, SDLoc(SrcVec), Subtarget))
{SrcVec}, 0, SrcVec.getOpcode(), SrcVec.getSimpleValueType(),
ShuffleMask, {}, /*Depth=*/1, X86::MaxShuffleCombineDepth,
/*AllowVarCrossLaneMask=*/true, /*AllowVarPerLaneMask=*/true,
/*IsMaskedShuffle=*/false, DAG, SDLoc(SrcVec), Subtarget))
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
N0.getOperand(1));
}
Expand Down
Loading