Skip to content

[AMDGPU][GFX12] Default component broadcast store #76212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 12, 2024
Merged

[AMDGPU][GFX12] Default component broadcast store #76212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
06117c6
[AMDGPU][GFX12] Default component broadcast store
May 10, 2023
04f0ba6
Add features for default components {zero,broadcast}
mariusz-sikora-at-amd Jan 5, 2024
beb9756
Merge main into gfx12-default-component-broadcast-store
mariusz-sikora-at-amd Jan 5, 2024
c2c8c3d
Merge remote-tracking branch main into masikora/gfx12-default-compone…
mariusz-sikora-at-amd Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions llvm/lib/Target/AMDGPU/AMDGPU.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,18 @@ def FeatureFlatAtomicFaddF32Inst
"Has flat_atomic_add_f32 instruction"
>;

def FeatureDefaultComponentZero : SubtargetFeature<"default-component-zero",
"HasDefaultComponentZero",
"true",
"BUFFER/IMAGE store instructions set unspecified components to zero"
>;

def FeatureDefaultComponentBroadcast : SubtargetFeature<"default-component-broadcast",
"HasDefaultComponentBroadcast",
"true",
"BUFFER/IMAGE store instructions set unspecified components to x component"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably should add gfx numbers to the description

>;

def FeatureSupportsSRAMECC : SubtargetFeature<"sramecc-support",
"SupportsSRAMECC",
"true",
Expand Down Expand Up @@ -1003,7 +1015,7 @@ def FeatureSouthernIslands : GCNSubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
FeatureWavefrontSize64, FeatureSMemTimeInst, FeatureMadMacF32Insts,
FeatureDsSrc2Insts, FeatureLDSBankCount32, FeatureMovrel,
FeatureTrigReducedRange, FeatureExtendedImageInsts, FeatureImageInsts,
FeatureGDS, FeatureGWS
FeatureGDS, FeatureGWS, FeatureDefaultComponentZero
]
>;

Expand All @@ -1014,7 +1026,7 @@ def FeatureSeaIslands : GCNSubtargetFeatureGeneration<"SEA_ISLANDS",
FeatureCIInsts, FeatureMovrel, FeatureTrigReducedRange,
FeatureGFX7GFX8GFX9Insts, FeatureSMemTimeInst, FeatureMadMacF32Insts,
FeatureDsSrc2Insts, FeatureExtendedImageInsts, FeatureUnalignedBufferAccess,
FeatureImageInsts, FeatureGDS, FeatureGWS
FeatureImageInsts, FeatureGDS, FeatureGWS, FeatureDefaultComponentZero
]
>;

Expand All @@ -1029,7 +1041,8 @@ def FeatureVolcanicIslands : GCNSubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
FeatureIntClamp, FeatureTrigReducedRange, FeatureGFX8Insts,
FeatureGFX7GFX8GFX9Insts, FeatureSMemTimeInst, FeatureMadMacF32Insts,
FeatureDsSrc2Insts, FeatureExtendedImageInsts, FeatureFastDenormalF32,
FeatureUnalignedBufferAccess, FeatureImageInsts, FeatureGDS, FeatureGWS
FeatureUnalignedBufferAccess, FeatureImageInsts, FeatureGDS, FeatureGWS,
FeatureDefaultComponentZero
]
>;

Expand All @@ -1047,7 +1060,7 @@ def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
FeatureScalarFlatScratchInsts, FeatureScalarAtomics, FeatureR128A16,
FeatureA16, FeatureSMemTimeInst, FeatureFastDenormalF32, FeatureSupportsXNACK,
FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess,
FeatureNegativeScratchOffsetBug, FeatureGWS
FeatureNegativeScratchOffsetBug, FeatureGWS, FeatureDefaultComponentZero
]
>;

Expand All @@ -1067,7 +1080,7 @@ def FeatureGFX10 : GCNSubtargetFeatureGeneration<"GFX10",
FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
FeatureA16, FeatureSMemTimeInst, FeatureFastDenormalF32, FeatureG16,
FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, FeatureImageInsts,
FeatureGDS, FeatureGWS
FeatureGDS, FeatureGWS, FeatureDefaultComponentZero
]
>;

Expand All @@ -1087,7 +1100,7 @@ def FeatureGFX11 : GCNSubtargetFeatureGeneration<"GFX11",
FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
FeatureA16, FeatureFastDenormalF32, FeatureG16,
FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, FeatureGDS,
FeatureGWS
FeatureGWS, FeatureDefaultComponentZero
]
>;

Expand All @@ -1107,7 +1120,7 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
FeatureNoDataDepHazard, FeaturePkFmacF16Inst,
FeatureA16, FeatureFastDenormalF32, FeatureG16,
FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess,
FeatureTrue16BitInsts
FeatureTrue16BitInsts, FeatureDefaultComponentBroadcast
]
>;

Expand Down Expand Up @@ -2013,6 +2026,13 @@ def HasFlatAtomicFaddF32Inst
: Predicate<"Subtarget->hasFlatAtomicFaddF32Inst()">,
AssemblerPredicate<(all_of FeatureFlatAtomicFaddF32Inst)>;

def HasDefaultComponentZero
: Predicate<"Subtarget->hasDefaultComponentZero()">,
AssemblerPredicate<(all_of FeatureDefaultComponentZero)>;
def HasDefaultComponentBroadcast
: Predicate<"Subtarget->hasDefaultComponentBroadcast()">,
AssemblerPredicate<(all_of FeatureDefaultComponentBroadcast)>;

def HasDsSrc2Insts : Predicate<"!Subtarget->hasDsSrc2Insts()">,
AssemblerPredicate<(all_of FeatureDsSrc2Insts)>;

Expand Down
38 changes: 36 additions & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,35 @@ static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV,
return DemandedElts;
}

// Trim elements of the end of the vector \p V, if they are
// equal to the first element of the vector.
static APInt defaultComponentBroadcast(Value *V) {
auto *VTy = cast<FixedVectorType>(V->getType());
unsigned VWidth = VTy->getNumElements();
APInt DemandedElts = APInt::getAllOnes(VWidth);
Value *FirstComponent = findScalarElement(V, 0);

SmallVector<int> ShuffleMask;
if (auto *SVI = dyn_cast<ShuffleVectorInst>(V))
SVI->getShuffleMask(ShuffleMask);

for (int I = VWidth - 1; I > 0; --I) {
if (ShuffleMask.empty()) {
auto *Elt = findScalarElement(V, I);
if (!Elt || (Elt != FirstComponent && !isa<UndefValue>(Elt)))
break;
} else {
// Detect identical elements in the shufflevector result, even though
// findScalarElement cannot tell us what that element is.
if (ShuffleMask[I] != ShuffleMask[0] && ShuffleMask[I] != PoisonMaskElem)
break;
}
DemandedElts.clearBit(I);
}

return DemandedElts;
}

static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
IntrinsicInst &II,
APInt DemandedElts,
Expand Down Expand Up @@ -1140,8 +1169,13 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
if (!isa<FixedVectorType>(II.getArgOperand(0)->getType()))
break;

APInt DemandedElts =
trimTrailingZerosInVector(IC, II.getArgOperand(0), &II);
APInt DemandedElts;
if (ST->hasDefaultComponentBroadcast())
DemandedElts = defaultComponentBroadcast(II.getArgOperand(0));
else if (ST->hasDefaultComponentZero())
DemandedElts = trimTrailingZerosInVector(IC, II.getArgOperand(0), &II);
else
break;

int DMaskIdx = getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID()) ? 1 : -1;
if (simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, DMaskIdx,
Expand Down
8 changes: 8 additions & 0 deletions llvm/lib/Target/AMDGPU/GCNSubtarget.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool HasAtomicCSubNoRtnInsts = false;
bool HasAtomicGlobalPkAddBF16Inst = false;
bool HasFlatAtomicFaddF32Inst = false;
bool HasDefaultComponentZero = false;
bool HasDefaultComponentBroadcast = false;
bool SupportsSRAMECC = false;

// This should not be used directly. 'TargetID' tracks the dynamic settings
Expand Down Expand Up @@ -802,6 +804,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,

bool hasFlatAtomicFaddF32Inst() const { return HasFlatAtomicFaddF32Inst; }

bool hasDefaultComponentZero() const { return HasDefaultComponentZero; }

bool hasDefaultComponentBroadcast() const {
return HasDefaultComponentBroadcast;
}

bool hasNoSdstCMPX() const {
return HasNoSdstCMPX;
}
Expand Down
Loading