Skip to content

[SPIR-V] Implement insertion of 'Group and Subgroup Instructions' using builtin functions #95176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 13, 2024
Merged

[SPIR-V] Implement insertion of 'Group and Subgroup Instructions' using builtin functions #95176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1de94d7
implement insertion of 'Group and Subgroup Instructions' using builti…
VyacheslavLevytskyy Jun 11, 2024
5fc6aa8
fix and add tests cases for instructions with a Group Operation
VyacheslavLevytskyy Jun 12, 2024
645c024
tweak error reporting
VyacheslavLevytskyy Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,35 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call,
const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
const SPIRV::GroupBuiltin *GroupBuiltin =
SPIRV::lookupGroupBuiltin(Builtin->Name);

MachineRegisterInfo *MRI = MIRBuilder.getMRI();
if (Call->isSpirvOp()) {
if (GroupBuiltin->NoGroupOperation)
return buildOpFromWrapper(MIRBuilder, GroupBuiltin->Opcode, Call,
GR->getSPIRVTypeID(Call->ReturnType));

// Group Operation is a literal
Register GroupOpReg = Call->Arguments[1];
const MachineInstr *MI = getDefInstrMaybeConstant(GroupOpReg, MRI);
if (!MI || MI->getOpcode() != TargetOpcode::G_CONSTANT)
report_fatal_error(
"Group Operation parameter must be an integer constant");
uint64_t GrpOp = MI->getOperand(1).getCImm()->getValue().getZExtValue();
Register ScopeReg = Call->Arguments[0];
if (!MRI->getRegClassOrNull(ScopeReg))
MRI->setRegClass(ScopeReg, &SPIRV::IDRegClass);
Register ValueReg = Call->Arguments[2];
if (!MRI->getRegClassOrNull(ValueReg))
MRI->setRegClass(ValueReg, &SPIRV::IDRegClass);
MIRBuilder.buildInstr(GroupBuiltin->Opcode)
.addDef(Call->ReturnRegister)
.addUse(GR->getSPIRVTypeID(Call->ReturnType))
.addUse(ScopeReg)
.addImm(GrpOp)
.addUse(ValueReg);
return true;
}

Register Arg0;
if (GroupBuiltin->HasBoolArg) {
Register ConstRegister = Call->Arguments[0];
Expand Down
16 changes: 16 additions & 0 deletions llvm/lib/Target/SPIRV/SPIRVBuiltins.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,9 +694,17 @@ multiclass DemangledGroupBuiltin<string name, int level /* OnlyWork/OnlySub/...
}
}

multiclass DemangledGroupBuiltinWrapper<string name, bits<8> minNumArgs, bits<8> maxNumArgs, Op operation> {
def : DemangledBuiltin<name, OpenCL_std, Group, minNumArgs, maxNumArgs>;
def : GroupBuiltin<name, operation>;
}

defm : DemangledGroupBuiltin<"group_all", WorkOrSub, OpGroupAll>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupAll", 2, 2, OpGroupAll>;
defm : DemangledGroupBuiltin<"group_any", WorkOrSub, OpGroupAny>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupAny", 2, 2, OpGroupAny>;
defm : DemangledGroupBuiltin<"group_broadcast", WorkOrSub, OpGroupBroadcast>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupBroadcast", 3, 3, OpGroupBroadcast>;
defm : DemangledGroupBuiltin<"group_non_uniform_broadcast", OnlySub, OpGroupNonUniformBroadcast>;
defm : DemangledGroupBuiltin<"group_broadcast_first", OnlySub, OpGroupNonUniformBroadcastFirst>;

Expand Down Expand Up @@ -731,41 +739,49 @@ defm : DemangledGroupBuiltin<"group_scan_inclusive_adds", WorkOrSub, OpGroupIAdd
defm : DemangledGroupBuiltin<"group_reduce_addu", WorkOrSub, OpGroupIAdd>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_addu", WorkOrSub, OpGroupIAdd>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_addu", WorkOrSub, OpGroupIAdd>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupIAdd", 3, 3, OpGroupIAdd>;

defm : DemangledGroupBuiltin<"group_fadd", WorkOrSub, OpGroupFAdd>;
defm : DemangledGroupBuiltin<"group_reduce_addf", WorkOrSub, OpGroupFAdd>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_addf", WorkOrSub, OpGroupFAdd>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_addf", WorkOrSub, OpGroupFAdd>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupFAdd", 3, 3, OpGroupFAdd>;

defm : DemangledGroupBuiltin<"group_fmin", WorkOrSub, OpGroupFMin>;
defm : DemangledGroupBuiltin<"group_reduce_minf", WorkOrSub, OpGroupFMin>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_minf", WorkOrSub, OpGroupFMin>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_minf", WorkOrSub, OpGroupFMin>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupFMin", 3, 3, OpGroupFMin>;

defm : DemangledGroupBuiltin<"group_umin", WorkOrSub, OpGroupUMin>;
defm : DemangledGroupBuiltin<"group_reduce_minu", WorkOrSub, OpGroupUMin>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_minu", WorkOrSub, OpGroupUMin>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_minu", WorkOrSub, OpGroupUMin>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupUMin", 3, 3, OpGroupUMin>;

defm : DemangledGroupBuiltin<"group_smin", WorkOrSub, OpGroupSMin>;
defm : DemangledGroupBuiltin<"group_reduce_mins", WorkOrSub, OpGroupSMin>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_mins", WorkOrSub, OpGroupSMin>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_mins", WorkOrSub, OpGroupSMin>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupSMin", 3, 3, OpGroupSMin>;

defm : DemangledGroupBuiltin<"group_fmax", WorkOrSub, OpGroupFMax>;
defm : DemangledGroupBuiltin<"group_reduce_maxf", WorkOrSub, OpGroupFMax>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_maxf", WorkOrSub, OpGroupFMax>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_maxf", WorkOrSub, OpGroupFMax>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupFMax", 3, 3, OpGroupFMax>;

defm : DemangledGroupBuiltin<"group_umax", WorkOrSub, OpGroupUMax>;
defm : DemangledGroupBuiltin<"group_reduce_maxu", WorkOrSub, OpGroupUMax>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_maxu", WorkOrSub, OpGroupUMax>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_maxu", WorkOrSub, OpGroupUMax>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupUMax", 3, 3, OpGroupUMax>;

defm : DemangledGroupBuiltin<"group_smax", WorkOrSub, OpGroupSMax>;
defm : DemangledGroupBuiltin<"group_reduce_maxs", WorkOrSub, OpGroupSMax>;
defm : DemangledGroupBuiltin<"group_scan_exclusive_maxs", WorkOrSub, OpGroupSMax>;
defm : DemangledGroupBuiltin<"group_scan_inclusive_maxs", WorkOrSub, OpGroupSMax>;
defm : DemangledGroupBuiltinWrapper<"__spirv_GroupSMax", 3, 3, OpGroupSMax>;

// cl_khr_subgroup_non_uniform_arithmetic
defm : DemangledGroupBuiltin<"group_non_uniform_iadd", WorkOrSub, OpGroupNonUniformIAdd>;
Expand Down
21 changes: 16 additions & 5 deletions llvm/test/CodeGen/SPIRV/transcoding/OpGroupAllAny.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,29 @@
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; CHECK-SPIRV: OpCapability Groups
; CHECK-SPIRV: %[[#BoolTypeID:]] = OpTypeBool
; CHECK-SPIRV: %[[#ConstID:]] = OpConstantTrue %[[#BoolTypeID]]
; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#ConstID]]
; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#ConstID]]
; CHECK-SPIRV-DAG: %[[#BoolTypeID:]] = OpTypeBool
; CHECK-SPIRV-DAG: %[[#True:]] = OpConstantTrue %[[#BoolTypeID]]
; CHECK-SPIRV-DAG: %[[#False:]] = OpConstantFalse %[[#BoolTypeID]]
; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#True]]
; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#True]]
; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#True]]
; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#False]]

define spir_kernel void @test(i32 addrspace(1)* nocapture readnone %i) {
entry:
%call = tail call spir_func i32 @_Z14work_group_alli(i32 5)
%call1 = tail call spir_func i32 @_Z14work_group_anyi(i32 5)
%call3 = tail call spir_func i32 @__spirv_GroupAll(i32 0, i1 1)
%call4 = tail call spir_func i32 @__spirv_GroupAny(i32 0, i1 0)
ret void
}

declare spir_func i32 @_Z14work_group_alli(i32)

declare spir_func i32 @_Z14work_group_anyi(i32)

declare spir_func i1 @__spirv_GroupAll(i32, i1)
declare spir_func i1 @__spirv_GroupAny(i32, i1)
42 changes: 41 additions & 1 deletion llvm/test/CodeGen/SPIRV/transcoding/group_ops.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}

; CHECK-SPIRV-DAG: %[[#int:]] = OpTypeInt 32 0
; CHECK-SPIRV-DAG: %[[#float:]] = OpTypeFloat 32
; CHECK-SPIRV-DAG: %[[#ScopeCrossWorkgroup:]] = OpConstant %[[#int]] 0
; CHECK-SPIRV-DAG: %[[#ScopeWorkgroup:]] = OpConstant %[[#int]] 2
; CHECK-SPIRV-DAG: %[[#ScopeSubgroup:]] = OpConstant %[[#int]] 3

Expand Down Expand Up @@ -247,7 +251,8 @@ entry:
declare spir_func i32 @_Z21work_group_reduce_minj(i32 noundef) local_unnamed_addr

; CHECK-SPIRV: OpFunction
; CHECK-SPIRV: %[[#]] = OpGroupBroadcast %[[#int]] %[[#ScopeWorkgroup]]
; CHECK-SPIRV: %[[#]] = OpGroupBroadcast %[[#int]] %[[#ScopeWorkgroup]] %[[#BroadcastValue:]] %[[#BroadcastLocalId:]]
; CHECK-SPIRV: %[[#]] = OpGroupBroadcast %[[#int]] %[[#ScopeCrossWorkgroup]] %[[#BroadcastValue]] %[[#BroadcastLocalId]]
; CHECK-SPIRV: OpFunctionEnd

;; kernel void testWorkGroupBroadcast(uint a, global size_t *id, global int *res) {
Expand All @@ -259,7 +264,42 @@ entry:
%0 = load i32, i32 addrspace(1)* %id, align 4
%call = call spir_func i32 @_Z20work_group_broadcastjj(i32 noundef %a, i32 noundef %0)
store i32 %call, i32 addrspace(1)* %res, align 4
%call1 = call spir_func i32 @__spirv_GroupBroadcast(i32 0, i32 noundef %a, i32 noundef %0)
ret void
}

declare spir_func i32 @_Z20work_group_broadcastjj(i32 noundef, i32 noundef) local_unnamed_addr
declare spir_func i32 @__spirv_GroupBroadcast(i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr

; CHECK-SPIRV: OpFunction
; CHECK-SPIRV: %[[#]] = OpGroupFAdd %[[#float]] %[[#ScopeCrossWorkgroup]] Reduce %[[#FValue:]]
; CHECK-SPIRV: %[[#]] = OpGroupFMin %[[#float]] %[[#ScopeWorkgroup]] InclusiveScan %[[#FValue]]
; CHECK-SPIRV: %[[#]] = OpGroupFMax %[[#float]] %[[#ScopeSubgroup]] ExclusiveScan %[[#FValue]]
; CHECK-SPIRV: %[[#]] = OpGroupIAdd %[[#int]] %[[#ScopeCrossWorkgroup]] Reduce %[[#IValue:]]
; CHECK-SPIRV: %[[#]] = OpGroupUMin %[[#int]] %[[#ScopeWorkgroup]] InclusiveScan %[[#IValue]]
; CHECK-SPIRV: %[[#]] = OpGroupSMin %[[#int]] %[[#ScopeSubgroup]] ExclusiveScan %[[#IValue]]
; CHECK-SPIRV: %[[#]] = OpGroupUMax %[[#int]] %[[#ScopeCrossWorkgroup]] Reduce %[[#IValue]]
; CHECK-SPIRV: %[[#]] = OpGroupSMax %[[#int]] %[[#ScopeWorkgroup]] InclusiveScan %[[#IValue]]
; CHECK-SPIRV: OpFunctionEnd

define spir_kernel void @foo(float %a, i32 %b) {
entry:
%f1 = call spir_func float @__spirv_GroupFAdd(i32 0, i32 0, float %a)
%f2 = call spir_func float @__spirv_GroupFMin(i32 2, i32 1, float %a)
%f3 = call spir_func float @__spirv_GroupFMax(i32 3, i32 2, float %a)
%i1 = call spir_func i32 @__spirv_GroupIAdd(i32 0, i32 0, i32 %b)
%i2 = call spir_func i32 @__spirv_GroupUMin(i32 2, i32 1, i32 %b)
%i3 = call spir_func i32 @__spirv_GroupSMin(i32 3, i32 2, i32 %b)
%i4 = call spir_func i32 @__spirv_GroupUMax(i32 0, i32 0, i32 %b)
%i5 = call spir_func i32 @__spirv_GroupSMax(i32 2, i32 1, i32 %b)
ret void
}

declare spir_func float @__spirv_GroupFAdd(i32, i32, float)
declare spir_func float @__spirv_GroupFMin(i32, i32, float)
declare spir_func float @__spirv_GroupFMax(i32, i32, float)
declare spir_func i32 @__spirv_GroupIAdd(i32, i32, i32)
declare spir_func i32 @__spirv_GroupUMin(i32, i32, i32)
declare spir_func i32 @__spirv_GroupSMin(i32, i32, i32)
declare spir_func i32 @__spirv_GroupUMax(i32, i32, i32)
declare spir_func i32 @__spirv_GroupSMax(i32, i32, i32)
Loading