Skip to content

Commit 34c8b73

Browse files
bwlodarczsys-ce-bb
bwlodarcz
authored and
sys-ce-bb
committed
Add IAddCarry and ISubBorrow builtins (#2392)
The fix adds support for IR builtin calls __spirv_IAddCarry and __spirv_ISubBorrow. It's also first part of fix which removes noncompliance of uadd/sub_with_overflow intrinsics. SPIRVUtil changes are needed to support situations where builtin don't have corresponding store instruction. Original commit: KhronosGroup/SPIRV-LLVM-Translator@63fd35158554695
1 parent b795aba commit 34c8b73

File tree

7 files changed

+365
-31
lines changed

7 files changed

+365
-31
lines changed

‎llvm-spirv/lib/SPIRV/SPIRVReader.cpp

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2508,20 +2508,12 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
25082508
return mapValue(BV,
25092509
transRelational(static_cast<SPIRVInstruction *>(BV), BB));
25102510
case OpIAddCarry: {
2511-
IRBuilder Builder(BB);
25122511
auto *BC = static_cast<SPIRVBinary *>(BV);
2513-
return mapValue(BV, Builder.CreateBinaryIntrinsic(
2514-
Intrinsic::uadd_with_overflow,
2515-
transValue(BC->getOperand(0), F, BB),
2516-
transValue(BC->getOperand(1), F, BB)));
2512+
return mapValue(BV, transBuiltinFromInst("__spirv_IAddCarry", BC, BB));
25172513
}
25182514
case OpISubBorrow: {
2519-
IRBuilder Builder(BB);
25202515
auto *BC = static_cast<SPIRVBinary *>(BV);
2521-
return mapValue(BV, Builder.CreateBinaryIntrinsic(
2522-
Intrinsic::usub_with_overflow,
2523-
transValue(BC->getOperand(0), F, BB),
2524-
transValue(BC->getOperand(1), F, BB)));
2516+
return mapValue(BV, transBuiltinFromInst("__spirv_ISubBorrow", BC, BB));
25252517
}
25262518
case OpGetKernelWorkGroupSize:
25272519
case OpGetKernelPreferredWorkGroupSizeMultiple:

‎llvm-spirv/lib/SPIRV/SPIRVUtil.cpp

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name) {
185185
}
186186

187187
void getFunctionTypeParameterTypes(llvm::FunctionType *FT,
188-
std::vector<Type *> &ArgTys) {
188+
SmallVector<Type *> &ArgTys) {
189189
for (auto I = FT->param_begin(), E = FT->param_end(); I != E; ++I) {
190190
ArgTys.push_back(*I);
191191
}
@@ -2194,23 +2194,38 @@ bool postProcessBuiltinReturningStruct(Function *F) {
21942194
SmallVector<Instruction *, 32> InstToRemove;
21952195
for (auto *U : F->users()) {
21962196
if (auto *CI = dyn_cast<CallInst>(U)) {
2197-
auto *ST = cast<StoreInst>(*(CI->user_begin()));
2198-
std::vector<Type *> ArgTys;
2197+
IRBuilder<> Builder(CI->getParent());
2198+
Builder.SetInsertPoint(CI);
2199+
SmallVector<User *> Users(CI->users());
2200+
Value *A = nullptr;
2201+
for (auto *U : Users) {
2202+
if (auto *SI = dyn_cast<StoreInst>(U)) {
2203+
A = SI->getPointerOperand();
2204+
InstToRemove.push_back(SI);
2205+
break;
2206+
}
2207+
}
2208+
if (!A) {
2209+
A = Builder.CreateAlloca(F->getReturnType());
2210+
}
2211+
SmallVector<Type *> ArgTys;
21992212
getFunctionTypeParameterTypes(F->getFunctionType(), ArgTys);
2200-
ArgTys.insert(ArgTys.begin(),
2201-
PointerType::get(F->getReturnType(), SPIRAS_Private));
2213+
ArgTys.insert(ArgTys.begin(), A->getType());
22022214
auto *NewF =
22032215
getOrCreateFunction(M, Type::getVoidTy(*Context), ArgTys, Name);
22042216
auto SretAttr = Attribute::get(*Context, Attribute::AttrKind::StructRet,
22052217
F->getReturnType());
22062218
NewF->addParamAttr(0, SretAttr);
22072219
NewF->setCallingConv(F->getCallingConv());
22082220
auto Args = getArguments(CI);
2209-
Args.insert(Args.begin(), ST->getPointerOperand());
2210-
auto *NewCI = CallInst::Create(NewF, Args, CI->getName(), CI);
2221+
Args.insert(Args.begin(), A);
2222+
CallInst *NewCI = Builder.CreateCall(NewF, Args, CI->getName());
22112223
NewCI->addParamAttr(0, SretAttr);
22122224
NewCI->setCallingConv(CI->getCallingConv());
2213-
InstToRemove.push_back(ST);
2225+
SmallVector<User *> CIUsers(CI->users());
2226+
for (auto *CIUser : CIUsers) {
2227+
CIUser->replaceUsesOfWith(CI, A);
2228+
}
22142229
InstToRemove.push_back(CI);
22152230
}
22162231
}

‎llvm-spirv/lib/SPIRV/SPIRVWriter.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6447,6 +6447,22 @@ LLVMToSPIRVBase::transBuiltinToInstWithoutDecoration(Op OC, CallInst *CI,
64476447
return BM->addCompositeConstructInst(transType(CI->getType()), Operands,
64486448
BB);
64496449
}
6450+
case OpIAddCarry: {
6451+
Function *F = CI->getCalledFunction();
6452+
StructType *St = cast<StructType>(F->getParamStructRetType(0));
6453+
SPIRVValue *V = BM->addBinaryInst(OpIAddCarry, transType(St),
6454+
transValue(CI->getArgOperand(1), BB),
6455+
transValue(CI->getArgOperand(2), BB), BB);
6456+
return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), V, {}, BB);
6457+
}
6458+
case OpISubBorrow: {
6459+
Function *F = CI->getCalledFunction();
6460+
StructType *St = cast<StructType>(F->getParamStructRetType(0));
6461+
SPIRVValue *V = BM->addBinaryInst(OpISubBorrow, transType(St),
6462+
transValue(CI->getArgOperand(1), BB),
6463+
transValue(CI->getArgOperand(2), BB), BB);
6464+
return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), V, {}, BB);
6465+
}
64506466
case OpGroupNonUniformShuffleDown: {
64516467
Function *F = CI->getCalledFunction();
64526468
if (F->arg_size() && F->getArg(0)->hasStructRetAttr()) {

‎llvm-spirv/test/iaddcarry_builtin.ll

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
; REQUIRES: spirv-dis
2+
; RUN: llvm-as %s -o %t.bc
3+
; RUN: llvm-spirv %t.bc -o %t.spv
4+
; RUN: spirv-dis --raw-id %t.spv | FileCheck --check-prefix CHECK-SPIRV %s
5+
; RUN: spirv-val %t.spv
6+
; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
7+
; RUN: llvm-dis %t.rev.bc -o - | FileCheck --check-prefix CHECK-LLVM %s
8+
9+
target triple = "spir64-unknown-unknown"
10+
11+
%i8struct = type {i8, i8}
12+
%i16struct = type {i16, i16}
13+
%i32struct = type {i32, i32}
14+
%i64struct = type {i64, i64}
15+
%vecstruct = type {<4 x i32>, <4 x i32>}
16+
17+
; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
18+
; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
19+
; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32
20+
; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
21+
; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid
22+
; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
23+
; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
24+
; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
25+
; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
26+
; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
27+
; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
28+
; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
29+
; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
30+
; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
31+
; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
32+
; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
33+
; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
34+
; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
35+
; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
36+
37+
; CHECK-LLVM-DAG: [[i8struct:%[a-z0-9_.]+]] = type { i8, i8 }
38+
; CHECK-LLVM-DAG: [[i16struct:%[a-z0-9_.]+]] = type { i16, i16 }
39+
; CHECK-LLVM-DAG: [[i32struct:%[a-z0-9_.]+]] = type { i32, i32 }
40+
; CHECK-LLVM-DAG: [[i64struct:%[a-z0-9_.]+]] = type { i64, i64 }
41+
; CHECK-LLVM-DAG: [[vecstruct:%[a-z0-9_.]+]] = type { <4 x i32>, <4 x i32> }
42+
; CHECK-LLVM-DAG: [[struct_anon:%[a-z0-9_.]+]] = type { i32, i32 }
43+
44+
define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) {
45+
entry:
46+
%0 = alloca %i8struct
47+
call void @_Z17__spirv_IAddCarrycc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
48+
ret void
49+
}
50+
; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
51+
; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
52+
; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel
53+
; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
54+
; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpIAddCarry [[i8struct]] [[a]] [[b]]
55+
; CHECK-SPIRV: OpStore [[var_11]] [[var_12]]
56+
; CHECK-SPIRV: OpReturn
57+
; CHECK-SPIRV: OpFunctionEnd
58+
59+
; CHECK-LLVM: %0 = alloca [[i8struct]], align 8
60+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarrycc(ptr sret([[i8struct]]) %0, i8 %a, i8 %b)
61+
; CHECK-LLVM: ret void
62+
define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) {
63+
entry:
64+
%0 = alloca %i16struct
65+
call void @_Z17__spirv_IAddCarryss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
66+
ret void
67+
}
68+
; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
69+
; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
70+
; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel
71+
; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
72+
; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpIAddCarry [[i16struct]] [[a_0]] [[b_0]]
73+
; CHECK-SPIRV: OpStore [[var_21]] [[var_22]]
74+
; CHECK-SPIRV: OpReturn
75+
; CHECK-SPIRV: OpFunctionEnd
76+
77+
; CHECK-LLVM: %0 = alloca [[i16struct]], align 8
78+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryss(ptr sret([[i16struct]]) %0, i16 %a, i16 %b)
79+
; CHECK-LLVM: ret void
80+
define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) {
81+
entry:
82+
%0 = alloca %i32struct
83+
call void @_Z17__spirv_IAddCarryii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
84+
ret void
85+
}
86+
; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
87+
; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
88+
; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel
89+
; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
90+
; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpIAddCarry [[i32struct]] [[a_1]] [[b_1]]
91+
; CHECK-SPIRV: OpStore [[var_31]] [[var_32]]
92+
; CHECK-SPIRV: OpReturn
93+
; CHECK-SPIRV: OpFunctionEnd
94+
95+
; CHECK-LLVM: %0 = alloca [[i32struct]], align 8
96+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryii(ptr sret([[i32struct]]) %0, i32 %a, i32 %b)
97+
; CHECK-LLVM: ret void
98+
define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) {
99+
entry:
100+
%0 = alloca %i64struct
101+
call void @_Z17__spirv_IAddCarryll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
102+
ret void
103+
}
104+
; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
105+
; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
106+
; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel
107+
; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
108+
; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpIAddCarry [[i64struct]] [[a_2]] [[b_2]]
109+
; CHECK-SPIRV: OpStore [[var_41]] [[var_42]]
110+
; CHECK-SPIRV: OpReturn
111+
; CHECK-SPIRV: OpFunctionEnd
112+
113+
; CHECK-LLVM: %0 = alloca [[i64struct]]
114+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryll(ptr sret([[i64struct]]) %0, i64 %a, i64 %b)
115+
; CHECK-LLVM: ret void
116+
define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
117+
entry:
118+
%0 = alloca %vecstruct
119+
call void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
120+
ret void
121+
}
122+
; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
123+
; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
124+
; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel
125+
; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
126+
; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpIAddCarry [[vecstruct]] [[a_3]] [[b_3]]
127+
; CHECK-SPIRV: OpStore [[var_51]] [[var_52]]
128+
; CHECK-SPIRV: OpReturn
129+
; CHECK-SPIRV: OpFunctionEnd
130+
131+
; CHECK-LLVM: %0 = alloca [[vecstruct]]
132+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret([[vecstruct]]) %0, <4 x i32> %a, <4 x i32> %b)
133+
; CHECK-LLVM: ret void
134+
135+
%struct.anon = type { i32, i32 }
136+
137+
define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) {
138+
entry:
139+
%0 = alloca %struct.anon
140+
%1 = addrspacecast ptr %0 to ptr addrspace(4)
141+
call spir_func void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
142+
ret void
143+
}
144+
; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
145+
; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
146+
; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel
147+
; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
148+
; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
149+
; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpIAddCarry [[struct_anon]] [[a_4]] [[b_4]]
150+
; CHECK-SPIRV: OpStore [[var_61]] [[var_62]]
151+
152+
; CHECK-LLVM: %0 = alloca [[struct_anon]], align 8
153+
; CHECK-LLVM: %1 = addrspacecast ptr %0 to ptr addrspace(4)
154+
; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryii.1(ptr addrspace(4) sret([[struct_anon]]) %1, i32 %a, i32 %b)
155+
; CHECK-LLVM: ret void
156+
157+
declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
158+
declare void @_Z17__spirv_IAddCarrycc(ptr sret(%i8struct), i8, i8)
159+
declare void @_Z17__spirv_IAddCarryss(ptr sret(%i16struct), i16, i16)
160+
declare void @_Z17__spirv_IAddCarryii(ptr sret(%i32struct), i32, i32)
161+
declare void @_Z17__spirv_IAddCarryll(ptr sret(%i64struct), i64, i64)
162+
declare void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)

0 commit comments

Comments
 (0)