Skip to content

Commit b337237

Browse files
Artem-BArtem-B
authored
[NVPTX] Fixed few more corner cases for v4i8 lowering. (llvm#69263)
Fixes llvm#69124
1 parent ddc30ff commit b337237

File tree

6 files changed

+180
-16
lines changed

6 files changed

+180
-16
lines changed

‎llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -504,13 +504,21 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
504504
// Only logical ops can be done on v4i8 directly, others must be done
505505
// elementwise.
506506
setOperationAction(
507-
{ISD::ADD, ISD::MUL, ISD::ABS, ISD::SMIN,
508-
ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::CTPOP,
509-
ISD::CTLZ, ISD::ADD, ISD::SUB, ISD::MUL,
510-
ISD::SHL, ISD::SREM, ISD::UREM, ISD::SDIV,
511-
ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,
512-
ISD::MULHU, ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
513-
ISD::UINT_TO_FP},
507+
{ISD::ABS, ISD::ADD, ISD::ADDC, ISD::ADDE,
508+
ISD::BITREVERSE, ISD::CTLZ, ISD::CTPOP, ISD::CTTZ,
509+
ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FSHL, ISD::FSHR,
510+
ISD::MUL, ISD::MULHS, ISD::MULHU, ISD::PARITY,
511+
ISD::ROTL, ISD::ROTR, ISD::SADDO, ISD::SADDO_CARRY,
512+
ISD::SADDSAT, ISD::SDIV, ISD::SDIVREM, ISD::SELECT_CC,
513+
ISD::SETCC, ISD::SHL, ISD::SINT_TO_FP, ISD::SMAX,
514+
ISD::SMIN, ISD::SMULO, ISD::SMUL_LOHI, ISD::SRA,
515+
ISD::SREM, ISD::SRL, ISD::SSHLSAT, ISD::SSUBO,
516+
ISD::SSUBO_CARRY, ISD::SSUBSAT, ISD::SUB, ISD::SUBC,
517+
ISD::SUBE, ISD::UADDO, ISD::UADDO_CARRY, ISD::UADDSAT,
518+
ISD::UDIV, ISD::UDIVREM, ISD::UINT_TO_FP, ISD::UMAX,
519+
ISD::UMIN, ISD::UMULO, ISD::UMUL_LOHI, ISD::UREM,
520+
ISD::USHLSAT, ISD::USUBO, ISD::USUBO_CARRY, ISD::VSELECT,
521+
ISD::USUBSAT},
514522
MVT::v4i8, Expand);
515523

516524
// Operations not directly supported by NVPTX.

‎llvm/lib/Target/NVPTX/NVPTXISelLowering.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,12 @@ class NVPTXTargetLowering : public TargetLowering {
586586
AtomicExpansionKind
587587
shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
588588

589+
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {
590+
// There's rarely any point of packing something into a vector type if we
591+
// already have the source data.
592+
return true;
593+
}
594+
589595
private:
590596
const NVPTXSubtarget &STI; // cache the subtarget here
591597
SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;

‎llvm/lib/Target/NVPTX/NVPTXInstrInfo.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3485,6 +3485,9 @@ def : Pat<(v2bf16 (build_vector (bf16 Int16Regs:$a), (bf16 Int16Regs:$b))),
34853485
def : Pat<(v2i16 (build_vector (i16 Int16Regs:$a), (i16 Int16Regs:$b))),
34863486
(V2I16toI32 Int16Regs:$a, Int16Regs:$b)>;
34873487

3488+
def: Pat<(v2i16 (scalar_to_vector (i16 Int16Regs:$a))),
3489+
(CVT_u32_u16 Int16Regs:$a, CvtNONE)>;
3490+
34883491
// Count leading zeros
34893492
let hasSideEffects = false in {
34903493
def CLZr32 : NVPTXInst<(outs Int32Regs:$d), (ins Int32Regs:$a),

‎llvm/test/CodeGen/NVPTX/f16x2-instructions.ll

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1319,10 +1319,8 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
13191319
; CHECK-DAG: and.b16 [[BX1:%rs[0-9]+]], [[B1]], -32768;
13201320
; CHECK-DAG: or.b16 [[R0:%rs[0-9]+]], [[AX0]], [[BX0]];
13211321
; CHECK-DAG: or.b16 [[R1:%rs[0-9]+]], [[AX1]], [[BX1]];
1322-
; CHECK-DAG: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}
1323-
; CHECK: mov.b32 {[[RX0:%rs[0-9]+]], [[RX1:%rs[0-9]+]]}, [[R]]
1324-
; CHECK-DAG: cvt.f32.f16 [[XR0:%f[0-9]+]], [[RX0]];
1325-
; CHECK-DAG: cvt.f32.f16 [[XR1:%f[0-9]+]], [[RX1]];
1322+
; CHECK-DAG: cvt.f32.f16 [[XR0:%f[0-9]+]], [[R0]];
1323+
; CHECK-DAG: cvt.f32.f16 [[XR1:%f[0-9]+]], [[R1]];
13261324
; CHECK: st.param.v2.f32 [func_retval0+0], {[[XR0]], [[XR1]]};
13271325
; CHECK: ret;
13281326
define <2 x float> @test_copysign_extended(<2 x half> %a, <2 x half> %b) #0 {

‎llvm/test/CodeGen/NVPTX/i8x4-instructions.ll

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1269,4 +1269,158 @@ define <4 x i8> @test_fptoui_2xhalf_to_2xi8(<4 x half> %a) #0 {
12691269
ret <4 x i8> %r
12701270
}
12711271

1272+
define void @test_srem_v4i8(ptr %a, ptr %b, ptr %c) {
1273+
; CHECK-LABEL: test_srem_v4i8(
1274+
; CHECK: {
1275+
; CHECK-NEXT: .reg .b16 %rs<13>;
1276+
; CHECK-NEXT: .reg .b32 %r<18>;
1277+
; CHECK-NEXT: .reg .b64 %rd<4>;
1278+
; CHECK-EMPTY:
1279+
; CHECK-NEXT: // %bb.0: // %entry
1280+
; CHECK-NEXT: ld.param.u64 %rd3, [test_srem_v4i8_param_2];
1281+
; CHECK-NEXT: ld.param.u64 %rd2, [test_srem_v4i8_param_1];
1282+
; CHECK-NEXT: ld.param.u64 %rd1, [test_srem_v4i8_param_0];
1283+
; CHECK-NEXT: ld.u32 %r1, [%rd1];
1284+
; CHECK-NEXT: ld.u32 %r2, [%rd2];
1285+
; CHECK-NEXT: bfe.s32 %r3, %r2, 0, 8;
1286+
; CHECK-NEXT: cvt.s8.s32 %rs1, %r3;
1287+
; CHECK-NEXT: bfe.s32 %r4, %r1, 0, 8;
1288+
; CHECK-NEXT: cvt.s8.s32 %rs2, %r4;
1289+
; CHECK-NEXT: rem.s16 %rs3, %rs2, %rs1;
1290+
; CHECK-NEXT: cvt.u32.u16 %r5, %rs3;
1291+
; CHECK-NEXT: bfe.s32 %r6, %r2, 8, 8;
1292+
; CHECK-NEXT: cvt.s8.s32 %rs4, %r6;
1293+
; CHECK-NEXT: bfe.s32 %r7, %r1, 8, 8;
1294+
; CHECK-NEXT: cvt.s8.s32 %rs5, %r7;
1295+
; CHECK-NEXT: rem.s16 %rs6, %rs5, %rs4;
1296+
; CHECK-NEXT: cvt.u32.u16 %r8, %rs6;
1297+
; CHECK-NEXT: bfi.b32 %r9, %r8, %r5, 8, 8;
1298+
; CHECK-NEXT: bfe.s32 %r10, %r2, 16, 8;
1299+
; CHECK-NEXT: cvt.s8.s32 %rs7, %r10;
1300+
; CHECK-NEXT: bfe.s32 %r11, %r1, 16, 8;
1301+
; CHECK-NEXT: cvt.s8.s32 %rs8, %r11;
1302+
; CHECK-NEXT: rem.s16 %rs9, %rs8, %rs7;
1303+
; CHECK-NEXT: cvt.u32.u16 %r12, %rs9;
1304+
; CHECK-NEXT: bfi.b32 %r13, %r12, %r9, 16, 8;
1305+
; CHECK-NEXT: bfe.s32 %r14, %r2, 24, 8;
1306+
; CHECK-NEXT: cvt.s8.s32 %rs10, %r14;
1307+
; CHECK-NEXT: bfe.s32 %r15, %r1, 24, 8;
1308+
; CHECK-NEXT: cvt.s8.s32 %rs11, %r15;
1309+
; CHECK-NEXT: rem.s16 %rs12, %rs11, %rs10;
1310+
; CHECK-NEXT: cvt.u32.u16 %r16, %rs12;
1311+
; CHECK-NEXT: bfi.b32 %r17, %r16, %r13, 24, 8;
1312+
; CHECK-NEXT: st.u32 [%rd3], %r17;
1313+
; CHECK-NEXT: ret;
1314+
entry:
1315+
%t57 = load <4 x i8>, ptr %a, align 4
1316+
%t59 = load <4 x i8>, ptr %b, align 4
1317+
%x = srem <4 x i8> %t57, %t59
1318+
store <4 x i8> %x, ptr %c, align 4
1319+
ret void
1320+
}
1321+
1322+
;; v3i8 lowering, especially for unaligned loads is terrible. We end up doing
1323+
;; tons of pointless scalar_to_vector/bitcast/extract_elt on v2i16/v4i8, which
1324+
;; is further complicated by LLVM trying to use i16 as an intermediate type,
1325+
;; because we don't have i8 registers. It's a mess.
1326+
;; Ideally we want to split it into element-wise ops, but legalizer can't handle
1327+
;; odd-sized vectors. TL;DR; don't use odd-sized vectors of v8.
1328+
define void @test_srem_v3i8(ptr %a, ptr %b, ptr %c) {
1329+
; CHECK-LABEL: test_srem_v3i8(
1330+
; CHECK: {
1331+
; CHECK-NEXT: .reg .b16 %rs<20>;
1332+
; CHECK-NEXT: .reg .b32 %r<16>;
1333+
; CHECK-NEXT: .reg .b64 %rd<4>;
1334+
; CHECK-EMPTY:
1335+
; CHECK-NEXT: // %bb.0: // %entry
1336+
; CHECK-NEXT: ld.param.u64 %rd3, [test_srem_v3i8_param_2];
1337+
; CHECK-NEXT: ld.param.u64 %rd2, [test_srem_v3i8_param_1];
1338+
; CHECK-NEXT: ld.param.u64 %rd1, [test_srem_v3i8_param_0];
1339+
; CHECK-NEXT: ld.u8 %rs1, [%rd1];
1340+
; CHECK-NEXT: ld.u8 %rs2, [%rd1+1];
1341+
; CHECK-NEXT: shl.b16 %rs3, %rs2, 8;
1342+
; CHECK-NEXT: or.b16 %rs4, %rs3, %rs1;
1343+
; CHECK-NEXT: cvt.u32.u16 %r1, %rs4;
1344+
; CHECK-NEXT: ld.s8 %rs5, [%rd1+2];
1345+
; CHECK-NEXT: ld.u8 %rs6, [%rd2];
1346+
; CHECK-NEXT: ld.u8 %rs7, [%rd2+1];
1347+
; CHECK-NEXT: shl.b16 %rs8, %rs7, 8;
1348+
; CHECK-NEXT: or.b16 %rs9, %rs8, %rs6;
1349+
; CHECK-NEXT: cvt.u32.u16 %r3, %rs9;
1350+
; CHECK-NEXT: ld.s8 %rs10, [%rd2+2];
1351+
; CHECK-NEXT: bfe.s32 %r5, %r3, 0, 8;
1352+
; CHECK-NEXT: cvt.s8.s32 %rs11, %r5;
1353+
; CHECK-NEXT: bfe.s32 %r6, %r1, 0, 8;
1354+
; CHECK-NEXT: cvt.s8.s32 %rs12, %r6;
1355+
; CHECK-NEXT: rem.s16 %rs13, %rs12, %rs11;
1356+
; CHECK-NEXT: cvt.u32.u16 %r7, %rs13;
1357+
; CHECK-NEXT: bfe.s32 %r8, %r3, 8, 8;
1358+
; CHECK-NEXT: cvt.s8.s32 %rs14, %r8;
1359+
; CHECK-NEXT: bfe.s32 %r9, %r1, 8, 8;
1360+
; CHECK-NEXT: cvt.s8.s32 %rs15, %r9;
1361+
; CHECK-NEXT: rem.s16 %rs16, %rs15, %rs14;
1362+
; CHECK-NEXT: cvt.u32.u16 %r10, %rs16;
1363+
; CHECK-NEXT: bfi.b32 %r11, %r10, %r7, 8, 8;
1364+
; CHECK-NEXT: // implicit-def: %r13
1365+
; CHECK-NEXT: bfi.b32 %r12, %r13, %r11, 16, 8;
1366+
; CHECK-NEXT: // implicit-def: %r15
1367+
; CHECK-NEXT: bfi.b32 %r14, %r15, %r12, 24, 8;
1368+
; CHECK-NEXT: rem.s16 %rs17, %rs5, %rs10;
1369+
; CHECK-NEXT: cvt.u16.u32 %rs18, %r14;
1370+
; CHECK-NEXT: st.u8 [%rd3], %rs18;
1371+
; CHECK-NEXT: shr.u16 %rs19, %rs18, 8;
1372+
; CHECK-NEXT: st.u8 [%rd3+1], %rs19;
1373+
; CHECK-NEXT: st.u8 [%rd3+2], %rs17;
1374+
; CHECK-NEXT: ret;
1375+
entry:
1376+
%t57 = load <3 x i8>, ptr %a, align 1
1377+
%t59 = load <3 x i8>, ptr %b, align 1
1378+
%x = srem <3 x i8> %t57, %t59
1379+
store <3 x i8> %x, ptr %c, align 1
1380+
ret void
1381+
}
1382+
1383+
define void @test_sext_v4i1_to_v4i8(ptr %a, ptr %b, ptr %c) {
1384+
; CHECK-LABEL: test_sext_v4i1_to_v4i8(
1385+
; CHECK: {
1386+
; CHECK-NEXT: .reg .pred %p<5>;
1387+
; CHECK-NEXT: .reg .b32 %r<18>;
1388+
; CHECK-NEXT: .reg .b64 %rd<4>;
1389+
; CHECK-EMPTY:
1390+
; CHECK-NEXT: // %bb.0: // %entry
1391+
; CHECK-NEXT: ld.param.u64 %rd3, [test_sext_v4i1_to_v4i8_param_2];
1392+
; CHECK-NEXT: ld.param.u64 %rd2, [test_sext_v4i1_to_v4i8_param_1];
1393+
; CHECK-NEXT: ld.param.u64 %rd1, [test_sext_v4i1_to_v4i8_param_0];
1394+
; CHECK-NEXT: ld.u32 %r1, [%rd1];
1395+
; CHECK-NEXT: ld.u32 %r2, [%rd2];
1396+
; CHECK-NEXT: bfe.s32 %r3, %r2, 24, 8;
1397+
; CHECK-NEXT: bfe.s32 %r4, %r1, 24, 8;
1398+
; CHECK-NEXT: setp.hi.u32 %p1, %r4, %r3;
1399+
; CHECK-NEXT: bfe.s32 %r5, %r2, 16, 8;
1400+
; CHECK-NEXT: bfe.s32 %r6, %r1, 16, 8;
1401+
; CHECK-NEXT: setp.hi.u32 %p2, %r6, %r5;
1402+
; CHECK-NEXT: bfe.s32 %r7, %r2, 8, 8;
1403+
; CHECK-NEXT: bfe.s32 %r8, %r1, 8, 8;
1404+
; CHECK-NEXT: setp.hi.u32 %p3, %r8, %r7;
1405+
; CHECK-NEXT: bfe.s32 %r9, %r2, 0, 8;
1406+
; CHECK-NEXT: bfe.s32 %r10, %r1, 0, 8;
1407+
; CHECK-NEXT: setp.hi.u32 %p4, %r10, %r9;
1408+
; CHECK-NEXT: selp.s32 %r11, -1, 0, %p4;
1409+
; CHECK-NEXT: selp.s32 %r12, -1, 0, %p3;
1410+
; CHECK-NEXT: bfi.b32 %r13, %r12, %r11, 8, 8;
1411+
; CHECK-NEXT: selp.s32 %r14, -1, 0, %p2;
1412+
; CHECK-NEXT: bfi.b32 %r15, %r14, %r13, 16, 8;
1413+
; CHECK-NEXT: selp.s32 %r16, -1, 0, %p1;
1414+
; CHECK-NEXT: bfi.b32 %r17, %r16, %r15, 24, 8;
1415+
; CHECK-NEXT: st.u32 [%rd3], %r17;
1416+
; CHECK-NEXT: ret;
1417+
entry:
1418+
%t1 = load <4 x i8>, ptr %a, align 4
1419+
%t2 = load <4 x i8>, ptr %b, align 4
1420+
%t5 = icmp ugt <4 x i8> %t1, %t2
1421+
%t6 = sext <4 x i1> %t5 to <4 x i8>
1422+
store <4 x i8> %t6, ptr %c, align 4
1423+
ret void
1424+
}
1425+
12721426
attributes #0 = { nounwind }

‎llvm/test/CodeGen/NVPTX/param-load-store.ll

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -364,10 +364,6 @@ define <4 x i16> @test_v4i16(<4 x i16> %a) {
364364
; CHECK-NEXT: .param .align 16 .b8 test_v5i16_param_0[16]
365365
; CHECK-DAG: ld.param.u16 [[E4:%rs[0-9]+]], [test_v5i16_param_0+8];
366366
; CHECK-DAG: ld.param.v4.u16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]], [[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [test_v5i16_param_0]
367-
; CHECK-DAG: mov.b32 [[R0:%r[0-9]+]], {[[E0]], [[E1]]};
368-
; CHECK-DAG: mov.b32 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[R0]];
369-
; CHECK-DAG: mov.b32 [[R1:%r[0-9]+]], {[[E2]], [[E3]]};
370-
; CHECK-DAG: mov.b32 {[[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [[R1]];
371367
; CHECK: .param .align 16 .b8 param0[16];
372368
; CHECK-DAG: st.param.v4.b16 [param0+0], {[[E0]], [[E1]], [[E2]], [[E3]]};
373369
; CHECK-DAG: st.param.b16 [param0+8], [[E4]];
@@ -496,7 +492,6 @@ define <4 x half> @test_v4f16(<4 x half> %a) {
496492
; CHECK-LABEL: test_v5f16(
497493
; CHECK: .param .align 16 .b8 test_v5f16_param_0[16]
498494
; CHECK-DAG: ld.param.v4.b16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]], [[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [test_v5f16_param_0];
499-
; CHECK-DAG: mov.b32 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[HH01]];
500495
; CHECK-DAG: ld.param.b16 [[E4:%rs[0-9]+]], [test_v5f16_param_0+8];
501496
; CHECK: .param .align 16 .b8 param0[16];
502497
; CHECK-DAG: st.param.v4.b16 [param0+0],

0 commit comments

Comments
 (0)