Skip to content

Commit 30f58ab

Browse files
lukel97lukel97
authored
[RISCV] Lower vector_reverse for zvfhmin/zvfbfmin (#110218)
Previously we crashed because we had no lowering for f16/bf16 scalable vectors. Because the lowering uses vrgather_vv_vl, we need to add bf16 patterns for it.
1 parent 4db1056 commit 30f58ab

File tree

3 files changed

+122
-9
lines changed

3 files changed

+122
-9
lines changed

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,8 +1077,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
10771077
ISD::VP_UINT_TO_FP},
10781078
VT, Custom);
10791079
setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1080-
ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_INTERLEAVE,
1081-
ISD::VECTOR_DEINTERLEAVE},
1080+
ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
1081+
ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE},
10821082
VT, Custom);
10831083
MVT EltVT = VT.getVectorElementType();
10841084
if (isTypeLegal(EltVT))

‎llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2952,7 +2952,7 @@ foreach vti = NoGroupFloatVectors in {
29522952
}
29532953
}
29542954

2955-
foreach vti = AllFloatVectors in {
2955+
foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
29562956
defvar ivti = GetIntVTypeInfo<vti>.Vti;
29572957
let Predicates = GetVTypePredicates<ivti>.Predicates in {
29582958
def : Pat<(vti.Vector

‎llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll

Lines changed: 119 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2-
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
3-
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
4-
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
5-
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
6-
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
7-
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
2+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
3+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
4+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
5+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
6+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
7+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
8+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
9+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
10+
; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
11+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
12+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
13+
; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
814

915
;
1016
; VECTOR_REVERSE - masks
@@ -1515,6 +1521,113 @@ define <vscale x 8 x i64> @reverse_nxv8i64(<vscale x 8 x i64> %a) {
15151521
; VECTOR_REVERSE - floating point
15161522
;
15171523

1524+
define <vscale x 1 x bfloat> @reverse_nxv1bf16(<vscale x 1 x bfloat> %a) {
1525+
; CHECK-LABEL: reverse_nxv1bf16:
1526+
; CHECK: # %bb.0:
1527+
; CHECK-NEXT: csrr a0, vlenb
1528+
; CHECK-NEXT: srli a0, a0, 3
1529+
; CHECK-NEXT: addi a0, a0, -1
1530+
; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma
1531+
; CHECK-NEXT: vid.v v9
1532+
; CHECK-NEXT: vrsub.vx v10, v9, a0
1533+
; CHECK-NEXT: vrgather.vv v9, v8, v10
1534+
; CHECK-NEXT: vmv1r.v v8, v9
1535+
; CHECK-NEXT: ret
1536+
%res = call <vscale x 1 x bfloat> @llvm.vector.reverse.nxv1bf16(<vscale x 1 x bfloat> %a)
1537+
ret <vscale x 1 x bfloat> %res
1538+
}
1539+
1540+
define <vscale x 2 x bfloat> @reverse_nxv2bf16(<vscale x 2 x bfloat> %a) {
1541+
; CHECK-LABEL: reverse_nxv2bf16:
1542+
; CHECK: # %bb.0:
1543+
; CHECK-NEXT: csrr a0, vlenb
1544+
; CHECK-NEXT: srli a0, a0, 2
1545+
; CHECK-NEXT: addi a0, a0, -1
1546+
; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma
1547+
; CHECK-NEXT: vid.v v9
1548+
; CHECK-NEXT: vrsub.vx v10, v9, a0
1549+
; CHECK-NEXT: vrgather.vv v9, v8, v10
1550+
; CHECK-NEXT: vmv1r.v v8, v9
1551+
; CHECK-NEXT: ret
1552+
%res = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> %a)
1553+
ret <vscale x 2 x bfloat> %res
1554+
}
1555+
1556+
define <vscale x 4 x bfloat> @reverse_nxv4bf16(<vscale x 4 x bfloat> %a) {
1557+
; CHECK-LABEL: reverse_nxv4bf16:
1558+
; CHECK: # %bb.0:
1559+
; CHECK-NEXT: csrr a0, vlenb
1560+
; CHECK-NEXT: srli a0, a0, 1
1561+
; CHECK-NEXT: addi a0, a0, -1
1562+
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
1563+
; CHECK-NEXT: vid.v v9
1564+
; CHECK-NEXT: vrsub.vx v10, v9, a0
1565+
; CHECK-NEXT: vrgather.vv v9, v8, v10
1566+
; CHECK-NEXT: vmv.v.v v8, v9
1567+
; CHECK-NEXT: ret
1568+
%res = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> %a)
1569+
ret <vscale x 4 x bfloat> %res
1570+
}
1571+
1572+
define <vscale x 8 x bfloat> @reverse_nxv8bf16(<vscale x 8 x bfloat> %a) {
1573+
; CHECK-LABEL: reverse_nxv8bf16:
1574+
; CHECK: # %bb.0:
1575+
; CHECK-NEXT: csrr a0, vlenb
1576+
; CHECK-NEXT: srli a0, a0, 1
1577+
; CHECK-NEXT: addi a0, a0, -1
1578+
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
1579+
; CHECK-NEXT: vid.v v10
1580+
; CHECK-NEXT: vrsub.vx v12, v10, a0
1581+
; CHECK-NEXT: vrgather.vv v11, v8, v12
1582+
; CHECK-NEXT: vrgather.vv v10, v9, v12
1583+
; CHECK-NEXT: vmv2r.v v8, v10
1584+
; CHECK-NEXT: ret
1585+
%res = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> %a)
1586+
ret <vscale x 8 x bfloat> %res
1587+
}
1588+
1589+
define <vscale x 16 x bfloat> @reverse_nxv16bf16(<vscale x 16 x bfloat> %a) {
1590+
; CHECK-LABEL: reverse_nxv16bf16:
1591+
; CHECK: # %bb.0:
1592+
; CHECK-NEXT: csrr a0, vlenb
1593+
; CHECK-NEXT: srli a0, a0, 1
1594+
; CHECK-NEXT: addi a0, a0, -1
1595+
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
1596+
; CHECK-NEXT: vid.v v12
1597+
; CHECK-NEXT: vrsub.vx v16, v12, a0
1598+
; CHECK-NEXT: vrgather.vv v15, v8, v16
1599+
; CHECK-NEXT: vrgather.vv v14, v9, v16
1600+
; CHECK-NEXT: vrgather.vv v13, v10, v16
1601+
; CHECK-NEXT: vrgather.vv v12, v11, v16
1602+
; CHECK-NEXT: vmv4r.v v8, v12
1603+
; CHECK-NEXT: ret
1604+
%res = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> %a)
1605+
ret <vscale x 16 x bfloat> %res
1606+
}
1607+
1608+
define <vscale x 32 x bfloat> @reverse_nxv32bf16(<vscale x 32 x bfloat> %a) {
1609+
; CHECK-LABEL: reverse_nxv32bf16:
1610+
; CHECK: # %bb.0:
1611+
; CHECK-NEXT: vmv8r.v v16, v8
1612+
; CHECK-NEXT: csrr a0, vlenb
1613+
; CHECK-NEXT: srli a0, a0, 1
1614+
; CHECK-NEXT: addi a0, a0, -1
1615+
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
1616+
; CHECK-NEXT: vid.v v8
1617+
; CHECK-NEXT: vrsub.vx v24, v8, a0
1618+
; CHECK-NEXT: vrgather.vv v15, v16, v24
1619+
; CHECK-NEXT: vrgather.vv v14, v17, v24
1620+
; CHECK-NEXT: vrgather.vv v13, v18, v24
1621+
; CHECK-NEXT: vrgather.vv v12, v19, v24
1622+
; CHECK-NEXT: vrgather.vv v11, v20, v24
1623+
; CHECK-NEXT: vrgather.vv v10, v21, v24
1624+
; CHECK-NEXT: vrgather.vv v9, v22, v24
1625+
; CHECK-NEXT: vrgather.vv v8, v23, v24
1626+
; CHECK-NEXT: ret
1627+
%res = call <vscale x 32 x bfloat> @llvm.vector.reverse.nxv32bf16(<vscale x 32 x bfloat> %a)
1628+
ret <vscale x 32 x bfloat> %res
1629+
}
1630+
15181631
define <vscale x 1 x half> @reverse_nxv1f16(<vscale x 1 x half> %a) {
15191632
; CHECK-LABEL: reverse_nxv1f16:
15201633
; CHECK: # %bb.0:

0 commit comments

Comments
 (0)