Skip to content

Commit c319c74

Browse files
michaelmaitlandmichaelmaitland
committed
[RISCV] Improve performCONCAT_VECTORCombine stride matching
If the load ptrs can be decomposed into a common (Base + Index) with a common constant stride, then return the constant stride.
1 parent 30ca258 commit c319c74

File tree

2 files changed

+43
-94
lines changed

2 files changed

+43
-94
lines changed

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "llvm/CodeGen/MachineInstrBuilder.h"
2828
#include "llvm/CodeGen/MachineJumpTableInfo.h"
2929
#include "llvm/CodeGen/MachineRegisterInfo.h"
30+
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
3031
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
3132
#include "llvm/CodeGen/ValueTypes.h"
3233
#include "llvm/IR/DiagnosticInfo.h"
@@ -13803,9 +13804,17 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
1380313804
Align = std::min(Align, Ld->getAlign());
1380413805
}
1380513806

13806-
using PtrDiff = std::pair<SDValue, bool>;
13807-
auto GetPtrDiff = [](LoadSDNode *Ld1,
13808-
LoadSDNode *Ld2) -> std::optional<PtrDiff> {
13807+
using PtrDiff = std::pair<std::variant<int64_t, SDValue>, bool>;
13808+
auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
13809+
LoadSDNode *Ld2) -> std::optional<PtrDiff> {
13810+
// If the load ptrs can be decomposed into a common (Base + Index) with a
13811+
// common constant stride, then return the constant stride.
13812+
BaseIndexOffset BIO1 = BaseIndexOffset::match(Ld1, DAG);
13813+
BaseIndexOffset BIO2 = BaseIndexOffset::match(Ld2, DAG);
13814+
if (BIO1.equalBaseIndex(BIO2, DAG))
13815+
return {{BIO2.getOffset() - BIO1.getOffset(), false}};
13816+
13817+
// Otherwise try to match (add LastPtr, Stride) or (add NextPtr, Stride)
1380913818
SDValue P1 = Ld1->getBasePtr();
1381013819
SDValue P2 = Ld2->getBasePtr();
1381113820
if (P2.getOpcode() == ISD::ADD && P2.getOperand(0) == P1)
@@ -13844,7 +13853,11 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
1384413853
if (!TLI.isLegalStridedLoadStore(WideVecVT, Align))
1384513854
return SDValue();
1384613855

13847-
auto [Stride, MustNegateStride] = *BaseDiff;
13856+
auto [StrideVariant, MustNegateStride] = *BaseDiff;
13857+
SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
13858+
? std::get<SDValue>(StrideVariant)
13859+
: DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
13860+
Lds[0]->getOffset().getValueType());
1384813861
if (MustNegateStride)
1384913862
Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
1385013863

‎llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll

Lines changed: 26 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,10 @@
77
define void @constant_forward_stride(ptr %s, ptr %d) {
88
; CHECK-LABEL: constant_forward_stride:
99
; CHECK: # %bb.0:
10-
; CHECK-NEXT: addi a2, a0, 16
11-
; CHECK-NEXT: addi a3, a0, 32
12-
; CHECK-NEXT: addi a4, a0, 48
13-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
14-
; CHECK-NEXT: vle8.v v8, (a0)
15-
; CHECK-NEXT: vle8.v v9, (a2)
16-
; CHECK-NEXT: vle8.v v10, (a3)
17-
; CHECK-NEXT: vle8.v v11, (a4)
18-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
19-
; CHECK-NEXT: vslideup.vi v8, v9, 2
20-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
21-
; CHECK-NEXT: vslideup.vi v8, v10, 4
22-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
23-
; CHECK-NEXT: vslideup.vi v8, v11, 6
24-
; CHECK-NEXT: vse8.v v8, (a1)
10+
; CHECK-NEXT: li a2, 16
11+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
12+
; CHECK-NEXT: vlse16.v v8, (a0), a2
13+
; CHECK-NEXT: vse16.v v8, (a1)
2514
; CHECK-NEXT: ret
2615
%1 = getelementptr inbounds i8, ptr %s, i64 16
2716
%2 = getelementptr inbounds i8, ptr %s, i64 32
@@ -40,21 +29,11 @@ define void @constant_forward_stride(ptr %s, ptr %d) {
4029
define void @constant_forward_stride2(ptr %s, ptr %d) {
4130
; CHECK-LABEL: constant_forward_stride2:
4231
; CHECK: # %bb.0:
43-
; CHECK-NEXT: addi a2, a0, -16
44-
; CHECK-NEXT: addi a3, a0, -32
45-
; CHECK-NEXT: addi a4, a0, -48
46-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
47-
; CHECK-NEXT: vle8.v v8, (a4)
48-
; CHECK-NEXT: vle8.v v9, (a3)
49-
; CHECK-NEXT: vle8.v v10, (a2)
50-
; CHECK-NEXT: vle8.v v11, (a0)
51-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
52-
; CHECK-NEXT: vslideup.vi v8, v9, 2
53-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
54-
; CHECK-NEXT: vslideup.vi v8, v10, 4
55-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
56-
; CHECK-NEXT: vslideup.vi v8, v11, 6
57-
; CHECK-NEXT: vse8.v v8, (a1)
32+
; CHECK-NEXT: addi a0, a0, -48
33+
; CHECK-NEXT: li a2, 16
34+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
35+
; CHECK-NEXT: vlse16.v v8, (a0), a2
36+
; CHECK-NEXT: vse16.v v8, (a1)
5837
; CHECK-NEXT: ret
5938
%1 = getelementptr inbounds i8, ptr %s, i64 -16
6039
%2 = getelementptr inbounds i8, ptr %s, i64 -32
@@ -73,21 +52,10 @@ define void @constant_forward_stride2(ptr %s, ptr %d) {
7352
define void @constant_forward_stride3(ptr %s, ptr %d) {
7453
; CHECK-LABEL: constant_forward_stride3:
7554
; CHECK: # %bb.0:
76-
; CHECK-NEXT: addi a2, a0, 16
77-
; CHECK-NEXT: addi a3, a0, 32
78-
; CHECK-NEXT: addi a4, a0, 48
79-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
80-
; CHECK-NEXT: vle8.v v8, (a0)
81-
; CHECK-NEXT: vle8.v v9, (a2)
82-
; CHECK-NEXT: vle8.v v10, (a3)
83-
; CHECK-NEXT: vle8.v v11, (a4)
84-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
85-
; CHECK-NEXT: vslideup.vi v8, v9, 2
86-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
87-
; CHECK-NEXT: vslideup.vi v8, v10, 4
88-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
89-
; CHECK-NEXT: vslideup.vi v8, v11, 6
90-
; CHECK-NEXT: vse8.v v8, (a1)
55+
; CHECK-NEXT: li a2, 16
56+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
57+
; CHECK-NEXT: vlse16.v v8, (a0), a2
58+
; CHECK-NEXT: vse16.v v8, (a1)
9159
; CHECK-NEXT: ret
9260
%1 = getelementptr inbounds i8, ptr %s, i64 16
9361
%2 = getelementptr inbounds i8, ptr %s, i64 32
@@ -109,21 +77,10 @@ define void @constant_forward_stride3(ptr %s, ptr %d) {
10977
define void @constant_back_stride(ptr %s, ptr %d) {
11078
; CHECK-LABEL: constant_back_stride:
11179
; CHECK: # %bb.0:
112-
; CHECK-NEXT: addi a2, a0, -16
113-
; CHECK-NEXT: addi a3, a0, -32
114-
; CHECK-NEXT: addi a4, a0, -48
115-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
116-
; CHECK-NEXT: vle8.v v8, (a0)
117-
; CHECK-NEXT: vle8.v v9, (a2)
118-
; CHECK-NEXT: vle8.v v10, (a3)
119-
; CHECK-NEXT: vle8.v v11, (a4)
120-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
121-
; CHECK-NEXT: vslideup.vi v8, v9, 2
122-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
123-
; CHECK-NEXT: vslideup.vi v8, v10, 4
124-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
125-
; CHECK-NEXT: vslideup.vi v8, v11, 6
126-
; CHECK-NEXT: vse8.v v8, (a1)
80+
; CHECK-NEXT: li a2, -16
81+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
82+
; CHECK-NEXT: vlse16.v v8, (a0), a2
83+
; CHECK-NEXT: vse16.v v8, (a1)
12784
; CHECK-NEXT: ret
12885
%1 = getelementptr inbounds i8, ptr %s, i64 -16
12986
%2 = getelementptr inbounds i8, ptr %s, i64 -32
@@ -142,21 +99,11 @@ define void @constant_back_stride(ptr %s, ptr %d) {
14299
define void @constant_back_stride2(ptr %s, ptr %d) {
143100
; CHECK-LABEL: constant_back_stride2:
144101
; CHECK: # %bb.0:
145-
; CHECK-NEXT: addi a2, a0, 16
146-
; CHECK-NEXT: addi a3, a0, 32
147-
; CHECK-NEXT: addi a4, a0, 48
148-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
149-
; CHECK-NEXT: vle8.v v8, (a4)
150-
; CHECK-NEXT: vle8.v v9, (a3)
151-
; CHECK-NEXT: vle8.v v10, (a2)
152-
; CHECK-NEXT: vle8.v v11, (a0)
153-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
154-
; CHECK-NEXT: vslideup.vi v8, v9, 2
155-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
156-
; CHECK-NEXT: vslideup.vi v8, v10, 4
157-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
158-
; CHECK-NEXT: vslideup.vi v8, v11, 6
159-
; CHECK-NEXT: vse8.v v8, (a1)
102+
; CHECK-NEXT: addi a0, a0, 48
103+
; CHECK-NEXT: li a2, -16
104+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
105+
; CHECK-NEXT: vlse16.v v8, (a0), a2
106+
; CHECK-NEXT: vse16.v v8, (a1)
160107
; CHECK-NEXT: ret
161108
%1 = getelementptr inbounds i8, ptr %s, i64 16
162109
%2 = getelementptr inbounds i8, ptr %s, i64 32
@@ -175,21 +122,10 @@ define void @constant_back_stride2(ptr %s, ptr %d) {
175122
define void @constant_back_stride3(ptr %s, ptr %d) {
176123
; CHECK-LABEL: constant_back_stride3:
177124
; CHECK: # %bb.0:
178-
; CHECK-NEXT: addi a2, a0, -16
179-
; CHECK-NEXT: addi a3, a0, -32
180-
; CHECK-NEXT: addi a4, a0, -48
181-
; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
182-
; CHECK-NEXT: vle8.v v8, (a0)
183-
; CHECK-NEXT: vle8.v v9, (a2)
184-
; CHECK-NEXT: vle8.v v10, (a3)
185-
; CHECK-NEXT: vle8.v v11, (a4)
186-
; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma
187-
; CHECK-NEXT: vslideup.vi v8, v9, 2
188-
; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma
189-
; CHECK-NEXT: vslideup.vi v8, v10, 4
190-
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
191-
; CHECK-NEXT: vslideup.vi v8, v11, 6
192-
; CHECK-NEXT: vse8.v v8, (a1)
125+
; CHECK-NEXT: li a2, -16
126+
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
127+
; CHECK-NEXT: vlse16.v v8, (a0), a2
128+
; CHECK-NEXT: vse16.v v8, (a1)
193129
; CHECK-NEXT: ret
194130
%1 = getelementptr inbounds i8, ptr %s, i64 -16
195131
%2 = getelementptr inbounds i8, ptr %s, i64 -32

0 commit comments

Comments
 (0)