Skip to content

Commit 2967e5f

Browse files
asbasb
authored
[RISCV] Enable store clustering by default (#73796)
Builds on #73789, enabling store clustering by default using the same heuristic.
1 parent cc8edbc commit 2967e5f

File tree

250 files changed

+9579
-9592
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

250 files changed

+9579
-9592
lines changed

‎llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@ static cl::opt<bool>
9494
cl::desc("Enable the loop data prefetch pass"),
9595
cl::init(true));
9696

97-
static cl::opt<bool> EnableMISchedLoadClustering(
98-
"riscv-misched-load-clustering", cl::Hidden,
99-
cl::desc("Enable load clustering in the machine scheduler"),
97+
static cl::opt<bool> EnableMISchedLoadStoreClustering(
98+
"riscv-misched-load-store-clustering", cl::Hidden,
99+
cl::desc("Enable load and store clustering in the machine scheduler"),
100100
cl::init(true));
101101

102102
static cl::opt<bool> EnableVSETVLIAfterRVVRegAlloc(
@@ -352,10 +352,12 @@ class RISCVPassConfig : public TargetPassConfig {
352352
ScheduleDAGInstrs *
353353
createMachineScheduler(MachineSchedContext *C) const override {
354354
ScheduleDAGMILive *DAG = nullptr;
355-
if (EnableMISchedLoadClustering) {
355+
if (EnableMISchedLoadStoreClustering) {
356356
DAG = createGenericSchedLive(C);
357357
DAG->addMutation(createLoadClusterDAGMutation(
358358
DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
359+
DAG->addMutation(createStoreClusterDAGMutation(
360+
DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
359361
}
360362
return DAG;
361363
}

‎llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,12 @@ define i32 @va1(ptr %fmt, ...) {
6969
; RV64-NEXT: sd a2, 32(sp)
7070
; RV64-NEXT: sd a3, 40(sp)
7171
; RV64-NEXT: sd a4, 48(sp)
72-
; RV64-NEXT: sd a5, 56(sp)
7372
; RV64-NEXT: addi a0, sp, 8
7473
; RV64-NEXT: addi a1, sp, 24
7574
; RV64-NEXT: sd a1, 8(sp)
7675
; RV64-NEXT: lw a0, 4(a0)
7776
; RV64-NEXT: lwu a1, 8(sp)
77+
; RV64-NEXT: sd a5, 56(sp)
7878
; RV64-NEXT: sd a6, 64(sp)
7979
; RV64-NEXT: sd a7, 72(sp)
8080
; RV64-NEXT: slli a0, a0, 32
@@ -129,12 +129,12 @@ define i32 @va1(ptr %fmt, ...) {
129129
; RV64-WITHFP-NEXT: sd a2, 16(s0)
130130
; RV64-WITHFP-NEXT: sd a3, 24(s0)
131131
; RV64-WITHFP-NEXT: sd a4, 32(s0)
132-
; RV64-WITHFP-NEXT: sd a5, 40(s0)
133132
; RV64-WITHFP-NEXT: addi a0, s0, -24
134133
; RV64-WITHFP-NEXT: addi a1, s0, 8
135134
; RV64-WITHFP-NEXT: sd a1, -24(s0)
136135
; RV64-WITHFP-NEXT: lw a0, 4(a0)
137136
; RV64-WITHFP-NEXT: lwu a1, -24(s0)
137+
; RV64-WITHFP-NEXT: sd a5, 40(s0)
138138
; RV64-WITHFP-NEXT: sd a6, 48(s0)
139139
; RV64-WITHFP-NEXT: sd a7, 56(s0)
140140
; RV64-WITHFP-NEXT: slli a0, a0, 32
@@ -844,11 +844,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
844844
; ILP32-LABEL: va3:
845845
; ILP32: # %bb.0:
846846
; ILP32-NEXT: addi sp, sp, -32
847-
; ILP32-NEXT: sw a3, 12(sp)
848-
; ILP32-NEXT: sw a4, 16(sp)
849847
; ILP32-NEXT: addi a0, sp, 12
850848
; ILP32-NEXT: sw a0, 4(sp)
851849
; ILP32-NEXT: lw a0, 4(sp)
850+
; ILP32-NEXT: sw a3, 12(sp)
851+
; ILP32-NEXT: sw a4, 16(sp)
852852
; ILP32-NEXT: sw a5, 20(sp)
853853
; ILP32-NEXT: sw a6, 24(sp)
854854
; ILP32-NEXT: sw a7, 28(sp)
@@ -868,11 +868,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
868868
; RV32D-ILP32-LABEL: va3:
869869
; RV32D-ILP32: # %bb.0:
870870
; RV32D-ILP32-NEXT: addi sp, sp, -48
871-
; RV32D-ILP32-NEXT: sw a3, 28(sp)
872-
; RV32D-ILP32-NEXT: sw a4, 32(sp)
873871
; RV32D-ILP32-NEXT: addi a0, sp, 28
874872
; RV32D-ILP32-NEXT: sw a0, 20(sp)
875873
; RV32D-ILP32-NEXT: lw a0, 20(sp)
874+
; RV32D-ILP32-NEXT: sw a3, 28(sp)
875+
; RV32D-ILP32-NEXT: sw a4, 32(sp)
876876
; RV32D-ILP32-NEXT: sw a5, 36(sp)
877877
; RV32D-ILP32-NEXT: sw a6, 40(sp)
878878
; RV32D-ILP32-NEXT: sw a7, 44(sp)
@@ -894,11 +894,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
894894
; RV32D-ILP32F-LABEL: va3:
895895
; RV32D-ILP32F: # %bb.0:
896896
; RV32D-ILP32F-NEXT: addi sp, sp, -48
897-
; RV32D-ILP32F-NEXT: sw a3, 28(sp)
898-
; RV32D-ILP32F-NEXT: sw a4, 32(sp)
899897
; RV32D-ILP32F-NEXT: addi a0, sp, 28
900898
; RV32D-ILP32F-NEXT: sw a0, 20(sp)
901899
; RV32D-ILP32F-NEXT: lw a0, 20(sp)
900+
; RV32D-ILP32F-NEXT: sw a3, 28(sp)
901+
; RV32D-ILP32F-NEXT: sw a4, 32(sp)
902902
; RV32D-ILP32F-NEXT: sw a5, 36(sp)
903903
; RV32D-ILP32F-NEXT: sw a6, 40(sp)
904904
; RV32D-ILP32F-NEXT: sw a7, 44(sp)
@@ -920,11 +920,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
920920
; RV32D-ILP32D-LABEL: va3:
921921
; RV32D-ILP32D: # %bb.0:
922922
; RV32D-ILP32D-NEXT: addi sp, sp, -48
923-
; RV32D-ILP32D-NEXT: sw a3, 28(sp)
924-
; RV32D-ILP32D-NEXT: sw a4, 32(sp)
925923
; RV32D-ILP32D-NEXT: addi a0, sp, 28
926924
; RV32D-ILP32D-NEXT: sw a0, 20(sp)
927925
; RV32D-ILP32D-NEXT: lw a0, 20(sp)
926+
; RV32D-ILP32D-NEXT: sw a3, 28(sp)
927+
; RV32D-ILP32D-NEXT: sw a4, 32(sp)
928928
; RV32D-ILP32D-NEXT: sw a5, 36(sp)
929929
; RV32D-ILP32D-NEXT: sw a6, 40(sp)
930930
; RV32D-ILP32D-NEXT: sw a7, 44(sp)
@@ -946,12 +946,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
946946
; RV64-LABEL: va3:
947947
; RV64: # %bb.0:
948948
; RV64-NEXT: addi sp, sp, -64
949-
; RV64-NEXT: sd a2, 16(sp)
950-
; RV64-NEXT: sd a3, 24(sp)
951-
; RV64-NEXT: sd a4, 32(sp)
952949
; RV64-NEXT: addi a0, sp, 16
953950
; RV64-NEXT: sd a0, 8(sp)
954951
; RV64-NEXT: ld a0, 8(sp)
952+
; RV64-NEXT: sd a2, 16(sp)
953+
; RV64-NEXT: sd a3, 24(sp)
954+
; RV64-NEXT: sd a4, 32(sp)
955955
; RV64-NEXT: sd a5, 40(sp)
956956
; RV64-NEXT: sd a6, 48(sp)
957957
; RV64-NEXT: sd a7, 56(sp)
@@ -970,11 +970,11 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
970970
; RV32-WITHFP-NEXT: sw ra, 20(sp) # 4-byte Folded Spill
971971
; RV32-WITHFP-NEXT: sw s0, 16(sp) # 4-byte Folded Spill
972972
; RV32-WITHFP-NEXT: addi s0, sp, 24
973-
; RV32-WITHFP-NEXT: sw a3, 4(s0)
974-
; RV32-WITHFP-NEXT: sw a4, 8(s0)
975973
; RV32-WITHFP-NEXT: addi a0, s0, 4
976974
; RV32-WITHFP-NEXT: sw a0, -12(s0)
977975
; RV32-WITHFP-NEXT: lw a0, -12(s0)
976+
; RV32-WITHFP-NEXT: sw a3, 4(s0)
977+
; RV32-WITHFP-NEXT: sw a4, 8(s0)
978978
; RV32-WITHFP-NEXT: sw a5, 12(s0)
979979
; RV32-WITHFP-NEXT: sw a6, 16(s0)
980980
; RV32-WITHFP-NEXT: sw a7, 20(s0)
@@ -999,12 +999,12 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
999999
; RV64-WITHFP-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
10001000
; RV64-WITHFP-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
10011001
; RV64-WITHFP-NEXT: addi s0, sp, 32
1002-
; RV64-WITHFP-NEXT: sd a2, 0(s0)
1003-
; RV64-WITHFP-NEXT: sd a3, 8(s0)
1004-
; RV64-WITHFP-NEXT: sd a4, 16(s0)
10051002
; RV64-WITHFP-NEXT: mv a0, s0
10061003
; RV64-WITHFP-NEXT: sd a0, -24(s0)
10071004
; RV64-WITHFP-NEXT: ld a0, -24(s0)
1005+
; RV64-WITHFP-NEXT: sd a2, 0(s0)
1006+
; RV64-WITHFP-NEXT: sd a3, 8(s0)
1007+
; RV64-WITHFP-NEXT: sd a4, 16(s0)
10081008
; RV64-WITHFP-NEXT: sd a5, 24(s0)
10091009
; RV64-WITHFP-NEXT: sd a6, 32(s0)
10101010
; RV64-WITHFP-NEXT: sd a7, 40(s0)
@@ -1622,9 +1622,6 @@ define i32 @va_large_stack(ptr %fmt, ...) {
16221622
; RV64-NEXT: lui a0, 24414
16231623
; RV64-NEXT: add a0, sp, a0
16241624
; RV64-NEXT: sd a4, 304(a0)
1625-
; RV64-NEXT: lui a0, 24414
1626-
; RV64-NEXT: add a0, sp, a0
1627-
; RV64-NEXT: sd a5, 312(a0)
16281625
; RV64-NEXT: addi a0, sp, 8
16291626
; RV64-NEXT: lui a1, 24414
16301627
; RV64-NEXT: addiw a1, a1, 280
@@ -1634,6 +1631,9 @@ define i32 @va_large_stack(ptr %fmt, ...) {
16341631
; RV64-NEXT: lwu a1, 8(sp)
16351632
; RV64-NEXT: lui a2, 24414
16361633
; RV64-NEXT: add a2, sp, a2
1634+
; RV64-NEXT: sd a5, 312(a2)
1635+
; RV64-NEXT: lui a2, 24414
1636+
; RV64-NEXT: add a2, sp, a2
16371637
; RV64-NEXT: sd a6, 320(a2)
16381638
; RV64-NEXT: lui a2, 24414
16391639
; RV64-NEXT: add a2, sp, a2

‎llvm/test/CodeGen/RISCV/abds-neg.ll

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -705,8 +705,8 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
705705
; RV32I-NEXT: sub a4, a4, a3
706706
; RV32I-NEXT: neg a1, a1
707707
; RV32I-NEXT: sw a1, 0(a0)
708-
; RV32I-NEXT: sw a4, 8(a0)
709708
; RV32I-NEXT: sw a2, 4(a0)
709+
; RV32I-NEXT: sw a4, 8(a0)
710710
; RV32I-NEXT: sw a5, 12(a0)
711711
; RV32I-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
712712
; RV32I-NEXT: addi sp, sp, 16
@@ -824,8 +824,8 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
824824
; RV32ZBB-NEXT: sub a4, a4, a3
825825
; RV32ZBB-NEXT: neg a1, a1
826826
; RV32ZBB-NEXT: sw a1, 0(a0)
827-
; RV32ZBB-NEXT: sw a4, 8(a0)
828827
; RV32ZBB-NEXT: sw a2, 4(a0)
828+
; RV32ZBB-NEXT: sw a4, 8(a0)
829829
; RV32ZBB-NEXT: sw a5, 12(a0)
830830
; RV32ZBB-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
831831
; RV32ZBB-NEXT: addi sp, sp, 16
@@ -952,8 +952,8 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
952952
; RV32I-NEXT: sub a4, a4, a3
953953
; RV32I-NEXT: neg a1, a1
954954
; RV32I-NEXT: sw a1, 0(a0)
955-
; RV32I-NEXT: sw a4, 8(a0)
956955
; RV32I-NEXT: sw a2, 4(a0)
956+
; RV32I-NEXT: sw a4, 8(a0)
957957
; RV32I-NEXT: sw a5, 12(a0)
958958
; RV32I-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
959959
; RV32I-NEXT: addi sp, sp, 16
@@ -1071,8 +1071,8 @@ define i128 @abd_ext_i128_undef(i128 %a, i128 %b) nounwind {
10711071
; RV32ZBB-NEXT: sub a4, a4, a3
10721072
; RV32ZBB-NEXT: neg a1, a1
10731073
; RV32ZBB-NEXT: sw a1, 0(a0)
1074-
; RV32ZBB-NEXT: sw a4, 8(a0)
10751074
; RV32ZBB-NEXT: sw a2, 4(a0)
1075+
; RV32ZBB-NEXT: sw a4, 8(a0)
10761076
; RV32ZBB-NEXT: sw a5, 12(a0)
10771077
; RV32ZBB-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
10781078
; RV32ZBB-NEXT: addi sp, sp, 16
@@ -1918,9 +1918,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
19181918
; RV32I-NEXT: sub a1, a1, t2
19191919
; RV32I-NEXT: sub a2, a2, a3
19201920
; RV32I-NEXT: .LBB22_11:
1921-
; RV32I-NEXT: sw a6, 8(a0)
1922-
; RV32I-NEXT: sw a1, 4(a0)
19231921
; RV32I-NEXT: sw a2, 0(a0)
1922+
; RV32I-NEXT: sw a1, 4(a0)
1923+
; RV32I-NEXT: sw a6, 8(a0)
19241924
; RV32I-NEXT: sw a5, 12(a0)
19251925
; RV32I-NEXT: ret
19261926
;
@@ -2005,9 +2005,9 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
20052005
; RV32ZBB-NEXT: sub a1, a1, t2
20062006
; RV32ZBB-NEXT: sub a2, a2, a3
20072007
; RV32ZBB-NEXT: .LBB22_11:
2008-
; RV32ZBB-NEXT: sw a6, 8(a0)
2009-
; RV32ZBB-NEXT: sw a1, 4(a0)
20102008
; RV32ZBB-NEXT: sw a2, 0(a0)
2009+
; RV32ZBB-NEXT: sw a1, 4(a0)
2010+
; RV32ZBB-NEXT: sw a6, 8(a0)
20112011
; RV32ZBB-NEXT: sw a5, 12(a0)
20122012
; RV32ZBB-NEXT: ret
20132013
;

0 commit comments

Comments
 (0)