Skip to content

Commit 38ab6d8

Browse files
ParkHanbumParkHanbum
committed
[AArch64] Optimize when storing symmetry constants
This change looks for instructions of storing symmetric constants instruction 32-bit units. usually consisting of several 'MOV' and one or less 'ORR'. If found, load only the lower 32-bit constant and change it to copy and save to the upper 32-bit using the 'STP' instruction. For example: renamable $x8 = MOVZXi 49370, 0 renamable $x8 = MOVKXi $x8, 320, 16 renamable $x8 = ORRXrs $x8, $x8, 32 STRXui killed renamable $x8, killed renamable $x0, 0 becomes $w8 = MOVZWi 49370, 0 $w8 = MOVKWi $w8, 320, 16 STPWi killed renamable $w8, killed renamable $w8, killed renamable $x0, 0
1 parent ed551e3 commit 38ab6d8

File tree

3 files changed

+207
-25
lines changed

3 files changed

+207
-25
lines changed

‎llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,14 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
201201
// Find and merge a base register updates before or after a ld/st instruction.
202202
bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
203203

204+
// Finds and collapses loads of symmetric constant value.
205+
bool tryFoldSymmetryConstantLoad(MachineBasicBlock::iterator &I,
206+
unsigned Limit);
207+
MachineBasicBlock::iterator
208+
doFoldSymmetryConstantLoad(MachineInstr &MI,
209+
SmallVectorImpl<MachineBasicBlock::iterator> &MIs,
210+
int SuccIndex, bool hasORR, int Accumulated);
211+
204212
bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
205213

206214
bool runOnMachineFunction(MachineFunction &Fn) override;
@@ -2252,6 +2260,167 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
22522260
return E;
22532261
}
22542262

2263+
static bool isSymmetric(MachineInstr &MI, Register BaseReg) {
2264+
auto MatchBaseReg = [&](unsigned Count) {
2265+
for (unsigned I = 0; I < Count; I++) {
2266+
auto OpI = MI.getOperand(I);
2267+
if (OpI.isReg() && OpI.getReg() != BaseReg)
2268+
return false;
2269+
}
2270+
return true;
2271+
};
2272+
2273+
unsigned Opc = MI.getOpcode();
2274+
switch (Opc) {
2275+
default:
2276+
return false;
2277+
case AArch64::MOVZXi:
2278+
return MatchBaseReg(1);
2279+
case AArch64::MOVKXi:
2280+
return MatchBaseReg(2);
2281+
case AArch64::ORRXrs:
2282+
MachineOperand &Imm = MI.getOperand(3);
2283+
// Fourth operand of ORR must be 32 which mean
2284+
// 32bit symmetric constant load.
2285+
// ex) renamable $x8 = ORRXrs $x8, $x8, 32
2286+
if (MatchBaseReg(3) && Imm.isImm() && Imm.getImm() == 32)
2287+
return true;
2288+
}
2289+
2290+
return false;
2291+
}
2292+
2293+
MachineBasicBlock::iterator AArch64LoadStoreOpt::doFoldSymmetryConstantLoad(
2294+
MachineInstr &MI, SmallVectorImpl<MachineBasicBlock::iterator> &MIs,
2295+
int SuccIndex, bool hasORR, int Accumulated) {
2296+
MachineBasicBlock::iterator I = MI.getIterator();
2297+
MachineBasicBlock::iterator E = I->getParent()->end();
2298+
MachineBasicBlock::iterator NextI = next_nodbg(I, E);
2299+
MachineBasicBlock::iterator FirstMovI;
2300+
MachineBasicBlock *MBB = MI.getParent();
2301+
uint64_t Mask = 0xFFFFUL;
2302+
Register DstRegW;
2303+
2304+
if (hasORR) {
2305+
(*MIs.begin())->eraseFromParent();
2306+
} else {
2307+
int Index = 0;
2308+
for (auto MI = MIs.begin(), E = MIs.end(); MI != E; ++MI, Index++) {
2309+
if (Index == SuccIndex - 1) {
2310+
FirstMovI = *MI;
2311+
break;
2312+
}
2313+
(*MI)->eraseFromParent();
2314+
}
2315+
DstRegW =
2316+
TRI->getSubReg(FirstMovI->getOperand(0).getReg(), AArch64::sub_32);
2317+
2318+
int Lower = Accumulated & Mask;
2319+
if (Lower) {
2320+
BuildMI(*MBB, FirstMovI, FirstMovI->getDebugLoc(),
2321+
TII->get(AArch64::MOVZWi), DstRegW)
2322+
.addImm(Lower)
2323+
.addImm(0);
2324+
Lower = (Accumulated >> 16) & Mask;
2325+
if (Lower) {
2326+
BuildMI(*MBB, FirstMovI, FirstMovI->getDebugLoc(),
2327+
TII->get(AArch64::MOVKWi), DstRegW)
2328+
.addUse(DstRegW)
2329+
.addImm(Lower)
2330+
.addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16));
2331+
}
2332+
} else {
2333+
Lower = Accumulated >> 16 & Mask;
2334+
BuildMI(*MBB, FirstMovI, FirstMovI->getDebugLoc(),
2335+
TII->get(AArch64::MOVZWi), DstRegW)
2336+
.addImm(Lower)
2337+
.addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16));
2338+
}
2339+
FirstMovI->eraseFromParent();
2340+
}
2341+
2342+
Register BaseReg = getLdStRegOp(MI).getReg();
2343+
const MachineOperand MO = AArch64InstrInfo::getLdStBaseOp(MI);
2344+
DstRegW = TRI->getSubReg(BaseReg, AArch64::sub_32);
2345+
unsigned DstRegState = getRegState(MI.getOperand(0));
2346+
BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AArch64::STPWi))
2347+
.addReg(DstRegW, DstRegState)
2348+
.addReg(DstRegW, DstRegState)
2349+
.addReg(MO.getReg(), getRegState(MO))
2350+
.add(AArch64InstrInfo::getLdStOffsetOp(MI))
2351+
.setMemRefs(MI.memoperands())
2352+
.setMIFlags(MI.getFlags());
2353+
I->eraseFromParent();
2354+
2355+
return NextI;
2356+
}
2357+
2358+
bool AArch64LoadStoreOpt::tryFoldSymmetryConstantLoad(
2359+
MachineBasicBlock::iterator &I, unsigned Limit) {
2360+
MachineInstr &MI = *I;
2361+
if (MI.getOpcode() != AArch64::STRXui)
2362+
return false;
2363+
2364+
MachineBasicBlock::iterator MBBI = I;
2365+
MachineBasicBlock::iterator B = I->getParent()->begin();
2366+
if (MBBI == B)
2367+
return false;
2368+
2369+
Register BaseReg = getLdStRegOp(MI).getReg();
2370+
unsigned Count = 0, SuccIndex = 0;
2371+
bool hasORR = false;
2372+
SmallVector<MachineBasicBlock::iterator> MIs;
2373+
ModifiedRegUnits.clear();
2374+
UsedRegUnits.clear();
2375+
2376+
uint64_t Accumulated = 0, Mask = 0xFFFFUL;
2377+
do {
2378+
MBBI = prev_nodbg(MBBI, B);
2379+
MachineInstr &MI = *MBBI;
2380+
if (!MI.isTransient())
2381+
++Count;
2382+
if (!isSymmetric(MI, BaseReg)) {
2383+
LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
2384+
TRI);
2385+
if (!ModifiedRegUnits.available(BaseReg) ||
2386+
!UsedRegUnits.available(BaseReg))
2387+
return false;
2388+
continue;
2389+
}
2390+
2391+
unsigned Opc = MI.getOpcode();
2392+
if (Opc == AArch64::ORRXrs) {
2393+
hasORR = true;
2394+
MIs.push_back(MBBI);
2395+
continue;
2396+
}
2397+
unsigned ValueOrder = Opc == AArch64::MOVZXi ? 1 : 2;
2398+
MachineOperand Value = MI.getOperand(ValueOrder);
2399+
MachineOperand Shift = MI.getOperand(ValueOrder + 1);
2400+
if (!Value.isImm() || !Shift.isImm())
2401+
return false;
2402+
2403+
uint64_t IValue = Value.getImm();
2404+
uint64_t IShift = Shift.getImm();
2405+
Accumulated -= (Accumulated & (Mask << IShift));
2406+
Accumulated += (IValue << IShift);
2407+
MIs.push_back(MBBI);
2408+
if (Accumulated != 0 &&
2409+
(((Accumulated >> 32) == (Accumulated & 0xffffffffULL)) ||
2410+
(hasORR && Accumulated >> 32 == 0))) {
2411+
SuccIndex = MIs.size();
2412+
break;
2413+
}
2414+
} while (MBBI != B && Count < Limit);
2415+
2416+
if (SuccIndex) {
2417+
I = doFoldSymmetryConstantLoad(MI, MIs, SuccIndex, hasORR, Accumulated);
2418+
return true;
2419+
}
2420+
2421+
return false;
2422+
}
2423+
22552424
bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
22562425
MachineBasicBlock::iterator &MBBI) {
22572426
MachineInstr &MI = *MBBI;
@@ -2518,6 +2687,26 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
25182687
++MBBI;
25192688
}
25202689

2690+
// We have an opportunity to optimize the `STRXui` instruction, which loads
2691+
// the same 32-bit value into a register twice. The `STPXi` instruction allows
2692+
// us to load a 32-bit value only once.
2693+
// Considering :
2694+
// renamable $x8 = MOVZXi 49370, 0
2695+
// renamable $x8 = MOVKXi $x8, 320, 16
2696+
// renamable $x8 = ORRXrs $x8, $x8, 32
2697+
// STRXui killed renamable $x8, killed renamable $x0, 0
2698+
// Transform :
2699+
// $w8 = MOVZWi 49370, 0
2700+
// $w8 = MOVKWi $w8, 320, 16
2701+
// STPWi killed renamable $w8, killed renamable $w8, killed renamable $x0, 0
2702+
for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
2703+
MBBI != E;) {
2704+
if (tryFoldSymmetryConstantLoad(MBBI, UpdateLimit))
2705+
Modified = true;
2706+
else
2707+
++MBBI;
2708+
}
2709+
25212710
return Modified;
25222711
}
25232712

‎llvm/test/CodeGen/AArch64/movimm-expand-ldst.ll

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,9 @@ define i64 @testuu0xf555f555f555f555() {
9797
define void @test_store_0x1234567812345678(ptr %x) {
9898
; CHECK-LABEL: test_store_0x1234567812345678:
9999
; CHECK: // %bb.0:
100-
; CHECK-NEXT: mov x8, #22136 // =0x5678
101-
; CHECK-NEXT: movk x8, #4660, lsl #16
102-
; CHECK-NEXT: orr x8, x8, x8, lsl #32
103-
; CHECK-NEXT: str x8, [x0]
100+
; CHECK-NEXT: mov w8, #22136 // =0x5678
101+
; CHECK-NEXT: movk w8, #4660, lsl #16
102+
; CHECK-NEXT: stp w8, w8, [x0]
104103
; CHECK-NEXT: ret
105104
store i64 u0x1234567812345678, ptr %x
106105
ret void
@@ -109,10 +108,9 @@ define void @test_store_0x1234567812345678(ptr %x) {
109108
define void @test_store_0xff3456ffff3456ff(ptr %x) {
110109
; CHECK-LABEL: test_store_0xff3456ffff3456ff:
111110
; CHECK: // %bb.0:
112-
; CHECK-NEXT: mov x8, #22271 // =0x56ff
113-
; CHECK-NEXT: movk x8, #65332, lsl #16
114-
; CHECK-NEXT: orr x8, x8, x8, lsl #32
115-
; CHECK-NEXT: str x8, [x0]
111+
; CHECK-NEXT: mov w8, #22271 // =0x56ff
112+
; CHECK-NEXT: movk w8, #65332, lsl #16
113+
; CHECK-NEXT: stp w8, w8, [x0]
116114
; CHECK-NEXT: ret
117115
store i64 u0xff3456ffff3456ff, ptr %x
118116
ret void
@@ -165,9 +163,8 @@ define void @test_store_0x0000555555555555(ptr %x) {
165163
define void @test_store_0x0000555500005555(ptr %x) {
166164
; CHECK-LABEL: test_store_0x0000555500005555:
167165
; CHECK: // %bb.0:
168-
; CHECK-NEXT: mov x8, #21845 // =0x5555
169-
; CHECK-NEXT: movk x8, #21845, lsl #32
170-
; CHECK-NEXT: str x8, [x0]
166+
; CHECK-NEXT: mov w8, #21845 // =0x5555
167+
; CHECK-NEXT: stp w8, w8, [x0]
171168
; CHECK-NEXT: ret
172169
store i64 u0x0000555500005555, ptr %x
173170
ret void
@@ -176,9 +173,8 @@ define void @test_store_0x0000555500005555(ptr %x) {
176173
define void @test_store_0x5555000055550000(ptr %x) {
177174
; CHECK-LABEL: test_store_0x5555000055550000:
178175
; CHECK: // %bb.0:
179-
; CHECK-NEXT: mov x8, #1431633920 // =0x55550000
180-
; CHECK-NEXT: movk x8, #21845, lsl #48
181-
; CHECK-NEXT: str x8, [x0]
176+
; CHECK-NEXT: mov w8, #1431633920 // =0x55550000
177+
; CHECK-NEXT: stp w8, w8, [x0]
182178
; CHECK-NEXT: ret
183179
store i64 u0x5555000055550000, ptr %x
184180
ret void

‎llvm/test/CodeGen/AArch64/movimm-expand-ldst.mir

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,9 @@ body: |
4242
; CHECK-LABEL: name: test_fold_repeating_constant_store
4343
; CHECK: liveins: $x0
4444
; CHECK-NEXT: {{ $}}
45-
; CHECK-NEXT: renamable $x8 = MOVZXi 49370, 0
46-
; CHECK-NEXT: renamable $x8 = MOVKXi $x8, 320, 16
47-
; CHECK-NEXT: renamable $x8 = ORRXrs $x8, $x8, 32
48-
; CHECK-NEXT: STRXui killed renamable $x8, killed renamable $x0, 0
45+
; CHECK-NEXT: $w8 = MOVZWi 49370, 0
46+
; CHECK-NEXT: $w8 = MOVKWi $w8, 320, 16
47+
; CHECK-NEXT: STPWi killed renamable $w8, killed renamable $w8, killed renamable $x0, 0
4948
; CHECK-NEXT: RET undef $lr
5049
renamable $x8 = MOVi64imm 90284035103834330
5150
STRXui killed renamable $x8, killed renamable $x0, 0
@@ -60,10 +59,9 @@ body: |
6059
; CHECK-LABEL: name: test_fold_repeating_constant_store_neg
6160
; CHECK: liveins: $x0
6261
; CHECK-NEXT: {{ $}}
63-
; CHECK-NEXT: renamable $x8 = MOVZXi 320, 0
64-
; CHECK-NEXT: renamable $x8 = MOVKXi $x8, 49370, 16
65-
; CHECK-NEXT: renamable $x8 = ORRXrs $x8, $x8, 32
66-
; CHECK-NEXT: STRXui killed renamable $x8, killed renamable $x0, 0
62+
; CHECK-NEXT: $w8 = MOVZWi 320, 0
63+
; CHECK-NEXT: $w8 = MOVKWi $w8, 49370, 16
64+
; CHECK-NEXT: STPWi killed renamable $w8, killed renamable $w8, killed renamable $x0, 0
6765
; CHECK-NEXT: RET undef $lr
6866
renamable $x8 = MOVi64imm -4550323095879417536
6967
STRXui killed renamable $x8, killed renamable $x0, 0
@@ -78,9 +76,8 @@ body: |
7876
; CHECK-LABEL: name: test_fold_repeating_constant_store_16bit_unit
7977
; CHECK: liveins: $x0
8078
; CHECK-NEXT: {{ $}}
81-
; CHECK-NEXT: renamable $x8 = MOVZXi 21845, 16
82-
; CHECK-NEXT: renamable $x8 = MOVKXi $x8, 21845, 48
83-
; CHECK-NEXT: STRXui killed renamable $x8, killed renamable $x0, 0
79+
; CHECK-NEXT: $w8 = MOVZWi 21845, 16
80+
; CHECK-NEXT: STPWi killed renamable $w8, killed renamable $w8, killed renamable $x0, 0
8481
; CHECK-NEXT: RET undef $lr
8582
renamable $x8 = MOVZXi 21845, 16
8683
renamable $x8 = MOVKXi $x8, 21845, 48

0 commit comments

Comments
 (0)