Skip to content

Commit e279053

Browse files
shiltianshiltian
committed
[GlobalISel] Handle div-by-pow2
This patch adds similar handling of div-by-pow2 as in `SelectionDAG`.
1 parent 6101cf3 commit e279053

File tree

4 files changed

+130
-7
lines changed

4 files changed

+130
-7
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,16 @@ class CombinerHelper {
673673
bool matchSDivByConst(MachineInstr &MI);
674674
void applySDivByConst(MachineInstr &MI);
675675

676+
/// Given an G_SDIV \p MI expressing a signed divided by a pow2 constant,
677+
/// return expressions that implements it by shifting.
678+
bool matchSDivByPow2(MachineInstr &MI);
679+
void applySDivByPow2(MachineInstr &MI);
680+
681+
/// Given an G_UDIV \p MI expressing an unsigned divided by a pow2 constant,
682+
/// return expressions that implements it by shifting.
683+
bool matchUDivByPow2(MachineInstr &MI);
684+
void applyUDivByPow2(MachineInstr &MI);
685+
676686
// G_UMULH x, (1 << c)) -> x >> (bitwidth - c)
677687
bool matchUMulHToLShr(MachineInstr &MI);
678688
void applyUMulHToLShr(MachineInstr &MI);

‎llvm/include/llvm/Target/GlobalISel/Combine.td

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ def combine_extracted_vector_load : GICombineRule<
264264
(match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
265265
[{ return Helper.matchCombineExtractedVectorLoad(*${root}, ${matchinfo}); }]),
266266
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
267-
267+
268268
def combine_indexed_load_store : GICombineRule<
269269
(defs root:$root, indexed_load_store_matchdata:$matchinfo),
270270
(match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD, G_STORE):$root,
@@ -1005,7 +1005,20 @@ def sdiv_by_const : GICombineRule<
10051005
[{ return Helper.matchSDivByConst(*${root}); }]),
10061006
(apply [{ Helper.applySDivByConst(*${root}); }])>;
10071007

1008-
def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const]>;
1008+
def sdiv_by_pow2 : GICombineRule<
1009+
(defs root:$root),
1010+
(match (wip_match_opcode G_SDIV):$root,
1011+
[{ return Helper.matchSDivByPow2(*${root}); }]),
1012+
(apply [{ Helper.applySDivByPow2(*${root}); }])>;
1013+
1014+
def udiv_by_pow2 : GICombineRule<
1015+
(defs root:$root),
1016+
(match (wip_match_opcode G_UDIV):$root,
1017+
[{ return Helper.matchUDivByPow2(*${root}); }]),
1018+
(apply [{ Helper.applyUDivByPow2(*${root}); }])>;
1019+
1020+
def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const,
1021+
sdiv_by_pow2, udiv_by_pow2]>;
10091022

10101023
def reassoc_ptradd : GICombineRule<
10111024
(defs root:$root, build_fn_matchinfo:$matchinfo),
@@ -1325,7 +1338,7 @@ def constant_fold_binops : GICombineGroup<[constant_fold_binop,
13251338

13261339
def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines,
13271340
extract_vec_elt_combines, combines_for_extload, combine_extracted_vector_load,
1328-
undef_combines, identity_combines, phi_combines,
1341+
undef_combines, identity_combines, phi_combines,
13291342
simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big,
13301343
reassocs, ptr_add_immed_chain,
13311344
shl_ashr_to_sext_inreg, sext_inreg_of_load,
@@ -1342,7 +1355,7 @@ def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines,
13421355
intdiv_combines, mulh_combines, redundant_neg_operands,
13431356
and_or_disjoint_mask, fma_combines, fold_binop_into_select,
13441357
sub_add_reg, select_to_minmax, redundant_binop_in_equality,
1345-
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
1358+
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
13461359
combine_concat_vector]>;
13471360

13481361
// A combine group used to for prelegalizer combiners at -O0. The combines in

‎llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1490,7 +1490,7 @@ void CombinerHelper::applyOptBrCondByInvertingCond(MachineInstr &MI,
14901490
Observer.changedInstr(*BrCond);
14911491
}
14921492

1493-
1493+
14941494
bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) {
14951495
MachineIRBuilder HelperBuilder(MI);
14961496
GISelObserverWrapper DummyObserver;
@@ -5286,6 +5286,106 @@ MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) {
52865286
return MIB.buildMul(Ty, Res, Factor);
52875287
}
52885288

5289+
bool CombinerHelper::matchSDivByPow2(MachineInstr &MI) {
5290+
assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5291+
if (MI.getFlag(MachineInstr::MIFlag::IsExact))
5292+
return false;
5293+
auto &SDiv = cast<GenericMachineInstr>(MI);
5294+
Register RHS = SDiv.getReg(2);
5295+
auto MatchPow2 = [&](const Constant *C) {
5296+
if (auto *CI = dyn_cast<ConstantInt>(C))
5297+
return CI->getValue().isPowerOf2() || CI->getValue().isNegatedPowerOf2();
5298+
return false;
5299+
};
5300+
return matchUnaryPredicate(MRI, RHS, MatchPow2, /* AllowUndefs */ false);
5301+
}
5302+
5303+
void CombinerHelper::applySDivByPow2(MachineInstr &MI) {
5304+
assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5305+
auto &SDiv = cast<GenericMachineInstr>(MI);
5306+
Register Dst = SDiv.getReg(0);
5307+
Register LHS = SDiv.getReg(1);
5308+
Register RHS = SDiv.getReg(2);
5309+
LLT Ty = MRI.getType(Dst);
5310+
LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty);
5311+
5312+
Builder.setInstrAndDebugLoc(MI);
5313+
5314+
auto RHSC = getIConstantVRegValWithLookThrough(RHS, MRI);
5315+
assert(RHSC.has_value() && "RHS must be a constant");
5316+
auto RHSCV = RHSC->Value;
5317+
auto Zero = Builder.buildConstant(Ty, 0);
5318+
5319+
// Special case: (sdiv X, 1) -> X
5320+
if (RHSCV.isOne()) {
5321+
replaceSingleDefInstWithReg(MI, LHS);
5322+
return;
5323+
}
5324+
// Special Case: (sdiv X, -1) -> 0-X
5325+
if (RHSCV.isAllOnes()) {
5326+
auto Sub = Builder.buildSub(Ty, Zero, LHS);
5327+
replaceSingleDefInstWithReg(MI, Sub->getOperand(0).getReg());
5328+
return;
5329+
}
5330+
5331+
unsigned Bitwidth = Ty.getScalarSizeInBits();
5332+
unsigned TrailingZeros = RHSCV.countTrailingZeros();
5333+
auto C1 = Builder.buildConstant(ShiftAmtTy, TrailingZeros);
5334+
auto Inexact = Builder.buildConstant(ShiftAmtTy, Bitwidth - TrailingZeros);
5335+
auto Sign = Builder.buildAShr(
5336+
Ty, LHS, Builder.buildConstant(ShiftAmtTy, Bitwidth - 1));
5337+
// Add (LHS < 0) ? abs2 - 1 : 0;
5338+
auto Srl = Builder.buildShl(Ty, Sign, Inexact);
5339+
auto Add = Builder.buildAdd(Ty, LHS, Srl);
5340+
auto Sra = Builder.buildAShr(Ty, Add, C1);
5341+
5342+
// If dividing by a positive value, we're done. Otherwise, the result must
5343+
// be negated.
5344+
auto Res = RHSCV.isNegative() ? Builder.buildSub(Ty, Zero, Sra) : Sra;
5345+
replaceSingleDefInstWithReg(MI, Res->getOperand(0).getReg());
5346+
}
5347+
5348+
bool CombinerHelper::matchUDivByPow2(MachineInstr &MI) {
5349+
assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
5350+
if (MI.getFlag(MachineInstr::MIFlag::IsExact))
5351+
return false;
5352+
auto &UDiv = cast<GenericMachineInstr>(MI);
5353+
Register RHS = UDiv.getReg(2);
5354+
auto MatchPow2 = [&](const Constant *C) {
5355+
if (auto *CI = dyn_cast<ConstantInt>(C))
5356+
return CI->getValue().isPowerOf2();
5357+
return false;
5358+
};
5359+
return matchUnaryPredicate(MRI, RHS, MatchPow2, /* AllowUndefs */ false);
5360+
}
5361+
5362+
void CombinerHelper::applyUDivByPow2(MachineInstr &MI) {
5363+
assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected SDIV");
5364+
auto &UDiv = cast<GenericMachineInstr>(MI);
5365+
Register Dst = UDiv.getReg(0);
5366+
Register LHS = UDiv.getReg(1);
5367+
Register RHS = UDiv.getReg(2);
5368+
LLT Ty = MRI.getType(Dst);
5369+
LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(Ty);
5370+
5371+
Builder.setInstrAndDebugLoc(MI);
5372+
5373+
auto RHSC = getIConstantVRegValWithLookThrough(RHS, MRI);
5374+
assert(RHSC.has_value() && "RHS must be a constant");
5375+
auto RHSCV = RHSC->Value;
5376+
5377+
// Special case: (udiv X, 1) -> X
5378+
if (RHSCV.isOne()) {
5379+
replaceSingleDefInstWithReg(MI, LHS);
5380+
return;
5381+
}
5382+
5383+
unsigned TrailingZeros = RHSCV.countTrailingZeros();
5384+
auto C1 = Builder.buildConstant(ShiftAmtTy, TrailingZeros);
5385+
auto Res = Builder.buildLShr(Ty, LHS, C1);
5386+
replaceSingleDefInstWithReg(MI, Res->getOperand(0).getReg());
5387+
}
5388+
52895389
bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) {
52905390
assert(MI.getOpcode() == TargetOpcode::G_UMULH);
52915391
Register RHS = MI.getOperand(2).getReg();

‎llvm/test/CodeGen/AMDGPU/div_i128.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
; RUN: llc -O0 -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-SDAG-O0 %s
44

55
; FIXME: GlobalISel missing the power-of-2 cases in legalization. https://github.com/llvm/llvm-project/issues/80671
6-
; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9 %s
7-
; xUN: llc -O0 -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-O0 %s
6+
; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9,GFX9 %s
7+
; RUN: llc -O0 -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - %s | FileCheck -check-prefixes=GFX9-O0,GFX9-O0 %s
88

99
define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
1010
; GFX9-LABEL: v_sdiv_i128_vv:

0 commit comments

Comments
 (0)