Skip to content

[AArch64] Check for negative numbers when adjusting icmps #141151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 30, 2025
Merged

[AArch64] Check for negative numbers when adjusting icmps #141151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6b9ea5c
[AArch64] Pre-commit test (NFC)
AZero13 May 20, 2025
3162b71
[AArch64] Check for immediates using isLegalICmpImmediate
AZero13 May 22, 2025
971a80c
[AArch64] Convert comparisons with 1 and -1 to 0 if it is profitable
AZero13 May 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 61 additions & 39 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3337,6 +3337,12 @@ static bool isLegalArithImmed(uint64_t C) {
return IsLegal;
}

bool isLegalCmpImmed(APInt C) {
// Works for negative immediates too, as it can be written as an ADDS
// instruction with a negated immediate.
return isLegalArithImmed(C.abs().getZExtValue());
}

static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
KnownBits KnownSrc = DAG.computeKnownBits(CheckedVal);
return !KnownSrc.getSignedMinValue().isMinSignedValue();
Expand Down Expand Up @@ -3762,58 +3768,82 @@ static unsigned getCmpOperandFoldingProfit(SDValue Op) {
return 0;
}

// emitComparison() converts comparison with one or negative one to comparison
// with 0. Note that this only works for signed comparisons because of how ANDS
// works.
static bool shouldBeAdjustedToZero(SDValue LHS, APInt C, ISD::CondCode &CC) {
// Only works for ANDS and AND.
if (LHS.getOpcode() != ISD::AND && LHS.getOpcode() != AArch64ISD::ANDS)
return false;

if (C.isOne() && (CC == ISD::SETLT || CC == ISD::SETGE)) {
CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
return true;
}

if (C.isAllOnes() && (CC == ISD::SETLE || CC == ISD::SETGT)) {
CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
return true;
}

return false;
}

static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
SDValue &AArch64cc, SelectionDAG &DAG,
const SDLoc &dl) {
if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
EVT VT = RHS.getValueType();
uint64_t C = RHSC->getZExtValue();
if (!isLegalArithImmed(C)) {
APInt C = RHSC->getAPIntValue();
// shouldBeAdjustedToZero is a special case to better fold with
// emitComparison().
if (shouldBeAdjustedToZero(LHS, C, CC)) {
// Adjust the constant to zero.
// CC has already been adjusted.
RHS = DAG.getConstant(0, dl, VT);
} else if (!isLegalCmpImmed(C)) {
// Constant does not fit, try adjusting it by one?
switch (CC) {
default:
break;
case ISD::SETLT:
case ISD::SETGE:
if ((VT == MVT::i32 && C != 0x80000000 &&
isLegalArithImmed((uint32_t)(C - 1))) ||
(VT == MVT::i64 && C != 0x80000000ULL &&
isLegalArithImmed(C - 1ULL))) {
CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
RHS = DAG.getConstant(C, dl, VT);
if (!C.isMinSignedValue()) {
APInt CMinusOne = C - 1;
if (isLegalCmpImmed(CMinusOne)) {
CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
RHS = DAG.getConstant(CMinusOne, dl, VT);
}
}
break;
case ISD::SETULT:
case ISD::SETUGE:
if ((VT == MVT::i32 && C != 0 &&
isLegalArithImmed((uint32_t)(C - 1))) ||
(VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
RHS = DAG.getConstant(C, dl, VT);
if (!C.isZero()) {
APInt CMinusOne = C - 1;
if (isLegalCmpImmed(CMinusOne)) {
CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
RHS = DAG.getConstant(CMinusOne, dl, VT);
}
}
break;
case ISD::SETLE:
case ISD::SETGT:
if ((VT == MVT::i32 && C != INT32_MAX &&
isLegalArithImmed((uint32_t)(C + 1))) ||
(VT == MVT::i64 && C != INT64_MAX &&
isLegalArithImmed(C + 1ULL))) {
CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
RHS = DAG.getConstant(C, dl, VT);
if (!C.isMaxSignedValue()) {
APInt CPlusOne = C + 1;
if (isLegalCmpImmed(CPlusOne)) {
CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
RHS = DAG.getConstant(CPlusOne, dl, VT);
}
}
break;
case ISD::SETULE:
case ISD::SETUGT:
if ((VT == MVT::i32 && C != UINT32_MAX &&
isLegalArithImmed((uint32_t)(C + 1))) ||
(VT == MVT::i64 && C != UINT64_MAX &&
isLegalArithImmed(C + 1ULL))) {
CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
RHS = DAG.getConstant(C, dl, VT);
if (!C.isAllOnes()) {
APInt CPlusOne = C + 1;
if (isLegalCmpImmed(CPlusOne)) {
CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
RHS = DAG.getConstant(CPlusOne, dl, VT);
}
}
break;
}
Expand All @@ -3830,8 +3860,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
// cmp w13, w12
// can be turned into:
// cmp w12, w11, lsl #1
if (!isa<ConstantSDNode>(RHS) ||
!isLegalArithImmed(RHS->getAsAPIntVal().abs().getZExtValue())) {
if (!isa<ConstantSDNode>(RHS) || !isLegalCmpImmed(RHS->getAsAPIntVal())) {
bool LHSIsCMN = isCMN(LHS, CC, DAG);
bool RHSIsCMN = isCMN(RHS, CC, DAG);
SDValue TheLHS = LHSIsCMN ? LHS.getOperand(1) : LHS;
Expand Down Expand Up @@ -17360,17 +17389,10 @@ LLT AArch64TargetLowering::getOptimalMemOpLLT(
// 12-bit optionally shifted immediates are legal for adds.
bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
if (Immed == std::numeric_limits<int64_t>::min()) {
LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed
<< ": avoid UB for INT64_MIN\n");
return false;
}
// Same encoding for add/sub, just flip the sign.
Immed = std::abs(Immed);
bool IsLegal = ((Immed >> 12) == 0 ||
((Immed & 0xfff) == 0 && Immed >> 24 == 0));
LLVM_DEBUG(dbgs() << "Is " << Immed
<< " legal add imm: " << (IsLegal ? "yes" : "no") << "\n");
return IsLegal;
return isLegalArithImmed((uint64_t)std::abs(Immed));
}

bool AArch64TargetLowering::isLegalAddScalableImmediate(int64_t Imm) const {
Expand Down
5 changes: 2 additions & 3 deletions llvm/test/CodeGen/AArch64/arm64-csel.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,8 @@ define i32 @foo7(i32 %a, i32 %b) nounwind {
; CHECK-NEXT: subs w8, w0, w1
; CHECK-NEXT: cneg w9, w8, mi
; CHECK-NEXT: cmn w8, #1
; CHECK-NEXT: csel w10, w9, w0, lt
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: csel w0, w10, w9, ge
; CHECK-NEXT: csel w8, w9, w0, lt
; CHECK-NEXT: csel w0, w8, w9, gt
; CHECK-NEXT: ret
entry:
%sub = sub nsw i32 %a, %b
Expand Down
16 changes: 8 additions & 8 deletions llvm/test/CodeGen/AArch64/check-sign-bit-before-extension.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ define i32 @f_i8_sign_extend_inreg(i8 %in, i32 %a, i32 %b) nounwind {
; CHECK-LABEL: f_i8_sign_extend_inreg:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sxtb w8, w0
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: csel w8, w1, w2, ge
; CHECK-NEXT: cmn w8, #1
; CHECK-NEXT: csel w8, w1, w2, gt
; CHECK-NEXT: add w0, w8, w0, uxtb
; CHECK-NEXT: ret
entry:
Expand All @@ -36,8 +36,8 @@ define i32 @f_i16_sign_extend_inreg(i16 %in, i32 %a, i32 %b) nounwind {
; CHECK-LABEL: f_i16_sign_extend_inreg:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: sxth w8, w0
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: csel w8, w1, w2, ge
; CHECK-NEXT: cmn w8, #1
; CHECK-NEXT: csel w8, w1, w2, gt
; CHECK-NEXT: add w0, w8, w0, uxth
; CHECK-NEXT: ret
entry:
Expand All @@ -57,8 +57,8 @@ B:
define i64 @f_i32_sign_extend_inreg(i32 %in, i64 %a, i64 %b) nounwind {
; CHECK-LABEL: f_i32_sign_extend_inreg:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: cmp w0, #0
; CHECK-NEXT: csel x8, x1, x2, ge
; CHECK-NEXT: cmn w0, #1
; CHECK-NEXT: csel x8, x1, x2, gt
; CHECK-NEXT: add x0, x8, w0, uxtw
; CHECK-NEXT: ret
entry:
Expand Down Expand Up @@ -145,8 +145,8 @@ define i64 @f_i32_sign_extend_i64(i32 %in, i64 %a, i64 %b) nounwind {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
; CHECK-NEXT: sxtw x8, w0
; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: csel x8, x1, x2, ge
; CHECK-NEXT: cmn x8, #1
; CHECK-NEXT: csel x8, x1, x2, gt
; CHECK-NEXT: add x0, x8, w0, uxtw
; CHECK-NEXT: ret
entry:
Expand Down
172 changes: 172 additions & 0 deletions llvm/test/CodeGen/AArch64/cmp-to-cmn.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,3 +430,175 @@ entry:
%cmp = icmp ne i32 %conv, %add
ret i1 %cmp
}

define i1 @cmn_large_imm(i32 %a) {
; CHECK-LABEL: cmn_large_imm:
; CHECK: // %bb.0:
; CHECK-NEXT: mov w8, #64765 // =0xfcfd
; CHECK-NEXT: movk w8, #64764, lsl #16
; CHECK-NEXT: cmp w0, w8
; CHECK-NEXT: cset w0, gt
; CHECK-NEXT: ret
%cmp = icmp sgt i32 %a, -50529027
ret i1 %cmp
}

define i1 @almost_immediate_neg_slt(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_slt:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, le
; CHECK-NEXT: ret
%cmp = icmp slt i32 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_slt_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_slt_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, le
; CHECK-NEXT: ret
%cmp = icmp slt i64 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_sge(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_sge:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, gt
; CHECK-NEXT: ret
%cmp = icmp sge i32 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_sge_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_sge_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, gt
; CHECK-NEXT: ret
%cmp = icmp sge i64 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_uge(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_uge:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp uge i32 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_uge_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_uge_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp uge i64 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_ult(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_ult:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, ls
; CHECK-NEXT: ret
%cmp = icmp ult i32 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_ult_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_ult_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584
; CHECK-NEXT: cset w0, ls
; CHECK-NEXT: ret
%cmp = icmp ult i64 %x, -16707583
ret i1 %cmp
}

define i1 @almost_immediate_neg_sle(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_sle:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, lt
; CHECK-NEXT: ret
%cmp = icmp sle i32 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_sle_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_sle_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, lt
; CHECK-NEXT: ret
%cmp = icmp sle i64 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_sgt(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_sgt:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, ge
; CHECK-NEXT: ret
%cmp = icmp sgt i32 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_sgt_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_sgt_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, ge
; CHECK-NEXT: ret
%cmp = icmp sgt i64 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_ule(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_ule:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ule i32 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_ule_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_ule_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ule i64 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_ugt(i32 %x) {
; CHECK-LABEL: almost_immediate_neg_ugt:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, hs
; CHECK-NEXT: ret
%cmp = icmp ugt i32 %x, -16773121
ret i1 %cmp
}

define i1 @almost_immediate_neg_ugt_64(i64 %x) {
; CHECK-LABEL: almost_immediate_neg_ugt_64:
; CHECK: // %bb.0:
; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120
; CHECK-NEXT: cset w0, hs
; CHECK-NEXT: ret
%cmp = icmp ugt i64 %x, -16773121
ret i1 %cmp
}
Loading