Skip to content

Commit 239ab16

Browse files
RKSimonRKSimon
committed
[X86] combineCMP - attempt to simplify KSHIFTR mask element extractions when just comparing against zero
We can just bitcast the pre-shifted mask as an integer and use TEST/BT directly. This can be extended further to better handle sub-i8 mask cases, but just getting rid of KSHIFTR nodes makes a notable difference.
1 parent e8af247 commit 239ab16

File tree

5 files changed

+67
-48
lines changed

5 files changed

+67
-48
lines changed

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53487,6 +53487,7 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
5348753487
SDLoc dl(N);
5348853488
SDValue Op = N->getOperand(0);
5348953489
EVT VT = Op.getValueType();
53490+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5349053491

5349153492
// If we have a constant logical shift that's only used in a comparison
5349253493
// against zero turn it into an equivalent AND. This allows turning it into
@@ -53510,12 +53511,42 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
5351053511
}
5351153512
}
5351253513

53514+
// If we're extracting from a avx512 bool vector and comparing against zero,
53515+
// then try to just bitcast the vector to an integer to use TEST/BT directly.
53516+
// TODO: Handle v2i1/v4i1 bool vector cases.
53517+
// (and (extract_elt (kshiftr vXi1, C), 0), 1) -> (and (bc vXi1), 1<<C)
53518+
if (Op.getOpcode() == ISD::AND && isOneConstant(Op.getOperand(1)) &&
53519+
Op.hasOneUse() && onlyZeroFlagUsed(SDValue(N, 0))) {
53520+
SDValue Src = Op.getOperand(0);
53521+
if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
53522+
isNullConstant(Src.getOperand(1)) &&
53523+
Src.getOperand(0).getValueType().getScalarType() == MVT::i1) {
53524+
SDValue BoolVec = Src.getOperand(0);
53525+
EVT VecVT = BoolVec.getValueType();
53526+
unsigned BitWidth = VecVT.getVectorNumElements();
53527+
EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth);
53528+
if (TLI.isTypeLegal(VecVT) && TLI.isTypeLegal(BCVT)) {
53529+
unsigned ShAmt = 0;
53530+
if (BoolVec.getOpcode() == X86ISD::KSHIFTR &&
53531+
BoolVec.getConstantOperandAPInt(1).ult(BitWidth)) {
53532+
ShAmt = BoolVec.getConstantOperandVal(1);
53533+
BoolVec = BoolVec.getOperand(0);
53534+
}
53535+
APInt Mask = APInt::getOneBitSet(BitWidth, ShAmt);
53536+
Op = DAG.getBitcast(BCVT, BoolVec);
53537+
Op = DAG.getNode(ISD::AND, dl, BCVT, Op,
53538+
DAG.getConstant(Mask, dl, BCVT));
53539+
return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53540+
DAG.getConstant(0, dl, VT));
53541+
}
53542+
}
53543+
}
53544+
5351353545
// Peek through any zero-extend if we're only testing for a zero result.
5351453546
if (Op.getOpcode() == ISD::ZERO_EXTEND && onlyZeroFlagUsed(SDValue(N, 0))) {
5351553547
SDValue Src = Op.getOperand(0);
5351653548
EVT SrcVT = Src.getValueType();
53517-
if (SrcVT.getScalarSizeInBits() >= 8 &&
53518-
DAG.getTargetLoweringInfo().isTypeLegal(SrcVT))
53549+
if (SrcVT.getScalarSizeInBits() >= 8 && TLI.isTypeLegal(SrcVT))
5351953550
return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,
5352053551
DAG.getConstant(0, dl, SrcVT));
5352153552
}

‎llvm/test/CodeGen/X86/avx512-insert-extract.ll

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,8 @@ define <16 x i32> @test11(<16 x i32>%a, <16 x i32>%b) nounwind {
175175
; KNL-LABEL: test11:
176176
; KNL: ## %bb.0:
177177
; KNL-NEXT: vpcmpltud %zmm1, %zmm0, %k0
178-
; KNL-NEXT: kshiftrw $4, %k0, %k0
179178
; KNL-NEXT: kmovw %k0, %eax
180-
; KNL-NEXT: testb $1, %al
179+
; KNL-NEXT: testb $16, %al
181180
; KNL-NEXT: je LBB10_2
182181
; KNL-NEXT: ## %bb.1: ## %A
183182
; KNL-NEXT: vmovdqa64 %zmm1, %zmm0
@@ -189,9 +188,8 @@ define <16 x i32> @test11(<16 x i32>%a, <16 x i32>%b) nounwind {
189188
; SKX-LABEL: test11:
190189
; SKX: ## %bb.0:
191190
; SKX-NEXT: vpcmpltud %zmm1, %zmm0, %k0
192-
; SKX-NEXT: kshiftrw $4, %k0, %k0
193191
; SKX-NEXT: kmovd %k0, %eax
194-
; SKX-NEXT: testb $1, %al
192+
; SKX-NEXT: testb $16, %al
195193
; SKX-NEXT: je LBB10_2
196194
; SKX-NEXT: ## %bb.1: ## %A
197195
; SKX-NEXT: vmovdqa64 %zmm1, %zmm0
@@ -276,9 +274,8 @@ define i64 @test14(<8 x i64>%a, <8 x i64>%b, i64 %a1, i64 %b1) nounwind {
276274
; KNL: ## %bb.0:
277275
; KNL-NEXT: movq %rdi, %rax
278276
; KNL-NEXT: vpcmpgtq %zmm0, %zmm1, %k0
279-
; KNL-NEXT: kshiftrw $4, %k0, %k0
280277
; KNL-NEXT: kmovw %k0, %ecx
281-
; KNL-NEXT: testb $1, %cl
278+
; KNL-NEXT: testb $16, %cl
282279
; KNL-NEXT: cmoveq %rsi, %rax
283280
; KNL-NEXT: vzeroupper
284281
; KNL-NEXT: retq
@@ -287,9 +284,8 @@ define i64 @test14(<8 x i64>%a, <8 x i64>%b, i64 %a1, i64 %b1) nounwind {
287284
; SKX: ## %bb.0:
288285
; SKX-NEXT: movq %rdi, %rax
289286
; SKX-NEXT: vpcmpgtq %zmm0, %zmm1, %k0
290-
; SKX-NEXT: kshiftrb $4, %k0, %k0
291287
; SKX-NEXT: kmovd %k0, %ecx
292-
; SKX-NEXT: testb $1, %cl
288+
; SKX-NEXT: testb $16, %cl
293289
; SKX-NEXT: cmoveq %rsi, %rax
294290
; SKX-NEXT: vzeroupper
295291
; SKX-NEXT: retq

‎llvm/test/CodeGen/X86/movmsk-cmp.ll

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4350,10 +4350,9 @@ define i32 @PR39665_c_ray(<2 x double> %x, <2 x double> %y) {
43504350
; KNL-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1
43514351
; KNL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
43524352
; KNL-NEXT: vcmpltpd %zmm0, %zmm1, %k0
4353-
; KNL-NEXT: kshiftrw $1, %k0, %k1
4354-
; KNL-NEXT: kmovw %k1, %eax
43554353
; KNL-NEXT: kmovw %k0, %ecx
4356-
; KNL-NEXT: testb $1, %al
4354+
; KNL-NEXT: kmovw %k0, %eax
4355+
; KNL-NEXT: testb $2, %al
43574356
; KNL-NEXT: movl $42, %eax
43584357
; KNL-NEXT: movl $99, %edx
43594358
; KNL-NEXT: cmovel %edx, %eax
@@ -4365,10 +4364,9 @@ define i32 @PR39665_c_ray(<2 x double> %x, <2 x double> %y) {
43654364
; SKX-LABEL: PR39665_c_ray:
43664365
; SKX: # %bb.0:
43674366
; SKX-NEXT: vcmpltpd %xmm0, %xmm1, %k0
4368-
; SKX-NEXT: kshiftrb $1, %k0, %k1
4369-
; SKX-NEXT: kmovd %k1, %eax
43704367
; SKX-NEXT: kmovd %k0, %ecx
4371-
; SKX-NEXT: testb $1, %al
4368+
; SKX-NEXT: kmovd %k0, %eax
4369+
; SKX-NEXT: testb $2, %al
43724370
; SKX-NEXT: movl $42, %eax
43734371
; SKX-NEXT: movl $99, %edx
43744372
; SKX-NEXT: cmovel %edx, %eax

‎llvm/test/CodeGen/X86/pr33349.ll

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,30 @@ target triple = "x86_64-unknown-linux-gnu"
1111
; KNL-NEXT: vpslld $31, %xmm0, %xmm0
1212
; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0
1313
; KNL-NEXT: kshiftrw $2, %k0, %k1
14-
; KNL-NEXT: kshiftrw $1, %k1, %k2
1514
; KNL-NEXT: kmovw %k1, %eax
16-
; KNL-NEXT: testb $1, %al
15+
; KNL-NEXT: kmovw %k1, %ecx
16+
; KNL-NEXT: testb $1, %cl
1717
; KNL-NEXT: fld1
1818
; KNL-NEXT: fldz
1919
; KNL-NEXT: fld %st(0)
2020
; KNL-NEXT: fcmovne %st(2), %st
21-
; KNL-NEXT: kmovw %k2, %eax
22-
; KNL-NEXT: testb $1, %al
21+
; KNL-NEXT: testb $2, %al
2322
; KNL-NEXT: fld %st(1)
2423
; KNL-NEXT: fcmovne %st(3), %st
25-
; KNL-NEXT: kshiftrw $1, %k0, %k1
26-
; KNL-NEXT: kmovw %k1, %eax
27-
; KNL-NEXT: testb $1, %al
24+
; KNL-NEXT: kmovw %k0, %eax
25+
; KNL-NEXT: kmovw %k0, %ecx
26+
; KNL-NEXT: testb $1, %cl
2827
; KNL-NEXT: fld %st(2)
2928
; KNL-NEXT: fcmovne %st(4), %st
30-
; KNL-NEXT: kmovw %k0, %eax
31-
; KNL-NEXT: testb $1, %al
29+
; KNL-NEXT: testb $2, %al
3230
; KNL-NEXT: fxch %st(3)
3331
; KNL-NEXT: fcmovne %st(4), %st
3432
; KNL-NEXT: fstp %st(4)
3533
; KNL-NEXT: fxch %st(3)
36-
; KNL-NEXT: fstpt (%rdi)
37-
; KNL-NEXT: fxch %st(1)
3834
; KNL-NEXT: fstpt 10(%rdi)
3935
; KNL-NEXT: fxch %st(1)
36+
; KNL-NEXT: fstpt (%rdi)
37+
; KNL-NEXT: fxch %st(1)
4038
; KNL-NEXT: fstpt 30(%rdi)
4139
; KNL-NEXT: fstpt 20(%rdi)
4240
; KNL-NEXT: vzeroupper
@@ -47,32 +45,30 @@ target triple = "x86_64-unknown-linux-gnu"
4745
; SKX-NEXT: vpslld $31, %xmm0, %xmm0
4846
; SKX-NEXT: vpmovd2m %xmm0, %k0
4947
; SKX-NEXT: kshiftrb $2, %k0, %k1
50-
; SKX-NEXT: kshiftrb $1, %k1, %k2
5148
; SKX-NEXT: kmovd %k1, %eax
52-
; SKX-NEXT: testb $1, %al
49+
; SKX-NEXT: kmovd %k1, %ecx
50+
; SKX-NEXT: testb $1, %cl
5351
; SKX-NEXT: fld1
5452
; SKX-NEXT: fldz
5553
; SKX-NEXT: fld %st(0)
5654
; SKX-NEXT: fcmovne %st(2), %st
57-
; SKX-NEXT: kmovd %k2, %eax
58-
; SKX-NEXT: testb $1, %al
55+
; SKX-NEXT: testb $2, %al
5956
; SKX-NEXT: fld %st(1)
6057
; SKX-NEXT: fcmovne %st(3), %st
61-
; SKX-NEXT: kshiftrb $1, %k0, %k1
62-
; SKX-NEXT: kmovd %k1, %eax
63-
; SKX-NEXT: testb $1, %al
58+
; SKX-NEXT: kmovd %k0, %eax
59+
; SKX-NEXT: kmovd %k0, %ecx
60+
; SKX-NEXT: testb $1, %cl
6461
; SKX-NEXT: fld %st(2)
6562
; SKX-NEXT: fcmovne %st(4), %st
66-
; SKX-NEXT: kmovd %k0, %eax
67-
; SKX-NEXT: testb $1, %al
63+
; SKX-NEXT: testb $2, %al
6864
; SKX-NEXT: fxch %st(3)
6965
; SKX-NEXT: fcmovne %st(4), %st
7066
; SKX-NEXT: fstp %st(4)
7167
; SKX-NEXT: fxch %st(3)
72-
; SKX-NEXT: fstpt (%rdi)
73-
; SKX-NEXT: fxch %st(1)
7468
; SKX-NEXT: fstpt 10(%rdi)
7569
; SKX-NEXT: fxch %st(1)
70+
; SKX-NEXT: fstpt (%rdi)
71+
; SKX-NEXT: fxch %st(1)
7672
; SKX-NEXT: fstpt 30(%rdi)
7773
; SKX-NEXT: fstpt 20(%rdi)
7874
; SKX-NEXT: retq

‎llvm/test/CodeGen/X86/pr34177.ll

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,24 +49,22 @@ define void @test(<4 x i64> %a, <4 x x86_fp80> %b, ptr %c) local_unnamed_addr {
4949
; AVX512VL-LABEL: test:
5050
; AVX512VL: # %bb.0:
5151
; AVX512VL-NEXT: vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %k0
52-
; AVX512VL-NEXT: kshiftrb $1, %k0, %k1
53-
; AVX512VL-NEXT: kshiftrb $2, %k0, %k2
5452
; AVX512VL-NEXT: kmovd %k0, %eax
55-
; AVX512VL-NEXT: testb $1, %al
53+
; AVX512VL-NEXT: kshiftrb $2, %k0, %k1
54+
; AVX512VL-NEXT: kmovd %k0, %ecx
55+
; AVX512VL-NEXT: testb $2, %cl
5656
; AVX512VL-NEXT: fld1
5757
; AVX512VL-NEXT: fldz
5858
; AVX512VL-NEXT: fld %st(0)
5959
; AVX512VL-NEXT: fcmovne %st(2), %st
60-
; AVX512VL-NEXT: kmovd %k1, %eax
6160
; AVX512VL-NEXT: testb $1, %al
6261
; AVX512VL-NEXT: fld %st(1)
6362
; AVX512VL-NEXT: fcmovne %st(3), %st
64-
; AVX512VL-NEXT: kshiftrb $1, %k2, %k0
65-
; AVX512VL-NEXT: kmovd %k0, %eax
66-
; AVX512VL-NEXT: testb $1, %al
63+
; AVX512VL-NEXT: kmovd %k1, %eax
64+
; AVX512VL-NEXT: kmovd %k1, %ecx
65+
; AVX512VL-NEXT: testb $2, %cl
6766
; AVX512VL-NEXT: fld %st(2)
6867
; AVX512VL-NEXT: fcmovne %st(4), %st
69-
; AVX512VL-NEXT: kmovd %k2, %eax
7068
; AVX512VL-NEXT: testb $1, %al
7169
; AVX512VL-NEXT: fxch %st(3)
7270
; AVX512VL-NEXT: fcmovne %st(4), %st
@@ -81,10 +79,10 @@ define void @test(<4 x i64> %a, <4 x x86_fp80> %b, ptr %c) local_unnamed_addr {
8179
; AVX512VL-NEXT: fstpt 10(%rdi)
8280
; AVX512VL-NEXT: fxch %st(1)
8381
; AVX512VL-NEXT: fadd %st, %st(0)
84-
; AVX512VL-NEXT: fstpt (%rdi)
85-
; AVX512VL-NEXT: fadd %st, %st(0)
8682
; AVX512VL-NEXT: fstpt 20(%rdi)
8783
; AVX512VL-NEXT: fadd %st, %st(0)
84+
; AVX512VL-NEXT: fstpt (%rdi)
85+
; AVX512VL-NEXT: fadd %st, %st(0)
8886
; AVX512VL-NEXT: fstpt 60(%rdi)
8987
; AVX512VL-NEXT: fadd %st, %st(0)
9088
; AVX512VL-NEXT: fstpt 40(%rdi)

0 commit comments

Comments
 (0)