Skip to content

Commit 6997f4d

Browse files
RKSimonRKSimon
committed
[X86] combineSetCCMOVMSK - fold allof(cmpeq(x,y)) -> ptest(sub(x,y)) (PR53379)
As suggested on PR53379, for all-of icmp-eq patterns, we can use ptest(sub(x,y)) on SSE41+ targets This is a generalization of the existing allof(cmpeq(x,0)) -> ptest(x) pattern We can probably extend this further, in particularly to handle 256-bit cases on pre-AVX2 targets, but this part of the generalization is pretty trivial Fixes Issue #53379
1 parent 699e22a commit 6997f4d

File tree

5 files changed

+75
-70
lines changed

5 files changed

+75
-70
lines changed

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44457,12 +44457,15 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
4445744457

4445844458
// MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X).
4445944459
// MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X).
44460+
// MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(SUB(X,Y),SUB(X,Y)).
44461+
// MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(SUB(X,Y),SUB(X,Y)).
4446044462
if (IsAllOf && Subtarget.hasSSE41()) {
4446144463
SDValue BC = peekThroughBitcasts(Vec);
44462-
if (BC.getOpcode() == X86ISD::PCMPEQ &&
44463-
ISD::isBuildVectorAllZeros(BC.getOperand(1).getNode())) {
44464+
if (BC.getOpcode() == X86ISD::PCMPEQ) {
4446444465
MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
44465-
SDValue V = DAG.getBitcast(TestVT, BC.getOperand(0));
44466+
SDValue V = DAG.getNode(ISD::SUB, SDLoc(BC), BC.getValueType(),
44467+
BC.getOperand(0), BC.getOperand(1));
44468+
V = DAG.getBitcast(TestVT, V);
4446644469
return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
4446744470
}
4446844471
}
@@ -44500,7 +44503,14 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
4450044503
VecOp1.getConstantOperandAPInt(1) == 8 &&
4450144504
(IsAnyOf || (SignExt0 && SignExt1))) {
4450244505
SDLoc DL(EFLAGS);
44503-
SDValue Result = DAG.getBitcast(MVT::v32i8, VecOp0.getOperand(0));
44506+
SDValue Result = peekThroughBitcasts(VecOp0.getOperand(0));
44507+
if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ) {
44508+
SDValue V = DAG.getNode(ISD::SUB, DL, Result.getValueType(),
44509+
Result.getOperand(0), Result.getOperand(1));
44510+
V = DAG.getBitcast(MVT::v4i64, V);
44511+
return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
44512+
}
44513+
Result = DAG.getBitcast(MVT::v32i8, Result);
4450444514
Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
4450544515
unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF;
4450644516
if (!SignExt0 || !SignExt1) {

‎llvm/test/CodeGen/X86/movmsk-cmp.ll

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4162,17 +4162,15 @@ define i1 @movmsk_or_v2i64(<2 x i64> %x, <2 x i64> %y) {
41624162
;
41634163
; SSE41-LABEL: movmsk_or_v2i64:
41644164
; SSE41: # %bb.0:
4165-
; SSE41-NEXT: pcmpeqq %xmm1, %xmm0
4166-
; SSE41-NEXT: movmskpd %xmm0, %eax
4167-
; SSE41-NEXT: cmpl $3, %eax
4165+
; SSE41-NEXT: psubq %xmm1, %xmm0
4166+
; SSE41-NEXT: ptest %xmm0, %xmm0
41684167
; SSE41-NEXT: setne %al
41694168
; SSE41-NEXT: retq
41704169
;
41714170
; AVX1OR2-LABEL: movmsk_or_v2i64:
41724171
; AVX1OR2: # %bb.0:
4173-
; AVX1OR2-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0
4174-
; AVX1OR2-NEXT: vmovmskpd %xmm0, %eax
4175-
; AVX1OR2-NEXT: cmpl $3, %eax
4172+
; AVX1OR2-NEXT: vpsubq %xmm1, %xmm0, %xmm0
4173+
; AVX1OR2-NEXT: vptest %xmm0, %xmm0
41764174
; AVX1OR2-NEXT: setne %al
41774175
; AVX1OR2-NEXT: retq
41784176
;

‎llvm/test/CodeGen/X86/vector-compare-all_of.ll

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1249,9 +1249,8 @@ define i1 @bool_reduction_v8i32(<8 x i32> %x, <8 x i32> %y) {
12491249
; AVX2-LABEL: bool_reduction_v8i32:
12501250
; AVX2: # %bb.0:
12511251
; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1
1252-
; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
1253-
; AVX2-NEXT: vmovmskps %ymm0, %eax
1254-
; AVX2-NEXT: cmpb $-1, %al
1252+
; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
1253+
; AVX2-NEXT: vptest %ymm0, %ymm0
12551254
; AVX2-NEXT: sete %al
12561255
; AVX2-NEXT: vzeroupper
12571256
; AVX2-NEXT: retq
@@ -1301,9 +1300,8 @@ define i1 @bool_reduction_v16i16(<16 x i16> %x, <16 x i16> %y) {
13011300
;
13021301
; AVX2-LABEL: bool_reduction_v16i16:
13031302
; AVX2: # %bb.0:
1304-
; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
1305-
; AVX2-NEXT: vpmovmskb %ymm0, %eax
1306-
; AVX2-NEXT: cmpl $-1, %eax
1303+
; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0
1304+
; AVX2-NEXT: vptest %ymm0, %ymm0
13071305
; AVX2-NEXT: sete %al
13081306
; AVX2-NEXT: vzeroupper
13091307
; AVX2-NEXT: retq
@@ -1354,9 +1352,8 @@ define i1 @bool_reduction_v32i8(<32 x i8> %x, <32 x i8> %y) {
13541352
;
13551353
; AVX2-LABEL: bool_reduction_v32i8:
13561354
; AVX2: # %bb.0:
1357-
; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
1358-
; AVX2-NEXT: vpmovmskb %ymm0, %eax
1359-
; AVX2-NEXT: cmpl $-1, %eax
1355+
; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
1356+
; AVX2-NEXT: vptest %ymm0, %ymm0
13601357
; AVX2-NEXT: sete %al
13611358
; AVX2-NEXT: vzeroupper
13621359
; AVX2-NEXT: retq

‎llvm/test/CodeGen/X86/vector-compare-any_of.ll

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -959,17 +959,15 @@ define i1 @bool_reduction_v2i64(<2 x i64> %x, <2 x i64> %y) {
959959
define i1 @bool_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
960960
; SSE-LABEL: bool_reduction_v4i32:
961961
; SSE: # %bb.0:
962-
; SSE-NEXT: pcmpeqd %xmm1, %xmm0
963-
; SSE-NEXT: movmskps %xmm0, %eax
964-
; SSE-NEXT: cmpl $15, %eax
962+
; SSE-NEXT: psubd %xmm1, %xmm0
963+
; SSE-NEXT: ptest %xmm0, %xmm0
965964
; SSE-NEXT: setne %al
966965
; SSE-NEXT: retq
967966
;
968967
; AVX-LABEL: bool_reduction_v4i32:
969968
; AVX: # %bb.0:
970-
; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
971-
; AVX-NEXT: vmovmskps %xmm0, %eax
972-
; AVX-NEXT: cmpl $15, %eax
969+
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
970+
; AVX-NEXT: vptest %xmm0, %xmm0
973971
; AVX-NEXT: setne %al
974972
; AVX-NEXT: retq
975973
;

‎llvm/test/CodeGen/X86/vector-reduce-and-bool.ll

Lines changed: 47 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1210,10 +1210,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
12101210
;
12111211
; AVX2-LABEL: icmp0_v16i16_v16i1:
12121212
; AVX2: # %bb.0:
1213-
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
1214-
; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
1215-
; AVX2-NEXT: vpmovmskb %ymm0, %eax
1216-
; AVX2-NEXT: cmpl $-1, %eax
1213+
; AVX2-NEXT: vptest %ymm0, %ymm0
12171214
; AVX2-NEXT: sete %al
12181215
; AVX2-NEXT: vzeroupper
12191216
; AVX2-NEXT: retq
@@ -1679,17 +1676,15 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
16791676
;
16801677
; SSE41-LABEL: icmp_v2i64_v2i1:
16811678
; SSE41: # %bb.0:
1682-
; SSE41-NEXT: pcmpeqq %xmm1, %xmm0
1683-
; SSE41-NEXT: movmskpd %xmm0, %eax
1684-
; SSE41-NEXT: cmpb $3, %al
1679+
; SSE41-NEXT: psubq %xmm1, %xmm0
1680+
; SSE41-NEXT: ptest %xmm0, %xmm0
16851681
; SSE41-NEXT: sete %al
16861682
; SSE41-NEXT: retq
16871683
;
16881684
; AVX-LABEL: icmp_v2i64_v2i1:
16891685
; AVX: # %bb.0:
1690-
; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0
1691-
; AVX-NEXT: vmovmskpd %xmm0, %eax
1692-
; AVX-NEXT: cmpb $3, %al
1686+
; AVX-NEXT: vpsubq %xmm1, %xmm0, %xmm0
1687+
; AVX-NEXT: vptest %xmm0, %xmm0
16931688
; AVX-NEXT: sete %al
16941689
; AVX-NEXT: retq
16951690
;
@@ -1728,19 +1723,25 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
17281723
}
17291724

17301725
define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
1731-
; SSE-LABEL: icmp_v4i32_v4i1:
1732-
; SSE: # %bb.0:
1733-
; SSE-NEXT: pcmpeqd %xmm1, %xmm0
1734-
; SSE-NEXT: movmskps %xmm0, %eax
1735-
; SSE-NEXT: cmpb $15, %al
1736-
; SSE-NEXT: sete %al
1737-
; SSE-NEXT: retq
1726+
; SSE2-LABEL: icmp_v4i32_v4i1:
1727+
; SSE2: # %bb.0:
1728+
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
1729+
; SSE2-NEXT: movmskps %xmm0, %eax
1730+
; SSE2-NEXT: cmpb $15, %al
1731+
; SSE2-NEXT: sete %al
1732+
; SSE2-NEXT: retq
1733+
;
1734+
; SSE41-LABEL: icmp_v4i32_v4i1:
1735+
; SSE41: # %bb.0:
1736+
; SSE41-NEXT: psubd %xmm1, %xmm0
1737+
; SSE41-NEXT: ptest %xmm0, %xmm0
1738+
; SSE41-NEXT: sete %al
1739+
; SSE41-NEXT: retq
17381740
;
17391741
; AVX-LABEL: icmp_v4i32_v4i1:
17401742
; AVX: # %bb.0:
1741-
; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
1742-
; AVX-NEXT: vmovmskps %xmm0, %eax
1743-
; AVX-NEXT: cmpb $15, %al
1743+
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
1744+
; AVX-NEXT: vptest %xmm0, %xmm0
17441745
; AVX-NEXT: sete %al
17451746
; AVX-NEXT: retq
17461747
;
@@ -1832,27 +1833,32 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
18321833
}
18331834

18341835
define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
1835-
; SSE-LABEL: icmp_v16i8_v16i1:
1836-
; SSE: # %bb.0:
1837-
; SSE-NEXT: pcmpeqb %xmm1, %xmm0
1838-
; SSE-NEXT: pmovmskb %xmm0, %eax
1839-
; SSE-NEXT: cmpw $-1, %ax
1840-
; SSE-NEXT: sete %al
1841-
; SSE-NEXT: retq
1836+
; SSE2-LABEL: icmp_v16i8_v16i1:
1837+
; SSE2: # %bb.0:
1838+
; SSE2-NEXT: pcmpeqb %xmm1, %xmm0
1839+
; SSE2-NEXT: pmovmskb %xmm0, %eax
1840+
; SSE2-NEXT: cmpw $-1, %ax
1841+
; SSE2-NEXT: sete %al
1842+
; SSE2-NEXT: retq
1843+
;
1844+
; SSE41-LABEL: icmp_v16i8_v16i1:
1845+
; SSE41: # %bb.0:
1846+
; SSE41-NEXT: psubb %xmm1, %xmm0
1847+
; SSE41-NEXT: ptest %xmm0, %xmm0
1848+
; SSE41-NEXT: sete %al
1849+
; SSE41-NEXT: retq
18421850
;
18431851
; AVX-LABEL: icmp_v16i8_v16i1:
18441852
; AVX: # %bb.0:
1845-
; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
1846-
; AVX-NEXT: vpmovmskb %xmm0, %eax
1847-
; AVX-NEXT: cmpw $-1, %ax
1853+
; AVX-NEXT: vpsubb %xmm1, %xmm0, %xmm0
1854+
; AVX-NEXT: vptest %xmm0, %xmm0
18481855
; AVX-NEXT: sete %al
18491856
; AVX-NEXT: retq
18501857
;
18511858
; AVX512F-LABEL: icmp_v16i8_v16i1:
18521859
; AVX512F: # %bb.0:
1853-
; AVX512F-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
1854-
; AVX512F-NEXT: vpmovmskb %xmm0, %eax
1855-
; AVX512F-NEXT: cmpw $-1, %ax
1860+
; AVX512F-NEXT: vpsubb %xmm1, %xmm0, %xmm0
1861+
; AVX512F-NEXT: vptest %xmm0, %xmm0
18561862
; AVX512F-NEXT: sete %al
18571863
; AVX512F-NEXT: retq
18581864
;
@@ -1917,9 +1923,8 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
19171923
;
19181924
; AVX2-LABEL: icmp_v4i64_v4i1:
19191925
; AVX2: # %bb.0:
1920-
; AVX2-NEXT: vpcmpeqq %ymm1, %ymm0, %ymm0
1921-
; AVX2-NEXT: vmovmskpd %ymm0, %eax
1922-
; AVX2-NEXT: cmpb $15, %al
1926+
; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
1927+
; AVX2-NEXT: vptest %ymm0, %ymm0
19231928
; AVX2-NEXT: sete %al
19241929
; AVX2-NEXT: vzeroupper
19251930
; AVX2-NEXT: retq
@@ -1986,9 +1991,8 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
19861991
;
19871992
; AVX2-LABEL: icmp_v8i32_v8i1:
19881993
; AVX2: # %bb.0:
1989-
; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
1990-
; AVX2-NEXT: vmovmskps %ymm0, %eax
1991-
; AVX2-NEXT: cmpb $-1, %al
1994+
; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
1995+
; AVX2-NEXT: vptest %ymm0, %ymm0
19921996
; AVX2-NEXT: sete %al
19931997
; AVX2-NEXT: vzeroupper
19941998
; AVX2-NEXT: retq
@@ -2054,9 +2058,8 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
20542058
;
20552059
; AVX2-LABEL: icmp_v16i16_v16i1:
20562060
; AVX2: # %bb.0:
2057-
; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
2058-
; AVX2-NEXT: vpmovmskb %ymm0, %eax
2059-
; AVX2-NEXT: cmpl $-1, %eax
2061+
; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0
2062+
; AVX2-NEXT: vptest %ymm0, %ymm0
20602063
; AVX2-NEXT: sete %al
20612064
; AVX2-NEXT: vzeroupper
20622065
; AVX2-NEXT: retq
@@ -2119,9 +2122,8 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
21192122
;
21202123
; AVX2-LABEL: icmp_v32i8_v32i1:
21212124
; AVX2: # %bb.0:
2122-
; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
2123-
; AVX2-NEXT: vpmovmskb %ymm0, %eax
2124-
; AVX2-NEXT: cmpl $-1, %eax
2125+
; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
2126+
; AVX2-NEXT: vptest %ymm0, %ymm0
21252127
; AVX2-NEXT: sete %al
21262128
; AVX2-NEXT: vzeroupper
21272129
; AVX2-NEXT: retq

0 commit comments

Comments
 (0)