Skip to content

Commit 62d77fc

Browse files
RKSimonRKSimon
authored
[X86] combineX86ShuffleChain - don't combine to VPERM2W/VPERM2B from just any single variable mask (#127914)
Despite them being more expensive than other variable mask shuffles, we were combining shuffle chains to VPERM2W/VPERM2B if any shuffle in the chain was a variable shuffle - including very cheap shuffles like PSHUFB or AND mask patterns. This patch adjusts the BWI VPERMV3 threshold - it still always permits the merge if the chain (of 2 or more shuffles) contains any X86ISD::VPERMV/VPERMV3 shuffles (including DQ variants), but otherwise only reduces the depth threshold based off the number of other variable shuffles we'd fold away.
1 parent 0cd5a1f commit 62d77fc

11 files changed

+187
-151
lines changed

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40064,23 +40064,29 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
4006440064
if (Depth < 1)
4006540065
return SDValue();
4006640066

40067-
bool HasVariableMask = llvm::any_of(SrcNodes, [](const SDNode *N) {
40067+
int NumVariableMasks = llvm::count_if(SrcNodes, [](const SDNode *N) {
4006840068
return isTargetShuffleVariableMask(N->getOpcode());
4006940069
});
40070+
bool HasSlowVariableMask = llvm::any_of(SrcNodes, [](const SDNode *N) {
40071+
return (N->getOpcode() == X86ISD::VPERMV3 ||
40072+
N->getOpcode() == X86ISD::VPERMV);
40073+
});
4007040074

4007140075
// Depth threshold above which we can efficiently use variable mask shuffles.
4007240076
int VariableCrossLaneShuffleDepth =
4007340077
Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2;
4007440078
int VariablePerLaneShuffleDepth =
4007540079
Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2;
4007640080
AllowVariableCrossLaneMask &=
40077-
(Depth >= VariableCrossLaneShuffleDepth) || HasVariableMask;
40081+
(Depth >= VariableCrossLaneShuffleDepth) || NumVariableMasks;
4007840082
AllowVariablePerLaneMask &=
40079-
(Depth >= VariablePerLaneShuffleDepth) || HasVariableMask;
40080-
// VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a
40083+
(Depth >= VariablePerLaneShuffleDepth) || NumVariableMasks;
40084+
// VPERM2W/VPERM2B are 3 uops on Skylake and Icelake so we require a
4008140085
// higher depth before combining them.
40086+
int BWIVPERMV3ShuffleDepth =
40087+
VariableCrossLaneShuffleDepth + 2 - NumVariableMasks;
4008240088
bool AllowBWIVPERMV3 =
40083-
(Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
40089+
(Depth >= BWIVPERMV3ShuffleDepth || HasSlowVariableMask);
4008440090

4008540091
// If root was a VPERMV3 node, always allow a variable shuffle.
4008640092
if (Root.getOpcode() == X86ISD::VPERMV3)

‎llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -225,15 +225,25 @@ define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16
225225
}
226226

227227
define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {
228-
; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
229-
; CHECK: # %bb.0:
230-
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
231-
; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
232-
; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1
233-
; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
234-
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
235-
; CHECK-NEXT: vzeroupper
236-
; CHECK-NEXT: retq
228+
; CHECK-FAST-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
229+
; CHECK-FAST: # %bb.0:
230+
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
231+
; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
232+
; CHECK-FAST-NEXT: vptestnmw %xmm1, %xmm1, %k1
233+
; CHECK-FAST-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
234+
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
235+
; CHECK-FAST-NEXT: vzeroupper
236+
; CHECK-FAST-NEXT: retq
237+
;
238+
; CHECK-FAST-PERLANE-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
239+
; CHECK-FAST-PERLANE: # %bb.0:
240+
; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm2
241+
; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3
242+
; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
243+
; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]
244+
; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm1, %xmm1, %k1
245+
; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm2, %xmm0 {%k1}
246+
; CHECK-FAST-PERLANE-NEXT: retq
237247
%vec = load <16 x i16>, ptr %vp
238248
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
239249
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
@@ -242,14 +252,24 @@ define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16>
242252
}
243253

244254
define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {
245-
; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
246-
; CHECK: # %bb.0:
247-
; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
248-
; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1
249-
; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
250-
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
251-
; CHECK-NEXT: vzeroupper
252-
; CHECK-NEXT: retq
255+
; CHECK-FAST-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
256+
; CHECK-FAST: # %bb.0:
257+
; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
258+
; CHECK-FAST-NEXT: vptestnmw %xmm0, %xmm0, %k1
259+
; CHECK-FAST-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
260+
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
261+
; CHECK-FAST-NEXT: vzeroupper
262+
; CHECK-FAST-NEXT: retq
263+
;
264+
; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
265+
; CHECK-FAST-PERLANE: # %bb.0:
266+
; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm1
267+
; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
268+
; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
269+
; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
270+
; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm0, %xmm0, %k1
271+
; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} {z}
272+
; CHECK-FAST-PERLANE-NEXT: retq
253273
%vec = load <16 x i16>, ptr %vp
254274
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
255275
%cmp = icmp eq <8 x i16> %mask, zeroinitializer

‎llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -293,8 +293,8 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
293293
; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
294294
; AVX512BW-FCP-NEXT: vpbroadcastw 4(%rdi), %xmm4
295295
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
296-
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,9,1,9,2,10,3,11]
297-
; AVX512BW-FCP-NEXT: vpermw (%rdi), %ymm5, %ymm5
296+
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
297+
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
298298
; AVX512BW-FCP-NEXT: vpbroadcastw 20(%rdi), %xmm6
299299
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm7
300300
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
@@ -307,7 +307,6 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
307307
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%r8)
308308
; AVX512BW-FCP-NEXT: vmovd %xmm6, (%r9)
309309
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rax)
310-
; AVX512BW-FCP-NEXT: vzeroupper
311310
; AVX512BW-FCP-NEXT: retq
312311
;
313312
; AVX512DQ-BW-LABEL: load_i16_stride6_vf2:
@@ -347,8 +346,8 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
347346
; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
348347
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 4(%rdi), %xmm4
349348
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
350-
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,9,1,9,2,10,3,11]
351-
; AVX512DQ-BW-FCP-NEXT: vpermw (%rdi), %ymm5, %ymm5
349+
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
350+
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
352351
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 20(%rdi), %xmm6
353352
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm7
354353
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
@@ -361,7 +360,6 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
361360
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%r8)
362361
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm6, (%r9)
363362
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rax)
364-
; AVX512DQ-BW-FCP-NEXT: vzeroupper
365363
; AVX512DQ-BW-FCP-NEXT: retq
366364
%wide.vec = load <12 x i16>, ptr %in.vec, align 64
367365
%strided.vec0 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 0, i32 6>

‎llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -321,23 +321,23 @@ define void @load_i16_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
321321
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
322322
; AVX512BW-FCP-NEXT: vpsrld $16, %xmm0, %xmm3
323323
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
324-
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
325-
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[8,9,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
326-
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
327-
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm6
328-
; AVX512BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm7
329-
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
330-
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
331-
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [6,13,5,13,6,14,7,15]
332-
; AVX512BW-FCP-NEXT: vpermw (%rdi), %ymm3, %ymm3
324+
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
325+
; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
326+
; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm7
327+
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
328+
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm8
329+
; AVX512BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm9
330+
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
331+
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
332+
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
333+
; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm0
333334
; AVX512BW-FCP-NEXT: vmovd %xmm2, (%rsi)
334335
; AVX512BW-FCP-NEXT: vmovd %xmm4, (%rdx)
335-
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%rcx)
336-
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%r8)
337-
; AVX512BW-FCP-NEXT: vmovd %xmm6, (%r9)
338-
; AVX512BW-FCP-NEXT: vmovd %xmm1, (%r10)
339-
; AVX512BW-FCP-NEXT: vmovd %xmm3, (%rax)
340-
; AVX512BW-FCP-NEXT: vzeroupper
336+
; AVX512BW-FCP-NEXT: vmovd %xmm7, (%rcx)
337+
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%r8)
338+
; AVX512BW-FCP-NEXT: vmovd %xmm8, (%r9)
339+
; AVX512BW-FCP-NEXT: vmovd %xmm3, (%r10)
340+
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rax)
341341
; AVX512BW-FCP-NEXT: retq
342342
;
343343
; AVX512DQ-BW-LABEL: load_i16_stride7_vf2:
@@ -379,23 +379,23 @@ define void @load_i16_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
379379
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
380380
; AVX512DQ-BW-FCP-NEXT: vpsrld $16, %xmm0, %xmm3
381381
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
382-
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
383-
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[8,9,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
384-
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
385-
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm6
386-
; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm7
387-
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
388-
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
389-
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [6,13,5,13,6,14,7,15]
390-
; AVX512DQ-BW-FCP-NEXT: vpermw (%rdi), %ymm3, %ymm3
382+
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
383+
; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
384+
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm7
385+
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
386+
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm8
387+
; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm9
388+
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
389+
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
390+
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
391+
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm0
391392
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%rsi)
392393
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm4, (%rdx)
393-
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%rcx)
394-
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%r8)
395-
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm6, (%r9)
396-
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%r10)
397-
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%rax)
398-
; AVX512DQ-BW-FCP-NEXT: vzeroupper
394+
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm7, (%rcx)
395+
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%r8)
396+
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm8, (%r9)
397+
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%r10)
398+
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rax)
399399
; AVX512DQ-BW-FCP-NEXT: retq
400400
%wide.vec = load <14 x i16>, ptr %in.vec, align 64
401401
%strided.vec0 = shufflevector <14 x i16> %wide.vec, <14 x i16> poison, <2 x i32> <i32 0, i32 7>

‎llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,9 @@ define void @store_i16_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
123123
; AVX512BW-NEXT: vmovdqa (%rdx), %xmm1
124124
; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
125125
; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
126-
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]
127-
; AVX512BW-NEXT: vpermi2w %xmm1, %xmm0, %xmm2
128-
; AVX512BW-NEXT: vmovdqa %xmm2, (%r8)
126+
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
127+
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]
128+
; AVX512BW-NEXT: vmovdqa %xmm0, (%r8)
129129
; AVX512BW-NEXT: retq
130130
;
131131
; AVX512BW-FCP-LABEL: store_i16_stride4_vf2:
@@ -145,9 +145,9 @@ define void @store_i16_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
145145
; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %xmm1
146146
; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
147147
; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
148-
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]
149-
; AVX512DQ-BW-NEXT: vpermi2w %xmm1, %xmm0, %xmm2
150-
; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%r8)
148+
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
149+
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]
150+
; AVX512DQ-BW-NEXT: vmovdqa %xmm0, (%r8)
151151
; AVX512DQ-BW-NEXT: retq
152152
;
153153
; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf2:

‎llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll

Lines changed: 10 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -495,23 +495,11 @@ define <16 x i8> @shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20(
495495
; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
496496
; AVX1-NEXT: retq
497497
;
498-
; AVX2-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
499-
; AVX2: # %bb.0:
500-
; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
501-
; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
502-
; AVX2-NEXT: retq
503-
;
504-
; AVX512VLBW-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
505-
; AVX512VLBW: # %bb.0:
506-
; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
507-
; AVX512VLBW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
508-
; AVX512VLBW-NEXT: retq
509-
;
510-
; AVX512VLVBMI-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
511-
; AVX512VLVBMI: # %bb.0:
512-
; AVX512VLVBMI-NEXT: vmovdqa {{.*#+}} xmm2 = [3,2,1,0,7,6,5,4,19,18,17,16,23,22,21,20]
513-
; AVX512VLVBMI-NEXT: vpermt2b %xmm1, %xmm2, %xmm0
514-
; AVX512VLVBMI-NEXT: retq
498+
; AVX2OR512VL-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
499+
; AVX2OR512VL: # %bb.0:
500+
; AVX2OR512VL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
501+
; AVX2OR512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
502+
; AVX2OR512VL-NEXT: retq
515503
;
516504
; XOP-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
517505
; XOP: # %bb.0:
@@ -853,23 +841,11 @@ define <16 x i8> @shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(
853841
; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
854842
; AVX1-NEXT: retq
855843
;
856-
; AVX2-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
857-
; AVX2: # %bb.0:
858-
; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
859-
; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
860-
; AVX2-NEXT: retq
861-
;
862-
; AVX512VLBW-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
863-
; AVX512VLBW: # %bb.0:
864-
; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
865-
; AVX512VLBW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
866-
; AVX512VLBW-NEXT: retq
867-
;
868-
; AVX512VLVBMI-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
869-
; AVX512VLVBMI: # %bb.0:
870-
; AVX512VLVBMI-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,20,2,20,2,20,2,20,2,20,2,20,2,20,2,20]
871-
; AVX512VLVBMI-NEXT: vpermt2b %xmm1, %xmm2, %xmm0
872-
; AVX512VLVBMI-NEXT: retq
844+
; AVX2OR512VL-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
845+
; AVX2OR512VL: # %bb.0:
846+
; AVX2OR512VL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
847+
; AVX2OR512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
848+
; AVX2OR512VL-NEXT: retq
873849
;
874850
; XOP-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
875851
; XOP: # %bb.0:

0 commit comments

Comments
 (0)