Skip to content

[X86] combineX86ShuffleChain - don't combine to VPERM2W/VPERM2B from just any single variable mask #127914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 20, 2025
Merged

[X86] combineX86ShuffleChain - don't combine to VPERM2W/VPERM2B from just any single variable mask #127914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cc24702
[X86] combineX86ShuffleChain - don't combine to VPERMI2W/VPERMI2B fro…
RKSimon Feb 19, 2025
aacbcc3
Simplify comment - VPERMI2W/VPERMI2B suggests it just works for that …
RKSimon Feb 20, 2025
71e4da7
Merge branch 'upstream' into x86-bw-permv3-depth
RKSimon Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40064,23 +40064,29 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
if (Depth < 1)
return SDValue();

bool HasVariableMask = llvm::any_of(SrcNodes, [](const SDNode *N) {
int NumVariableMasks = llvm::count_if(SrcNodes, [](const SDNode *N) {
return isTargetShuffleVariableMask(N->getOpcode());
});
bool HasSlowVariableMask = llvm::any_of(SrcNodes, [](const SDNode *N) {
return (N->getOpcode() == X86ISD::VPERMV3 ||
N->getOpcode() == X86ISD::VPERMV);
});

// Depth threshold above which we can efficiently use variable mask shuffles.
int VariableCrossLaneShuffleDepth =
Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2;
int VariablePerLaneShuffleDepth =
Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2;
AllowVariableCrossLaneMask &=
(Depth >= VariableCrossLaneShuffleDepth) || HasVariableMask;
(Depth >= VariableCrossLaneShuffleDepth) || NumVariableMasks;
AllowVariablePerLaneMask &=
(Depth >= VariablePerLaneShuffleDepth) || HasVariableMask;
// VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a
(Depth >= VariablePerLaneShuffleDepth) || NumVariableMasks;
// VPERM2W/VPERM2B are 3 uops on Skylake and Icelake so we require a
// higher depth before combining them.
int BWIVPERMV3ShuffleDepth =
VariableCrossLaneShuffleDepth + 2 - NumVariableMasks;
bool AllowBWIVPERMV3 =
(Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
(Depth >= BWIVPERMV3ShuffleDepth || HasSlowVariableMask);

// If root was a VPERMV3 node, always allow a variable shuffle.
if (Root.getOpcode() == X86ISD::VPERMV3)
Expand Down
54 changes: 37 additions & 17 deletions llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,15 +225,25 @@ define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16
}

define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {
; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1
; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
; CHECK-FAST-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
; CHECK-FAST: # %bb.0:
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
; CHECK-FAST-NEXT: vptestnmw %xmm1, %xmm1, %k1
; CHECK-FAST-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-FAST-NEXT: vzeroupper
; CHECK-FAST-NEXT: retq
;
; CHECK-FAST-PERLANE-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
; CHECK-FAST-PERLANE: # %bb.0:
; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm2
; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3
; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]
; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm1, %xmm1, %k1
; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm2, %xmm0 {%k1}
; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
Expand All @@ -242,14 +252,24 @@ define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16>
}

define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {
; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1
; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
; CHECK-FAST-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
; CHECK-FAST: # %bb.0:
; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
; CHECK-FAST-NEXT: vptestnmw %xmm0, %xmm0, %k1
; CHECK-FAST-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-FAST-NEXT: vzeroupper
; CHECK-FAST-NEXT: retq
;
; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
; CHECK-FAST-PERLANE: # %bb.0:
; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm1
; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm0, %xmm0, %k1
; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} {z}
; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
Expand Down
10 changes: 4 additions & 6 deletions llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,8 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
; AVX512BW-FCP-NEXT: vpbroadcastw 4(%rdi), %xmm4
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,9,1,9,2,10,3,11]
; AVX512BW-FCP-NEXT: vpermw (%rdi), %ymm5, %ymm5
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpbroadcastw 20(%rdi), %xmm6
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm7
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
Expand All @@ -307,7 +307,6 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%r8)
; AVX512BW-FCP-NEXT: vmovd %xmm6, (%r9)
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rax)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i16_stride6_vf2:
Expand Down Expand Up @@ -347,8 +346,8 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 4(%rdi), %xmm4
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm5 = [3,9,1,9,2,10,3,11]
; AVX512DQ-BW-FCP-NEXT: vpermw (%rdi), %ymm5, %ymm5
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 20(%rdi), %xmm6
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm7
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
Expand All @@ -361,7 +360,6 @@ define void @load_i16_stride6_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%r8)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm6, (%r9)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <12 x i16>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <2 x i32> <i32 0, i32 6>
Expand Down
60 changes: 30 additions & 30 deletions llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,23 +321,23 @@ define void @load_i16_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpsrld $16, %xmm0, %xmm3
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[8,9,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm6
; AVX512BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm7
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [6,13,5,13,6,14,7,15]
; AVX512BW-FCP-NEXT: vpermw (%rdi), %ymm3, %ymm3
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm7
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm8
; AVX512BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm9
; AVX512BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
; AVX512BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX512BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm0
; AVX512BW-FCP-NEXT: vmovd %xmm2, (%rsi)
; AVX512BW-FCP-NEXT: vmovd %xmm4, (%rdx)
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%rcx)
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%r8)
; AVX512BW-FCP-NEXT: vmovd %xmm6, (%r9)
; AVX512BW-FCP-NEXT: vmovd %xmm1, (%r10)
; AVX512BW-FCP-NEXT: vmovd %xmm3, (%rax)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: vmovd %xmm7, (%rcx)
; AVX512BW-FCP-NEXT: vmovd %xmm5, (%r8)
; AVX512BW-FCP-NEXT: vmovd %xmm8, (%r9)
; AVX512BW-FCP-NEXT: vmovd %xmm3, (%r10)
; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rax)
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i16_stride7_vf2:
Expand Down Expand Up @@ -379,23 +379,23 @@ define void @load_i16_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpsrld $16, %xmm0, %xmm3
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[8,9,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm6
; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm7
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [6,13,5,13,6,14,7,15]
; AVX512DQ-BW-FCP-NEXT: vpermw (%rdi), %ymm3, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm7
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastw 8(%rdi), %xmm8
; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %xmm1, %xmm9
; AVX512DQ-BW-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
; AVX512DQ-BW-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm0
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%rsi)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm4, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%rcx)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%r8)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm6, (%r9)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%r10)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm7, (%rcx)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm5, (%r8)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm8, (%r9)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%r10)
; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rax)
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <14 x i16>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <14 x i16> %wide.vec, <14 x i16> poison, <2 x i32> <i32 0, i32 7>
Expand Down
12 changes: 6 additions & 6 deletions llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ define void @store_i16_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vmovdqa (%rdx), %xmm1
; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]
; AVX512BW-NEXT: vpermi2w %xmm1, %xmm0, %xmm2
; AVX512BW-NEXT: vmovdqa %xmm2, (%r8)
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]
; AVX512BW-NEXT: vmovdqa %xmm0, (%r8)
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: store_i16_stride4_vf2:
Expand All @@ -145,9 +145,9 @@ define void @store_i16_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %xmm1
; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]
; AVX512DQ-BW-NEXT: vpermi2w %xmm1, %xmm0, %xmm2
; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%r8)
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]
; AVX512DQ-BW-NEXT: vmovdqa %xmm0, (%r8)
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf2:
Expand Down
44 changes: 10 additions & 34 deletions llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,23 +495,11 @@ define <16 x i8> @shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20(
; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
; AVX1-NEXT: retq
;
; AVX2-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
; AVX2: # %bb.0:
; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
; AVX2-NEXT: retq
;
; AVX512VLBW-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
; AVX512VLBW: # %bb.0:
; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX512VLBW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
; AVX512VLBW-NEXT: retq
;
; AVX512VLVBMI-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
; AVX512VLVBMI: # %bb.0:
; AVX512VLVBMI-NEXT: vmovdqa {{.*#+}} xmm2 = [3,2,1,0,7,6,5,4,19,18,17,16,23,22,21,20]
; AVX512VLVBMI-NEXT: vpermt2b %xmm1, %xmm2, %xmm0
; AVX512VLVBMI-NEXT: retq
; AVX2OR512VL-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
; AVX2OR512VL: # %bb.0:
; AVX2OR512VL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX2OR512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,4,2,0,14,12,10,8,7,5,3,1,15,13,11,9]
; AVX2OR512VL-NEXT: retq
;
; XOP-LABEL: shuffle_v16i8_03_02_01_00_07_06_05_04_19_18_17_16_23_22_21_20:
; XOP: # %bb.0:
Expand Down Expand Up @@ -853,23 +841,11 @@ define <16 x i8> @shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(
; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX1-NEXT: retq
;
; AVX2-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
; AVX2: # %bb.0:
; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX2-NEXT: retq
;
; AVX512VLBW-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
; AVX512VLBW: # %bb.0:
; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX512VLBW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512VLBW-NEXT: retq
;
; AVX512VLVBMI-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
; AVX512VLVBMI: # %bb.0:
; AVX512VLVBMI-NEXT: vpbroadcastw {{.*#+}} xmm2 = [2,20,2,20,2,20,2,20,2,20,2,20,2,20,2,20]
; AVX512VLVBMI-NEXT: vpermt2b %xmm1, %xmm2, %xmm0
; AVX512VLVBMI-NEXT: retq
; AVX2OR512VL-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
; AVX2OR512VL: # %bb.0:
; AVX2OR512VL-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
; AVX2OR512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX2OR512VL-NEXT: retq
;
; XOP-LABEL: shuffle_v16i8_02_20_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
; XOP: # %bb.0:
Expand Down
Loading