Skip to content

[X86] combineX86ShuffleChain - always combine to a new VPERMV node if the root shuffle was a VPERMV node #128183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40090,8 +40090,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
bool AllowBWIVPERMV3 =
(Depth >= BWIVPERMV3ShuffleDepth || HasSlowVariableMask);

// If root was a VPERMV3 node, always allow a variable shuffle.
if (Root.getOpcode() == X86ISD::VPERMV3)
// If root was a VPERMV/VPERMV3 node, always allow a variable shuffle.
if ((UnaryShuffle && Root.getOpcode() == X86ISD::VPERMV) ||
Root.getOpcode() == X86ISD::VPERMV3)
AllowVariableCrossLaneMask = AllowVariablePerLaneMask = true;

bool MaskContainsZeros = isAnyZero(Mask);
Expand Down
57 changes: 27 additions & 30 deletions llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
Original file line number Diff line number Diff line change
Expand Up @@ -588,20 +588,19 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero
; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [5,7,1,3,7,0,0,0]
; AVX2-FCP-NEXT: vpermd %ymm4, %ymm1, %ymm1
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5],zero,zero,zero,zero,zero,zero,ymm1[10,11,14,15,2,3,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,3,5,7,1,3,5,7]
; AVX2-FCP-NEXT: # ymm5 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermd %ymm0, %ymm5, %ymm5
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,ymm5[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm5[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FCP-NEXT: vpor %ymm5, %ymm1, %ymm1
; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [1,5,3,7,1,5,3,7]
; AVX2-FCP-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [5,7,1,3,7,0,0,0]
; AVX2-FCP-NEXT: vpermd %ymm4, %ymm5, %ymm5
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,4,5],zero,zero,zero,zero,zero,zero,ymm5[10,11,14,15,2,3,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FCP-NEXT: vpor %ymm1, %ymm5, %ymm1
; AVX2-FCP-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,ymm2[0,1,8,9,4,5,6,7,4,5],zero,zero,ymm2[26,27],zero,zero,zero,zero,ymm2[24,25,20,21,22,23,20,21,28,29]
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[0,1,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,3],zero,zero,ymm4[18,19,26,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FCP-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,4,2,6,0,4,2,6]
; AVX2-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm0
; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,0,1,4,5,8,9,u,u,u,u,u,u,u,u,18,19,22,23,26,27,u,u,u,u]
Expand Down Expand Up @@ -670,17 +669,16 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm4
; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero
; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
; AVX512-FCP-NEXT: vpermi2q %ymm5, %ymm0, %ymm1
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[18,19,22,23,26,27],zero,zero,zero,zero
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,3,5,7,1,3,5,7]
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [0,4,2,6,0,4,2,6]
; AVX512-FCP-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[18,19,22,23,26,27],zero,zero,zero,zero
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,5,3,7,1,5,3,7]
; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm5, %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vpermd %ymm0, %ymm5, %ymm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[0,1,8,9],zero,zero,zero,zero,ymm4[u,u,u,u,u,u,2,3],zero,zero,ymm4[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,ymm2[0,1,8,9,u,u,u,u,u,u],zero,zero,ymm2[26,27],zero,zero,zero,zero,ymm2[u,u,u,u,u,u,20,21,28,29]
Expand Down Expand Up @@ -753,17 +751,16 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm4
; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
; AVX512DQ-FCP-NEXT: vpermi2q %ymm5, %ymm0, %ymm1
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[18,19,22,23,26,27],zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,3,5,7,1,3,5,7]
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [0,4,2,6,0,4,2,6]
; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[18,19,22,23,26,27],zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,5,3,7,1,5,3,7]
; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm5, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm5, %ymm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1,4,5,8,9],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[18,19,22,23,26,27],zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[0,1,8,9],zero,zero,zero,zero,ymm4[u,u,u,u,u,u,2,3],zero,zero,ymm4[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,ymm2[0,1,8,9,u,u,u,u,u,u],zero,zero,ymm2[26,27],zero,zero,zero,zero,ymm2[u,u,u,u,u,u,20,21,28,29]
Expand Down
61 changes: 30 additions & 31 deletions llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,17 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX2-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,2,2,1,4,6,6,5]
; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,2,1]
; AVX2-NEXT: vshufps {{.*#+}} ymm3 = ymm0[0,2,2,3,4,6,6,7]
; AVX2-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,2]
; AVX2-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6],ymm1[7]
; AVX2-NEXT: vmovaps {{.*#+}} xmm3 = [3,5,7,u]
; AVX2-NEXT: vpermps %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [3,5,0,1,3,5,0,1]
; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm1 = [0,4,2,6,0,4,2,6]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpermps %ymm0, %ymm1, %ymm1
; AVX2-NEXT: vshufps {{.*#+}} ymm3 = ymm2[0,2,2,1,4,6,6,5]
; AVX2-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,2,2,1]
; AVX2-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6],ymm3[7]
; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [5,3,0,1,5,3,0,1]
; AVX2-NEXT: # ymm3 = mem[0,1,0,1]
; AVX2-NEXT: vpermps %ymm0, %ymm3, %ymm0
; AVX2-NEXT: vmovaps {{.*#+}} xmm3 = [3,5,7,u]
; AVX2-NEXT: vpermps %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3]
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovlps %xmm0, 48(%rax)
Expand All @@ -130,17 +130,17 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX2-FP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-FP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-FP-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-FP-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,2,2,1,4,6,6,5]
; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,2,1]
; AVX2-FP-NEXT: vshufps {{.*#+}} ymm3 = ymm0[0,2,2,3,4,6,6,7]
; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,2]
; AVX2-FP-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6],ymm1[7]
; AVX2-FP-NEXT: vmovaps {{.*#+}} xmm3 = [3,5,7,u]
; AVX2-FP-NEXT: vpermps %ymm2, %ymm3, %ymm2
; AVX2-FP-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [3,5,0,1,3,5,0,1]
; AVX2-FP-NEXT: vbroadcastf128 {{.*#+}} ymm1 = [0,4,2,6,0,4,2,6]
; AVX2-FP-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-FP-NEXT: vpermps %ymm0, %ymm1, %ymm1
; AVX2-FP-NEXT: vshufps {{.*#+}} ymm3 = ymm2[0,2,2,1,4,6,6,5]
; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,2,2,1]
; AVX2-FP-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6],ymm3[7]
; AVX2-FP-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [5,3,0,1,5,3,0,1]
; AVX2-FP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX2-FP-NEXT: vpermps %ymm0, %ymm3, %ymm0
; AVX2-FP-NEXT: vmovaps {{.*#+}} xmm3 = [3,5,7,u]
; AVX2-FP-NEXT: vpermps %ymm2, %ymm3, %ymm2
; AVX2-FP-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3]
; AVX2-FP-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-FP-NEXT: vmovlps %xmm0, 48(%rax)
Expand All @@ -165,23 +165,22 @@ define void @store_i32_stride7_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX2-FCP-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-FCP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-FCP-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-FCP-NEXT: vmovaps {{.*#+}} xmm1 = [3,5,7,u]
; AVX2-FCP-NEXT: vpermps %ymm2, %ymm1, %ymm1
; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm3 = [3,5,0,1,3,5,0,1]
; AVX2-FCP-NEXT: # ymm3 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermps %ymm0, %ymm3, %ymm3
; AVX2-FCP-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3]
; AVX2-FCP-NEXT: vmovaps {{.*#+}} ymm4 = [0,2,4,6,u,u,u,1]
; AVX2-FCP-NEXT: vpermps %ymm2, %ymm4, %ymm2
; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm4 = [0,2,4,0,0,2,4,0]
; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm1 = [5,3,0,1,5,3,0,1]
; AVX2-FCP-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm1
; AVX2-FCP-NEXT: vmovaps {{.*#+}} xmm3 = [3,5,7,u]
; AVX2-FCP-NEXT: vpermps %ymm2, %ymm3, %ymm3
; AVX2-FCP-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1,2],xmm1[3]
; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm4 = [0,4,2,0,0,4,2,0]
; AVX2-FCP-NEXT: # ymm4 = mem[0,1,0,1]
; AVX2-FCP-NEXT: vpermps %ymm0, %ymm4, %ymm0
; AVX2-FCP-NEXT: vmovaps {{.*#+}} ymm4 = [0,2,4,6,u,u,u,1]
; AVX2-FCP-NEXT: vpermps %ymm2, %ymm4, %ymm2
; AVX2-FCP-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6],ymm2[7]
; AVX2-FCP-NEXT: vextractf128 $1, %ymm3, %xmm2
; AVX2-FCP-NEXT: vmovlps %xmm2, 48(%rax)
; AVX2-FCP-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX2-FCP-NEXT: vmovlps %xmm1, 48(%rax)
; AVX2-FCP-NEXT: vmovaps %ymm0, (%rax)
; AVX2-FCP-NEXT: vmovaps %xmm1, 32(%rax)
; AVX2-FCP-NEXT: vmovaps %xmm3, 32(%rax)
; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
Expand Down
Loading