@@ -10005,8 +10005,8 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
10005
10005
10006
10006
// X86 has dedicated unpack instructions that can handle specific blend
10007
10007
// operations: UNPCKH and UNPCKL.
10008
- static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
10009
- ArrayRef<int> Mask, SDValue V1, SDValue V2 ,
10008
+ static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1,
10009
+ SDValue V2, ArrayRef<int> Mask,
10010
10010
SelectionDAG &DAG) {
10011
10011
SmallVector<int, 8> Unpckl;
10012
10012
createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
@@ -10032,9 +10032,9 @@ static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
10032
10032
10033
10033
/// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
10034
10034
/// followed by unpack 256-bit.
10035
- static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
10036
- ArrayRef<int> Mask, SDValue V1 ,
10037
- SDValue V2, SelectionDAG &DAG) {
10035
+ static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1,
10036
+ SDValue V2, ArrayRef<int> Mask,
10037
+ SelectionDAG &DAG) {
10038
10038
SmallVector<int, 32> Unpckl, Unpckh;
10039
10039
createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
10040
10040
createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
@@ -10432,9 +10432,10 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
10432
10432
return false;
10433
10433
}
10434
10434
10435
- static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
10436
- SDValue V1, SDValue V2, SelectionDAG &DAG,
10437
- const X86Subtarget &Subtarget) {
10435
+ static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1,
10436
+ SDValue V2, ArrayRef<int> Mask,
10437
+ const X86Subtarget &Subtarget,
10438
+ SelectionDAG &DAG) {
10438
10439
MVT PackVT;
10439
10440
unsigned PackOpcode;
10440
10441
unsigned SizeBits = VT.getSizeInBits();
@@ -12861,7 +12862,7 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12861
12862
return Blend;
12862
12863
12863
12864
// Use dedicated unpack instructions for masks that match their pattern.
12864
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
12865
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask , DAG))
12865
12866
return V;
12866
12867
12867
12868
unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
@@ -12938,7 +12939,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12938
12939
return Blend;
12939
12940
12940
12941
// Use dedicated unpack instructions for masks that match their pattern.
12941
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
12942
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask , DAG))
12942
12943
return V;
12943
12944
12944
12945
// Try to use byte rotation instructions.
@@ -13160,7 +13161,7 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13160
13161
}
13161
13162
13162
13163
// Use dedicated unpack instructions for masks that match their pattern.
13163
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13164
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask , DAG))
13164
13165
return V;
13165
13166
13166
13167
// Otherwise fall back to a SHUFPS lowering strategy.
@@ -13253,7 +13254,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13253
13254
return Masked;
13254
13255
13255
13256
// Use dedicated unpack instructions for masks that match their pattern.
13256
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
13257
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask , DAG))
13257
13258
return V;
13258
13259
13259
13260
// Try to use byte rotation instructions.
@@ -13908,12 +13909,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13908
13909
return Rotate;
13909
13910
13910
13911
// Use dedicated unpack instructions for masks that match their pattern.
13911
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13912
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask , DAG))
13912
13913
return V;
13913
13914
13914
13915
// Use dedicated pack instructions for masks that match their pattern.
13915
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13916
- Subtarget))
13916
+ if (SDValue V =
13917
+ lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG ))
13917
13918
return V;
13918
13919
13919
13920
// Try to use byte rotation instructions.
@@ -13962,12 +13963,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13962
13963
return Masked;
13963
13964
13964
13965
// Use dedicated unpack instructions for masks that match their pattern.
13965
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13966
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask , DAG))
13966
13967
return V;
13967
13968
13968
13969
// Use dedicated pack instructions for masks that match their pattern.
13969
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13970
- Subtarget))
13970
+ if (SDValue V =
13971
+ lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG ))
13971
13972
return V;
13972
13973
13973
13974
// Try to use lower using a truncation.
@@ -14168,8 +14169,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14168
14169
return Rotate;
14169
14170
14170
14171
// Use dedicated pack instructions for masks that match their pattern.
14171
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14172
- Subtarget))
14172
+ if (SDValue V =
14173
+ lowerShuffleWithPACK(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG ))
14173
14174
return V;
14174
14175
14175
14176
// Try to use a zext lowering.
@@ -14206,7 +14207,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14206
14207
Subtarget, DAG))
14207
14208
return Rotate;
14208
14209
14209
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14210
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask , DAG))
14210
14211
return V;
14211
14212
14212
14213
// Check whether we can widen this to an i16 shuffle by duplicating bytes.
@@ -14310,7 +14311,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14310
14311
return Masked;
14311
14312
14312
14313
// Use dedicated unpack instructions for masks that match their pattern.
14313
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14314
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask , DAG))
14314
14315
return V;
14315
14316
14316
14317
// Try to use byte shift instructions to mask.
@@ -15882,7 +15883,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15882
15883
}
15883
15884
15884
15885
// Use dedicated unpack instructions for masks that match their pattern.
15885
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
15886
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask , DAG))
15886
15887
return V;
15887
15888
15888
15889
if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
@@ -16021,7 +16022,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16021
16022
return Rotate;
16022
16023
16023
16024
// Use dedicated unpack instructions for masks that match their pattern.
16024
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
16025
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask , DAG))
16025
16026
return V;
16026
16027
16027
16028
bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
@@ -16110,7 +16111,7 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16110
16111
getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16111
16112
16112
16113
// Use dedicated unpack instructions for masks that match their pattern.
16113
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
16114
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask , DAG))
16114
16115
return V;
16115
16116
16116
16117
// Otherwise, fall back to a SHUFPS sequence. Here it is important that we
@@ -16251,7 +16252,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16251
16252
getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16252
16253
16253
16254
// Use dedicated unpack instructions for masks that match their pattern.
16254
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16255
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask , DAG))
16255
16256
return V;
16256
16257
}
16257
16258
@@ -16291,7 +16292,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16291
16292
if (V2.isUndef()) {
16292
16293
// Try to produce a fixed cross-128-bit lane permute followed by unpack
16293
16294
// because that should be faster than the variable permute alternatives.
16294
- if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
16295
+ if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask , DAG))
16295
16296
return V;
16296
16297
16297
16298
// If the shuffle patterns aren't repeated but it's a single input, directly
@@ -16352,12 +16353,12 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16352
16353
return Blend;
16353
16354
16354
16355
// Use dedicated unpack instructions for masks that match their pattern.
16355
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16356
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask , DAG))
16356
16357
return V;
16357
16358
16358
16359
// Use dedicated pack instructions for masks that match their pattern.
16359
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16360
- Subtarget))
16360
+ if (SDValue V =
16361
+ lowerShuffleWithPACK(DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG ))
16361
16362
return V;
16362
16363
16363
16364
// Try to use lower using a truncation.
@@ -16390,7 +16391,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16390
16391
16391
16392
// Try to produce a fixed cross-128-bit lane permute followed by unpack
16392
16393
// because that should be faster than the variable permute alternatives.
16393
- if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
16394
+ if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask , DAG))
16394
16395
return V;
16395
16396
16396
16397
// There are no generalized cross-lane shuffle operations available on i16
@@ -16475,12 +16476,12 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16475
16476
return Blend;
16476
16477
16477
16478
// Use dedicated unpack instructions for masks that match their pattern.
16478
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16479
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask , DAG))
16479
16480
return V;
16480
16481
16481
16482
// Use dedicated pack instructions for masks that match their pattern.
16482
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16483
- Subtarget))
16483
+ if (SDValue V =
16484
+ lowerShuffleWithPACK(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG ))
16484
16485
return V;
16485
16486
16486
16487
// Try to use lower using a truncation.
@@ -16516,7 +16517,7 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16516
16517
if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
16517
16518
// Try to produce a fixed cross-128-bit lane permute followed by unpack
16518
16519
// because that should be faster than the variable permute alternatives.
16519
- if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
16520
+ if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask , DAG))
16520
16521
return V;
16521
16522
16522
16523
if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
@@ -16783,7 +16784,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16783
16784
V2, Subtarget, DAG))
16784
16785
return Shuf128;
16785
16786
16786
- if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
16787
+ if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask , DAG))
16787
16788
return Unpck;
16788
16789
16789
16790
// Check if the blend happens to exactly fit that of SHUFPD.
@@ -16828,7 +16829,7 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16828
16829
getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16829
16830
16830
16831
// Use dedicated unpack instructions for masks that match their pattern.
16831
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
16832
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask , DAG))
16832
16833
return V;
16833
16834
16834
16835
if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
@@ -16927,7 +16928,7 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16927
16928
Subtarget, DAG))
16928
16929
return Rotate;
16929
16930
16930
- if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
16931
+ if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask , DAG))
16931
16932
return Unpck;
16932
16933
16933
16934
// If we have AVX512F support, we can use VEXPAND.
@@ -16985,7 +16986,7 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16985
16986
getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16986
16987
16987
16988
// Use dedicated unpack instructions for masks that match their pattern.
16988
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
16989
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask , DAG))
16989
16990
return V;
16990
16991
}
16991
16992
@@ -17057,12 +17058,12 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17057
17058
return ZExt;
17058
17059
17059
17060
// Use dedicated unpack instructions for masks that match their pattern.
17060
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
17061
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask , DAG))
17061
17062
return V;
17062
17063
17063
17064
// Use dedicated pack instructions for masks that match their pattern.
17064
17065
if (SDValue V =
17065
- lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG , Subtarget))
17066
+ lowerShuffleWithPACK(DL, MVT::v32i16, V1, V2, Mask , Subtarget, DAG ))
17066
17067
return V;
17067
17068
17068
17069
// Try to use shift instructions.
@@ -17121,12 +17122,12 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17121
17122
return ZExt;
17122
17123
17123
17124
// Use dedicated unpack instructions for masks that match their pattern.
17124
- if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
17125
+ if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask , DAG))
17125
17126
return V;
17126
17127
17127
17128
// Use dedicated pack instructions for masks that match their pattern.
17128
- if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
17129
- Subtarget))
17129
+ if (SDValue V =
17130
+ lowerShuffleWithPACK(DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG ))
17130
17131
return V;
17131
17132
17132
17133
// Try to use shift instructions.
0 commit comments