@@ -2094,6 +2094,50 @@ static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) {
2094
2094
return true ;
2095
2095
}
2096
2096
2097
+ // Find constant NewC that has property:
2098
+ // shuffle(NewC, ShMask) = C
2099
+ // Returns nullptr if such a constant does not exist e.g. ShMask=<0,0> C=<1,2>
2100
+ //
2101
+ // A 1-to-1 mapping is not required. Example:
2102
+ // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <poison,5,6,poison>
2103
+ static Constant *unshuffleConstant (ArrayRef<int > ShMask, Constant *C,
2104
+ VectorType *NewCTy) {
2105
+ if (isa<ScalableVectorType>(NewCTy)) {
2106
+ Constant *Splat = C->getSplatValue ();
2107
+ if (!Splat)
2108
+ return nullptr ;
2109
+ return ConstantVector::getSplat (
2110
+ cast<VectorType>(C->getType ())->getElementCount (), Splat);
2111
+ }
2112
+
2113
+ if (cast<FixedVectorType>(NewCTy)->getNumElements () >
2114
+ cast<FixedVectorType>(C->getType ())->getNumElements ())
2115
+ return nullptr ;
2116
+
2117
+ unsigned NewCNumElts = cast<FixedVectorType>(NewCTy)->getNumElements ();
2118
+ PoisonValue *PoisonScalar = PoisonValue::get (C->getType ()->getScalarType ());
2119
+ SmallVector<Constant *, 16 > NewVecC (NewCNumElts, PoisonScalar);
2120
+ unsigned NumElts = cast<FixedVectorType>(C->getType ())->getNumElements ();
2121
+ for (unsigned I = 0 ; I < NumElts; ++I) {
2122
+ Constant *CElt = C->getAggregateElement (I);
2123
+ if (ShMask[I] >= 0 ) {
2124
+ assert (ShMask[I] < (int )NumElts && " Not expecting narrowing shuffle" );
2125
+ Constant *NewCElt = NewVecC[ShMask[I]];
2126
+ // Bail out if:
2127
+ // 1. The constant vector contains a constant expression.
2128
+ // 2. The shuffle needs an element of the constant vector that can't
2129
+ // be mapped to a new constant vector.
2130
+ // 3. This is a widening shuffle that copies elements of V1 into the
2131
+ // extended elements (extending with poison is allowed).
2132
+ if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) ||
2133
+ I >= NewCNumElts)
2134
+ return nullptr ;
2135
+ NewVecC[ShMask[I]] = CElt;
2136
+ }
2137
+ }
2138
+ return ConstantVector::get (NewVecC);
2139
+ }
2140
+
2097
2141
Instruction *InstCombinerImpl::foldVectorBinop (BinaryOperator &Inst) {
2098
2142
if (!isa<VectorType>(Inst.getType ()))
2099
2143
return nullptr ;
@@ -2213,50 +2257,15 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
2213
2257
// other binops, so they can be folded. It may also enable demanded elements
2214
2258
// transforms.
2215
2259
Constant *C;
2216
- auto *InstVTy = dyn_cast<FixedVectorType>(Inst.getType ());
2217
- if (InstVTy &&
2218
- match (&Inst, m_c_BinOp (m_OneUse (m_Shuffle (m_Value (V1), m_Poison (),
2260
+ if (match (&Inst, m_c_BinOp (m_OneUse (m_Shuffle (m_Value (V1), m_Poison (),
2219
2261
m_Mask (Mask))),
2220
- m_ImmConstant (C))) &&
2221
- cast<FixedVectorType>(V1->getType ())->getNumElements () <=
2222
- InstVTy->getNumElements ()) {
2223
- assert (InstVTy->getScalarType () == V1->getType ()->getScalarType () &&
2262
+ m_ImmConstant (C)))) {
2263
+ assert (Inst.getType ()->getScalarType () == V1->getType ()->getScalarType () &&
2224
2264
" Shuffle should not change scalar type" );
2225
2265
2226
- // Find constant NewC that has property:
2227
- // shuffle(NewC, ShMask) = C
2228
- // If such constant does not exist (example: ShMask=<0,0> and C=<1,2>)
2229
- // reorder is not possible. A 1-to-1 mapping is not required. Example:
2230
- // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
2231
2266
bool ConstOp1 = isa<Constant>(RHS);
2232
- ArrayRef<int > ShMask = Mask;
2233
- unsigned SrcVecNumElts =
2234
- cast<FixedVectorType>(V1->getType ())->getNumElements ();
2235
- PoisonValue *PoisonScalar = PoisonValue::get (C->getType ()->getScalarType ());
2236
- SmallVector<Constant *, 16 > NewVecC (SrcVecNumElts, PoisonScalar);
2237
- bool MayChange = true ;
2238
- unsigned NumElts = InstVTy->getNumElements ();
2239
- for (unsigned I = 0 ; I < NumElts; ++I) {
2240
- Constant *CElt = C->getAggregateElement (I);
2241
- if (ShMask[I] >= 0 ) {
2242
- assert (ShMask[I] < (int )NumElts && " Not expecting narrowing shuffle" );
2243
- Constant *NewCElt = NewVecC[ShMask[I]];
2244
- // Bail out if:
2245
- // 1. The constant vector contains a constant expression.
2246
- // 2. The shuffle needs an element of the constant vector that can't
2247
- // be mapped to a new constant vector.
2248
- // 3. This is a widening shuffle that copies elements of V1 into the
2249
- // extended elements (extending with poison is allowed).
2250
- if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) ||
2251
- I >= SrcVecNumElts) {
2252
- MayChange = false ;
2253
- break ;
2254
- }
2255
- NewVecC[ShMask[I]] = CElt;
2256
- }
2257
- }
2258
- if (MayChange) {
2259
- Constant *NewC = ConstantVector::get (NewVecC);
2267
+ if (Constant *NewC =
2268
+ unshuffleConstant (Mask, C, cast<VectorType>(V1->getType ()))) {
2260
2269
// It may not be safe to execute a binop on a vector with poison elements
2261
2270
// because the entire instruction can be folded to undef or create poison
2262
2271
// that did not exist in the original code.
@@ -2272,27 +2281,6 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
2272
2281
}
2273
2282
}
2274
2283
2275
- // Similar to the combine above, but handles the case for scalable vectors
2276
- // where both shuffle(V1, 0) and C are splats.
2277
- //
2278
- // Op(shuffle(V1, 0), (splat C)) -> shuffle(Op(V1, (splat C)), 0)
2279
- if (isa<ScalableVectorType>(Inst.getType ()) &&
2280
- match (&Inst, m_c_BinOp (m_OneUse (m_Shuffle (m_Value (V1), m_Poison (),
2281
- m_ZeroMask ())),
2282
- m_ImmConstant (C)))) {
2283
- if (Constant *Splat = C->getSplatValue ()) {
2284
- bool ConstOp1 = isa<Constant>(RHS);
2285
- VectorType *V1Ty = cast<VectorType>(V1->getType ());
2286
- Constant *NewC = ConstantVector::getSplat (V1Ty->getElementCount (), Splat);
2287
-
2288
- Value *NewLHS = ConstOp1 ? V1 : NewC;
2289
- Value *NewRHS = ConstOp1 ? NewC : V1;
2290
- VectorType *VTy = cast<VectorType>(Inst.getType ());
2291
- SmallVector<int > Mask (VTy->getElementCount ().getKnownMinValue (), 0 );
2292
- return createBinOpShuffle (NewLHS, NewRHS, Mask);
2293
- }
2294
- }
2295
-
2296
2284
// Try to reassociate to sink a splat shuffle after a binary operation.
2297
2285
if (Inst.isAssociative () && Inst.isCommutative ()) {
2298
2286
// Canonicalize shuffle operand as LHS.
0 commit comments