@@ -649,14 +649,15 @@ namespace {
649
649
bool DemandHighBits = true);
650
650
SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
651
651
SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
652
- SDValue InnerPos, SDValue InnerNeg, bool HasPos ,
653
- unsigned PosOpcode , unsigned NegOpcode ,
654
- const SDLoc &DL);
652
+ SDValue InnerPos, SDValue InnerNeg, bool FromAdd ,
653
+ bool HasPos , unsigned PosOpcode ,
654
+ unsigned NegOpcode, const SDLoc &DL);
655
655
SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
656
- SDValue InnerPos, SDValue InnerNeg, bool HasPos,
657
- unsigned PosOpcode, unsigned NegOpcode,
658
- const SDLoc &DL);
659
- SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
656
+ SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
657
+ bool HasPos, unsigned PosOpcode,
658
+ unsigned NegOpcode, const SDLoc &DL);
659
+ SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
660
+ bool FromAdd);
660
661
SDValue MatchLoadCombine(SDNode *N);
661
662
SDValue mergeTruncStores(StoreSDNode *N);
662
663
SDValue reduceLoadWidth(SDNode *N);
@@ -2986,6 +2987,9 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
2986
2987
if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
2987
2988
return V;
2988
2989
2990
+ if (SDValue V = MatchRotate(N0, N1, SDLoc(N), /*FromAdd=*/true))
2991
+ return V;
2992
+
2989
2993
// Try to match AVGFLOOR fixedwidth pattern
2990
2994
if (SDValue V = foldAddToAvg(N, DL))
2991
2995
return V;
@@ -8175,7 +8179,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
8175
8179
return V;
8176
8180
8177
8181
// See if this is some rotate idiom.
8178
- if (SDValue Rot = MatchRotate(N0, N1, DL))
8182
+ if (SDValue Rot = MatchRotate(N0, N1, DL, /*FromAdd=*/false ))
8179
8183
return Rot;
8180
8184
8181
8185
if (SDValue Load = MatchLoadCombine(N))
@@ -8364,7 +8368,7 @@ static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
8364
8368
// The IsRotate flag should be set when the LHS of both shifts is the same.
8365
8369
// Otherwise if matching a general funnel shift, it should be clear.
8366
8370
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
8367
- SelectionDAG &DAG, bool IsRotate) {
8371
+ SelectionDAG &DAG, bool IsRotate, bool FromAdd ) {
8368
8372
const auto &TLI = DAG.getTargetLoweringInfo();
8369
8373
// If EltSize is a power of 2 then:
8370
8374
//
@@ -8403,7 +8407,7 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
8403
8407
// NOTE: We can only do this when matching operations which won't modify the
8404
8408
// least Log2(EltSize) significant bits and not a general funnel shift.
8405
8409
unsigned MaskLoBits = 0;
8406
- if (IsRotate && isPowerOf2_64(EltSize)) {
8410
+ if (IsRotate && !FromAdd && isPowerOf2_64(EltSize)) {
8407
8411
unsigned Bits = Log2_64(EltSize);
8408
8412
unsigned NegBits = Neg.getScalarValueSizeInBits();
8409
8413
if (NegBits >= Bits) {
@@ -8486,9 +8490,9 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
8486
8490
// Neg with outer conversions stripped away.
8487
8491
SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
8488
8492
SDValue Neg, SDValue InnerPos,
8489
- SDValue InnerNeg, bool HasPos ,
8490
- unsigned PosOpcode , unsigned NegOpcode ,
8491
- const SDLoc &DL) {
8493
+ SDValue InnerNeg, bool FromAdd ,
8494
+ bool HasPos , unsigned PosOpcode ,
8495
+ unsigned NegOpcode, const SDLoc &DL) {
8492
8496
// fold (or (shl x, (*ext y)),
8493
8497
// (srl x, (*ext (sub 32, y)))) ->
8494
8498
// (rotl x, y) or (rotr x, (sub 32, y))
@@ -8498,10 +8502,9 @@ SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
8498
8502
// (rotr x, y) or (rotl x, (sub 32, y))
8499
8503
EVT VT = Shifted.getValueType();
8500
8504
if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG,
8501
- /*IsRotate*/ true)) {
8505
+ /*IsRotate*/ true, FromAdd))
8502
8506
return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
8503
8507
HasPos ? Pos : Neg);
8504
- }
8505
8508
8506
8509
return SDValue();
8507
8510
}
@@ -8514,9 +8517,9 @@ SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
8514
8517
// TODO: Merge with MatchRotatePosNeg.
8515
8518
SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
8516
8519
SDValue Neg, SDValue InnerPos,
8517
- SDValue InnerNeg, bool HasPos ,
8518
- unsigned PosOpcode , unsigned NegOpcode ,
8519
- const SDLoc &DL) {
8520
+ SDValue InnerNeg, bool FromAdd ,
8521
+ bool HasPos , unsigned PosOpcode ,
8522
+ unsigned NegOpcode, const SDLoc &DL) {
8520
8523
EVT VT = N0.getValueType();
8521
8524
unsigned EltBits = VT.getScalarSizeInBits();
8522
8525
@@ -8527,10 +8530,10 @@ SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
8527
8530
// fold (or (shl x0, (*ext (sub 32, y))),
8528
8531
// (srl x1, (*ext y))) ->
8529
8532
// (fshr x0, x1, y) or (fshl x0, x1, (sub 32, y))
8530
- if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1)) {
8533
+ if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1,
8534
+ FromAdd))
8531
8535
return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1,
8532
8536
HasPos ? Pos : Neg);
8533
- }
8534
8537
8535
8538
// Matching the shift+xor cases, we can't easily use the xor'd shift amount
8536
8539
// so for now just use the PosOpcode case if its legal.
@@ -8569,11 +8572,12 @@ SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
8569
8572
return SDValue();
8570
8573
}
8571
8574
8572
- // MatchRotate - Handle an 'or' of two operands. If this is one of the many
8573
- // idioms for rotate, and if the target supports rotation instructions, generate
8574
- // a rot[lr]. This also matches funnel shift patterns, similar to rotation but
8575
- // with different shifted sources.
8576
- SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
8575
+ // MatchRotate - Handle an 'or' or 'add' of two operands. If this is one of the
8576
+ // many idioms for rotate, and if the target supports rotation instructions,
8577
+ // generate a rot[lr]. This also matches funnel shift patterns, similar to
8578
+ // rotation but with different shifted sources.
8579
+ SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
8580
+ bool FromAdd) {
8577
8581
EVT VT = LHS.getValueType();
8578
8582
8579
8583
// The target must have at least one rotate/funnel flavor.
@@ -8600,9 +8604,9 @@ SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
8600
8604
if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
8601
8605
LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
8602
8606
assert(LHS.getValueType() == RHS.getValueType());
8603
- if (SDValue Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
8607
+ if (SDValue Rot =
8608
+ MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL, FromAdd))
8604
8609
return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
8605
- }
8606
8610
}
8607
8611
8608
8612
// Match "(X shl/srl V1) & V2" where V2 may not be present.
@@ -8782,29 +8786,25 @@ SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
8782
8786
}
8783
8787
8784
8788
if (IsRotate && (HasROTL || HasROTR)) {
8785
- SDValue TryL =
8786
- MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
8787
- RExtOp0, HasROTL, ISD::ROTL, ISD::ROTR, DL);
8788
- if (TryL)
8789
+ if (SDValue TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
8790
+ LExtOp0, RExtOp0, FromAdd, HasROTL,
8791
+ ISD::ROTL, ISD::ROTR, DL))
8789
8792
return TryL;
8790
8793
8791
- SDValue TryR =
8792
- MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
8793
- LExtOp0, HasROTR, ISD::ROTR, ISD::ROTL, DL);
8794
- if (TryR)
8794
+ if (SDValue TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
8795
+ RExtOp0, LExtOp0, FromAdd, HasROTR,
8796
+ ISD::ROTR, ISD::ROTL, DL))
8795
8797
return TryR;
8796
8798
}
8797
8799
8798
- SDValue TryL =
8799
- MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
8800
- LExtOp0, RExtOp0, HasFSHL, ISD::FSHL, ISD::FSHR, DL);
8801
- if (TryL)
8800
+ if (SDValue TryL = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt,
8801
+ RHSShiftAmt, LExtOp0, RExtOp0, FromAdd,
8802
+ HasFSHL, ISD::FSHL, ISD::FSHR, DL))
8802
8803
return TryL;
8803
8804
8804
- SDValue TryR =
8805
- MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
8806
- RExtOp0, LExtOp0, HasFSHR, ISD::FSHR, ISD::FSHL, DL);
8807
- if (TryR)
8805
+ if (SDValue TryR = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt,
8806
+ LHSShiftAmt, RExtOp0, LExtOp0, FromAdd,
8807
+ HasFSHR, ISD::FSHR, ISD::FSHL, DL))
8808
8808
return TryR;
8809
8809
8810
8810
return SDValue();
0 commit comments