rebase, fix tests, improve comment

Author: huntergr-arm

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -14649,7 +14649,7 @@ bool AArch64TargetLowering::shouldSinkOperands(
     }
   }
 
-  // Sink vscales close to uses for better isel
+  // Sink vscales closer to uses for better isel
   switch (I->getOpcode()) {
   case Instruction::GetElementPtr:
   case Instruction::Add:

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll

@@ -20,24 +20,25 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    whilelo p1.d, xzr, x9
 ; CHECK-NEXT:    cntd x10
 ; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:    mov x11, x10
+; CHECK-NEXT:    rdvl x11, #2
+; CHECK-NEXT:    mov x12, x10
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
-; CHECK-NEXT:    add x12, x0, x8
-; CHECK-NEXT:    add x13, x1, x8
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
 ; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
-; CHECK-NEXT:    whilelo p1.d, x11, x9
-; CHECK-NEXT:    addvl x8, x8, #2
-; CHECK-NEXT:    add x11, x11, x10
-; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x12, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x12]
-; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x13]
+; CHECK-NEXT:    whilelo p1.d, x12, x9
+; CHECK-NEXT:    add x8, x8, x11
+; CHECK-NEXT:    add x12, x12, x10
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
@@ -120,25 +121,26 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr %
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    mov x9, xzr
 ; CHECK-NEXT:    and x11, x11, x12
+; CHECK-NEXT:    rdvl x12, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB1_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ld1w { z2.d }, p0/z, [x2, x9, lsl #2]
-; CHECK-NEXT:    add x12, x0, x8
-; CHECK-NEXT:    add x13, x1, x8
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
 ; CHECK-NEXT:    add x9, x9, x10
-; CHECK-NEXT:    addvl x8, x8, #2
+; CHECK-NEXT:    add x8, x8, x12
 ; CHECK-NEXT:    cmpne p1.d, p0/z, z2.d, #0
 ; CHECK-NEXT:    cmp x11, x9
 ; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    zip1 p1.d, p1.d, p1.d
-; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x12, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x12]
-; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x13]
+; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p1/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p1/z, [x14]
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
@@ -222,25 +224,26 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    mov x9, xzr
 ; CHECK-NEXT:    cntd x11
+; CHECK-NEXT:    rdvl x12, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB2_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ld1w { z2.d }, p1/z, [x2, x9, lsl #2]
-; CHECK-NEXT:    add x12, x0, x8
-; CHECK-NEXT:    add x13, x1, x8
+; CHECK-NEXT:    add x13, x0, x8
+; CHECK-NEXT:    add x14, x1, x8
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
 ; CHECK-NEXT:    add x9, x9, x11
-; CHECK-NEXT:    addvl x8, x8, #2
+; CHECK-NEXT:    add x8, x8, x12
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z2.d, #0
 ; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
 ; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    whilelo p1.d, x9, x10
-; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x12, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x12]
-; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll

@@ -22,18 +22,19 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov w10, #100 // =0x64
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    and x10, x9, x10
+; CHECK-NEXT:    rdvl x11, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x11, x0, x8
-; CHECK-NEXT:    add x12, x1, x8
+; CHECK-NEXT:    add x12, x0, x8
+; CHECK-NEXT:    add x13, x1, x8
 ; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x11, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x12, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
 ; CHECK-NEXT:    adds x10, x10, x9
-; CHECK-NEXT:    addvl x8, x8, #2
+; CHECK-NEXT:    add x8, x8, x11
 ; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #90
@@ -110,21 +111,22 @@ define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    neg x9, x9
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    and x10, x9, x10
+; CHECK-NEXT:    rdvl x11, #2
 ; CHECK-NEXT:    sel z3.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    mov z1.d, p0/m, z2.d
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z3.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z3.d
 ; CHECK-NEXT:  .LBB1_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x11, x0, x8
-; CHECK-NEXT:    add x12, x1, x8
+; CHECK-NEXT:    add x12, x0, x8
+; CHECK-NEXT:    add x13, x1, x8
 ; CHECK-NEXT:    ld1b { z2.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x11, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x12, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x12, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
 ; CHECK-NEXT:    adds x10, x10, x9
-; CHECK-NEXT:    addvl x8, x8, #2
+; CHECK-NEXT:    add x8, x8, x11
 ; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z0.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z1.d, p0/m, z4.d, z2.d, #90
@@ -193,30 +195,32 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    neg x9, x9
 ; CHECK-NEXT:    mov w10, #1000 // =0x3e8
+; CHECK-NEXT:    rdvl x12, #2
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    and x10, x9, x10
-; CHECK-NEXT:    addvl x11, x1, #2
 ; CHECK-NEXT:    zip2 z0.d, z1.d, z1.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
-; CHECK-NEXT:    addvl x12, x0, #2
+; CHECK-NEXT:    add x11, x1, x12
+; CHECK-NEXT:    add x12, x0, x12
+; CHECK-NEXT:    rdvl x13, #4
 ; CHECK-NEXT:    mov z2.d, z1.d
 ; CHECK-NEXT:    mov z3.d, z0.d
 ; CHECK-NEXT:  .LBB2_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    add x13, x0, x8
-; CHECK-NEXT:    add x14, x12, x8
-; CHECK-NEXT:    add x15, x1, x8
-; CHECK-NEXT:    add x16, x11, x8
+; CHECK-NEXT:    add x14, x0, x8
+; CHECK-NEXT:    add x15, x12, x8
+; CHECK-NEXT:    add x16, x1, x8
+; CHECK-NEXT:    add x17, x11, x8
 ; CHECK-NEXT:    ld1b { z4.b }, p1/z, [x0, x8]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x14, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x12, x8]
-; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x15, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z16.b }, p1/z, [x1, x8]
-; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x15, #1, mul vl]
+; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x16, #1, mul vl]
 ; CHECK-NEXT:    ld1b { z18.b }, p1/z, [x11, x8]
-; CHECK-NEXT:    ld1d { z19.d }, p0/z, [x16, #1, mul vl]
+; CHECK-NEXT:    ld1d { z19.d }, p0/z, [x17, #1, mul vl]
 ; CHECK-NEXT:    adds x10, x10, x9
-; CHECK-NEXT:    addvl x8, x8, #4
+; CHECK-NEXT:    add x8, x8, x13
 ; CHECK-NEXT:    fcmla z1.d, p0/m, z16.d, z4.d, #0
 ; CHECK-NEXT:    fcmla z0.d, p0/m, z17.d, z5.d, #0
 ; CHECK-NEXT:    fcmla z2.d, p0/m, z18.d, z6.d, #0
@@ -326,6 +330,7 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia
 ; CHECK-NEXT:    mov w11, #100 // =0x64
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    and x10, x10, x11
+; CHECK-NEXT:    rdvl x11, #2
 ; CHECK-NEXT:    zip2 z0.d, z2.d, z2.d
 ; CHECK-NEXT:    zip1 z1.d, z2.d, z2.d
 ; CHECK-NEXT:  .LBB3_1: // %vector.body
@@ -334,7 +339,7 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia
 ; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x0]
 ; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x0, #1, mul vl]
 ; CHECK-NEXT:    add x8, x8, x9
-; CHECK-NEXT:    addvl x0, x0, #2
+; CHECK-NEXT:    add x0, x0, x11
 ; CHECK-NEXT:    cmp x10, x8
 ; CHECK-NEXT:    fadd z0.d, z5.d, z0.d
 ; CHECK-NEXT:    fadd z1.d, z4.d, z1.d