|
| 1 | +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5 |
| 2 | +; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s |
| 3 | +; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s |
| 4 | +; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s |
| 5 | + |
| 6 | +define i64 @select_decreasing_induction_icmp_const_start(ptr %a) { |
| 7 | +; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_const_start( |
| 8 | +; CHECK-SAME: ptr [[A:%.*]]) { |
| 9 | +; CHECK-NEXT: [[ENTRY:.*]]: |
| 10 | +; CHECK-NEXT: br label %[[LOOP:.*]] |
| 11 | +; CHECK: [[LOOP]]: |
| 12 | +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| 13 | +; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ] |
| 14 | +; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] |
| 15 | +; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8 |
| 16 | +; CHECK-NEXT: [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3 |
| 17 | +; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]] |
| 18 | +; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1 |
| 19 | +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0 |
| 20 | +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]] |
| 21 | +; CHECK: [[EXIT]]: |
| 22 | +; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ] |
| 23 | +; CHECK-NEXT: ret i64 [[SPEC_SELECT_LCSSA]] |
| 24 | +; |
| 25 | +entry: |
| 26 | + br label %loop |
| 27 | + |
| 28 | +loop: ; preds = %entry, %loop |
| 29 | + %iv = phi i64 [ 19999, %entry ], [ %iv.next, %loop ] |
| 30 | + %rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ] |
| 31 | + %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv |
| 32 | + %ld.a = load i64, ptr %gep.a.iv, align 8 |
| 33 | + %cmp.a.3 = icmp sgt i64 %ld.a, 3 |
| 34 | + %spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx |
| 35 | + %iv.next = add nsw i64 %iv, -1 |
| 36 | + %exit.cond = icmp eq i64 %iv, 0 |
| 37 | + br i1 %exit.cond, label %exit, label %loop |
| 38 | + |
| 39 | +exit: ; preds = %loop |
| 40 | + ret i64 %spec.select |
| 41 | +} |
| 42 | + |
| 43 | +@table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1 |
| 44 | + |
| 45 | +define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) { |
| 46 | +; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_i16( |
| 47 | +; CHECK-SAME: i16 noundef [[VAL:%.*]]) { |
| 48 | +; CHECK-NEXT: [[ENTRY:.*]]: |
| 49 | +; CHECK-NEXT: br label %[[LOOP:.*]] |
| 50 | +; CHECK: [[LOOP]]: |
| 51 | +; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| 52 | +; CHECK-NEXT: [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ] |
| 53 | +; CHECK-NEXT: [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]] |
| 54 | +; CHECK-NEXT: [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1 |
| 55 | +; CHECK-NEXT: [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]] |
| 56 | +; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1 |
| 57 | +; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]] |
| 58 | +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0 |
| 59 | +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]] |
| 60 | +; CHECK: [[EXIT]]: |
| 61 | +; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ] |
| 62 | +; CHECK-NEXT: ret i16 [[SPEC_SELECT_LCSSA]] |
| 63 | +; |
| 64 | +entry: |
| 65 | + br label %loop |
| 66 | + |
| 67 | +loop: ; preds = %entry, %loop |
| 68 | + %iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ] |
| 69 | + %rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ] |
| 70 | + %gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv |
| 71 | + %ld.table = load i16, ptr %gep.table.iv, align 1 |
| 72 | + %cmp.table.val = icmp ugt i16 %ld.table, %val |
| 73 | + %iv.next = add nsw i16 %iv, -1 |
| 74 | + %spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx |
| 75 | + %exit.cond = icmp eq i16 %iv.next, 0 |
| 76 | + br i1 %exit.cond, label %exit, label %loop |
| 77 | + |
| 78 | +exit: ; preds = %loop |
| 79 | + %spec.select.lcssa = phi i16 [ %spec.select, %loop ] |
| 80 | + ret i16 %spec.select.lcssa |
| 81 | +} |
| 82 | + |
| 83 | +@tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1 |
| 84 | + |
| 85 | +define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) { |
| 86 | +; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_half( |
| 87 | +; CHECK-SAME: half noundef [[VAL:%.*]]) { |
| 88 | +; CHECK-NEXT: [[ENTRY:.*]]: |
| 89 | +; CHECK-NEXT: br label %[[LOOP:.*]] |
| 90 | +; CHECK: [[LOOP]]: |
| 91 | +; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| 92 | +; CHECK-NEXT: [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ] |
| 93 | +; CHECK-NEXT: [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]] |
| 94 | +; CHECK-NEXT: [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1 |
| 95 | +; CHECK-NEXT: [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]] |
| 96 | +; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1 |
| 97 | +; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]] |
| 98 | +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0 |
| 99 | +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]] |
| 100 | +; CHECK: [[EXIT]]: |
| 101 | +; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ] |
| 102 | +; CHECK-NEXT: ret i16 [[SPEC_SELECT_LCSSA]] |
| 103 | +; |
| 104 | +entry: |
| 105 | + br label %loop |
| 106 | + |
| 107 | +loop: ; preds = %entry, %loop |
| 108 | + %iv = phi i16 [ 12, %entry ], [ %iv.next, %loop ] |
| 109 | + %rdx = phi i16 [ 0, %entry ], [ %spec.select, %loop ] |
| 110 | + %gep.table.iv = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 %iv |
| 111 | + %ld.table = load half, ptr %gep.table.iv, align 1 |
| 112 | + %cmp.table.val = fcmp ugt half %ld.table, %val |
| 113 | + %iv.next = add nsw i16 %iv, -1 |
| 114 | + %spec.select = select i1 %cmp.table.val, i16 %iv.next, i16 %rdx |
| 115 | + %exit.cond = icmp eq i16 %iv.next, 0 |
| 116 | + br i1 %exit.cond, label %exit, label %loop |
| 117 | + |
| 118 | +exit: ; preds = %loop |
| 119 | + %spec.select.lcssa = phi i16 [ %spec.select, %loop ] |
| 120 | + ret i16 %spec.select.lcssa |
| 121 | +} |
| 122 | + |
| 123 | +define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) { |
| 124 | +; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_non_const_start( |
| 125 | +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) { |
| 126 | +; CHECK-NEXT: [[ENTRY:.*]]: |
| 127 | +; CHECK-NEXT: br label %[[LOOP:.*]] |
| 128 | +; CHECK: [[LOOP]]: |
| 129 | +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ] |
| 130 | +; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[LOOP]] ], [ [[RDX_START]], %[[ENTRY]] ] |
| 131 | +; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1 |
| 132 | +; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_NEXT]] |
| 133 | +; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8 |
| 134 | +; CHECK-NEXT: [[GEP_B_IV:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV_NEXT]] |
| 135 | +; CHECK-NEXT: [[LD_B:%.*]] = load i64, ptr [[GEP_B_IV]], align 8 |
| 136 | +; CHECK-NEXT: [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]] |
| 137 | +; CHECK-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[IV_NEXT]], i64 [[RDX]] |
| 138 | +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1 |
| 139 | +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT:.*]] |
| 140 | +; CHECK: [[EXIT]]: |
| 141 | +; CHECK-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ] |
| 142 | +; CHECK-NEXT: ret i64 [[COND_LCSSA]] |
| 143 | +; |
| 144 | +entry: |
| 145 | + br label %loop |
| 146 | + |
| 147 | +loop: ; preds = %entry, %loop |
| 148 | + %iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ] |
| 149 | + %rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ] |
| 150 | + %iv.next = add nsw i64 %iv, -1 |
| 151 | + %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv.next |
| 152 | + %ld.a = load i64, ptr %gep.a.iv, align 8 |
| 153 | + %gep.b.iv = getelementptr inbounds i64, ptr %b, i64 %iv.next |
| 154 | + %ld.b = load i64, ptr %gep.b.iv, align 8 |
| 155 | + %cmp.a.b = icmp sgt i64 %ld.a, %ld.b |
| 156 | + %cond = select i1 %cmp.a.b, i64 %iv.next, i64 %rdx |
| 157 | + %exit.cond = icmp ugt i64 %iv, 1 |
| 158 | + br i1 %exit.cond, label %loop, label %exit |
| 159 | + |
| 160 | +exit: ; preds = %loop |
| 161 | + ret i64 %cond |
| 162 | +} |
| 163 | + |
| 164 | +; The sentinel value for decreasing-IV vectorization is LONG_MAX, and since |
| 165 | +; the IV hits this value, it is impossible to vectorize this case. |
| 166 | +define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound(ptr %a) { |
| 167 | +; CHECK-LABEL: define i64 @not_vectorized_select_decreasing_induction_icmp_iv_out_of_bound( |
| 168 | +; CHECK-SAME: ptr [[A:%.*]]) { |
| 169 | +; CHECK-NEXT: [[ENTRY:.*]]: |
| 170 | +; CHECK-NEXT: br label %[[LOOP:.*]] |
| 171 | +; CHECK: [[LOOP]]: |
| 172 | +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| 173 | +; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ] |
| 174 | +; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] |
| 175 | +; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8 |
| 176 | +; CHECK-NEXT: [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3 |
| 177 | +; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]] |
| 178 | +; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1 |
| 179 | +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0 |
| 180 | +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]] |
| 181 | +; CHECK: [[EXIT]]: |
| 182 | +; CHECK-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ] |
| 183 | +; CHECK-NEXT: ret i64 [[SPEC_SELECT_LCSSA]] |
| 184 | +; |
| 185 | +entry: |
| 186 | + br label %loop |
| 187 | + |
| 188 | +loop: ; preds = %entry, %loop |
| 189 | + %iv = phi i64 [ 9223372036854775807, %entry ], [ %iv.next, %loop ] |
| 190 | + %rdx = phi i64 [ 331, %entry ], [ %spec.select, %loop ] |
| 191 | + %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv |
| 192 | + %ld.a = load i64, ptr %gep.a.iv, align 8 |
| 193 | + %cmp.a.3 = icmp sgt i64 %ld.a, 3 |
| 194 | + %spec.select = select i1 %cmp.a.3, i64 %iv, i64 %rdx |
| 195 | + %iv.next = add nsw i64 %iv, -1 |
| 196 | + %exit.cond = icmp eq i64 %iv, 0 |
| 197 | + br i1 %exit.cond, label %exit, label %loop |
| 198 | + |
| 199 | +exit: ; preds = %loop |
| 200 | + ret i64 %spec.select |
| 201 | +} |
| 202 | + |
0 commit comments