@@ -359,77 +359,100 @@ func.func @vector_arm_neon_k_unroll_vecmat(%lhs: vector<1x32xi8>, %rhs: vector<2
359
359
360
360
// Test with more than one iteration across all of the M, N and K dimensions.
361
361
// Shows multiple independent accumulators as well as accumulator reuse.
362
+ // For each iterarion [I, J, K] sub-tiles are extracted from offsets as follows:
363
+ // LHS: [2*I, 8*K]
364
+ // RHS: [2*J, 8*K]
365
+ // ACC: [2*I, 2*J]
366
+ // Sub-tile insert offsets for the result are as like ACC (there are redundant
367
+ // inserts).
362
368
363
369
// CHECK-LABEL: @vector_arm_neon_mnk_unroll
364
- // CHECK-SAME: %arg0: vector<4x16xi8>
365
- // CHECK-SAME: %arg1: vector<4x16xi8>
366
- // CHECK-SAME: %arg2: vector<4x4xi32>
370
+ // CHECK-SAME: %[[LHS: arg0]] : vector<4x16xi8>
371
+ // CHECK-SAME: %[[RHS: arg1]] : vector<4x16xi8>
372
+ // CHECK-SAME: %[[ACC: arg2]] : vector<4x4xi32>
367
373
// CHECK-SAME: -> vector<4x4xi32> {
368
374
// CHECK-NEXT: %cst = arith.constant dense<0> : vector<4x4xi32>
369
- // CHECK-NEXT: %[[VAL_0:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
370
- // CHECK-NEXT: %[[VAL_1:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
371
- // CHECK-NEXT: %[[VAL_2:[0-9]+]] = vector.extract_strided_slice %arg2 {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
375
+
376
+ // Iteration: [0, 0, 0]
377
+ // CHECK-NEXT: %[[VAL_0:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
378
+ // CHECK-NEXT: %[[VAL_1:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
379
+ // CHECK-NEXT: %[[VAL_2:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
372
380
// CHECK-NEXT: %[[VAL_3:[0-9]+]] = vector.shape_cast %[[VAL_0]] : vector<2x8xi8> to vector<16xi8>
373
381
// CHECK-NEXT: %[[VAL_4:[0-9]+]] = vector.shape_cast %[[VAL_1]] : vector<2x8xi8> to vector<16xi8>
374
382
// CHECK-NEXT: %[[VAL_5:[0-9]+]] = vector.shape_cast %[[VAL_2]] : vector<2x2xi32> to vector<4xi32>
375
383
// CHECK-NEXT: %[[KACC_0_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_5]], %[[VAL_3]], %[[VAL_4]] : vector<16xi8> to vector<4xi32>
376
384
// CHECK-NEXT: %[[VAL_7:[0-9]+]] = vector.shape_cast %[[KACC_0_v0]] : vector<4xi32> to vector<2x2xi32>
377
385
// CHECK-NEXT: %[[VAL_8:[0-9]+]] = vector.insert_strided_slice %[[VAL_7]], %cst {offsets = [0, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
378
- // CHECK-NEXT: %[[VAL_9:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
379
- // CHECK-NEXT: %[[VAL_10:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
386
+
387
+ // Iteration: [0, 0, 1]
388
+ // CHECK-NEXT: %[[VAL_9:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
389
+ // CHECK-NEXT: %[[VAL_10:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
380
390
// CHECK-NEXT: %[[VAL_11:[0-9]+]] = vector.shape_cast %[[VAL_9]] : vector<2x8xi8> to vector<16xi8>
381
391
// CHECK-NEXT: %[[VAL_12:[0-9]+]] = vector.shape_cast %[[VAL_10]] : vector<2x8xi8> to vector<16xi8>
382
392
// CHECK-NEXT: %[[KACC_0_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_0_v0]], %[[VAL_11]], %[[VAL_12]] : vector<16xi8> to vector<4xi32>
383
393
// CHECK-NEXT: %[[VAL_14:[0-9]+]] = vector.shape_cast %[[KACC_0_v1]] : vector<4xi32> to vector<2x2xi32>
384
394
// CHECK-NEXT: %[[VAL_15:[0-9]+]] = vector.insert_strided_slice %[[VAL_14]], %[[VAL_8]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
385
- // CHECK-NEXT: %[[VAL_16:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
386
- // CHECK-NEXT: %[[VAL_17:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
387
- // CHECK-NEXT: %[[VAL_18:[0-9]+]] = vector.extract_strided_slice %arg2 {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
395
+
396
+ // Iteration: [0, 1, 0]
397
+ // CHECK-NEXT: %[[VAL_16:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
398
+ // CHECK-NEXT: %[[VAL_17:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
399
+ // CHECK-NEXT: %[[VAL_18:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
388
400
// CHECK-NEXT: %[[VAL_19:[0-9]+]] = vector.shape_cast %[[VAL_16]] : vector<2x8xi8> to vector<16xi8>
389
401
// CHECK-NEXT: %[[VAL_20:[0-9]+]] = vector.shape_cast %[[VAL_17]] : vector<2x8xi8> to vector<16xi8>
390
402
// CHECK-NEXT: %[[VAL_21:[0-9]+]] = vector.shape_cast %[[VAL_18]] : vector<2x2xi32> to vector<4xi32>
391
403
// CHECK-NEXT: %[[KACC_1_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_21]], %[[VAL_19]], %[[VAL_20]] : vector<16xi8> to vector<4xi32>
392
404
// CHECK-NEXT: %[[VAL_23:[0-9]+]] = vector.shape_cast %[[KACC_1_v0]] : vector<4xi32> to vector<2x2xi32>
393
405
// CHECK-NEXT: %[[VAL_24:[0-9]+]] = vector.insert_strided_slice %[[VAL_23]], %[[VAL_15]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
394
- // CHECK-NEXT: %[[VAL_25:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
395
- // CHECK-NEXT: %[[VAL_26:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
406
+
407
+ // Iteration: [0, 1, 1]
408
+ // CHECK-NEXT: %[[VAL_25:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
409
+ // CHECK-NEXT: %[[VAL_26:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
396
410
// CHECK-NEXT: %[[VAL_27:[0-9]+]] = vector.shape_cast %[[VAL_25]] : vector<2x8xi8> to vector<16xi8>
397
411
// CHECK-NEXT: %[[VAL_28:[0-9]+]] = vector.shape_cast %[[VAL_26]] : vector<2x8xi8> to vector<16xi8>
398
412
// CHECK-NEXT: %[[KACC_1_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_1_v0]], %[[VAL_27]], %[[VAL_28]] : vector<16xi8> to vector<4xi32>
399
413
// CHECK-NEXT: %[[VAL_30:[0-9]+]] = vector.shape_cast %[[KACC_1_v1]] : vector<4xi32> to vector<2x2xi32>
400
414
// CHECK-NEXT: %[[VAL_31:[0-9]+]] = vector.insert_strided_slice %[[VAL_30]], %[[VAL_24]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
401
- // CHECK-NEXT: %[[VAL_32:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
402
- // CHECK-NEXT: %[[VAL_33:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
403
- // CHECK-NEXT: %[[VAL_34:[0-9]+]] = vector.extract_strided_slice %arg2 {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
415
+
416
+ // Iteration: [1, 0, 0]
417
+ // CHECK-NEXT: %[[VAL_32:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
418
+ // CHECK-NEXT: %[[VAL_33:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
419
+ // CHECK-NEXT: %[[VAL_34:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
404
420
// CHECK-NEXT: %[[VAL_35:[0-9]+]] = vector.shape_cast %[[VAL_32]] : vector<2x8xi8> to vector<16xi8>
405
421
// CHECK-NEXT: %[[VAL_36:[0-9]+]] = vector.shape_cast %[[VAL_33]] : vector<2x8xi8> to vector<16xi8>
406
422
// CHECK-NEXT: %[[VAL_37:[0-9]+]] = vector.shape_cast %[[VAL_34]] : vector<2x2xi32> to vector<4xi32>
407
423
// CHECK-NEXT: %[[KACC_2_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_37]], %[[VAL_35]], %[[VAL_36]] : vector<16xi8> to vector<4xi32>
408
424
// CHECK-NEXT: %[[VAL_39:[0-9]+]] = vector.shape_cast %[[KACC_2_v0]] : vector<4xi32> to vector<2x2xi32>
409
425
// CHECK-NEXT: %[[VAL_40:[0-9]+]] = vector.insert_strided_slice %[[VAL_39]], %[[VAL_31]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
410
- // CHECK-NEXT: %[[VAL_41:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
411
- // CHECK-NEXT: %[[VAL_42:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
426
+
427
+ // Iteration: [1, 0, 1]
428
+ // CHECK-NEXT: %[[VAL_41:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
429
+ // CHECK-NEXT: %[[VAL_42:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [0, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
412
430
// CHECK-NEXT: %[[VAL_43:[0-9]+]] = vector.shape_cast %[[VAL_41]] : vector<2x8xi8> to vector<16xi8>
413
431
// CHECK-NEXT: %[[VAL_44:[0-9]+]] = vector.shape_cast %[[VAL_42]] : vector<2x8xi8> to vector<16xi8>
414
432
// CHECK-NEXT: %[[KACC_2_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_2_v0]], %[[VAL_43]], %[[VAL_44]] : vector<16xi8> to vector<4xi32>
415
433
// CHECK-NEXT: %[[VAL_46:[0-9]+]] = vector.shape_cast %[[KACC_2_v1]] : vector<4xi32> to vector<2x2xi32>
416
434
// CHECK-NEXT: %[[VAL_47:[0-9]+]] = vector.insert_strided_slice %[[VAL_46]], %[[VAL_40]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
417
- // CHECK-NEXT: %[[VAL_48:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
418
- // CHECK-NEXT: %[[VAL_49:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
419
- // CHECK-NEXT: %[[VAL_50:[0-9]+]] = vector.extract_strided_slice %arg2 {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
435
+
436
+ // Iteration: [1, 1, 0]
437
+ // CHECK-NEXT: %[[VAL_48:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
438
+ // CHECK-NEXT: %[[VAL_49:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
439
+ // CHECK-NEXT: %[[VAL_50:[0-9]+]] = vector.extract_strided_slice %[[ACC]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xi32> to vector<2x2xi32>
420
440
// CHECK-NEXT: %[[VAL_51:[0-9]+]] = vector.shape_cast %[[VAL_48]] : vector<2x8xi8> to vector<16xi8>
421
441
// CHECK-NEXT: %[[VAL_52:[0-9]+]] = vector.shape_cast %[[VAL_49]] : vector<2x8xi8> to vector<16xi8>
422
442
// CHECK-NEXT: %[[VAL_53:[0-9]+]] = vector.shape_cast %[[VAL_50]] : vector<2x2xi32> to vector<4xi32>
423
443
// CHECK-NEXT: %[[KACC_3_v0:[0-9]+]] = arm_neon.intr.smmla %[[VAL_53]], %[[VAL_51]], %[[VAL_52]] : vector<16xi8> to vector<4xi32>
424
444
// CHECK-NEXT: %[[VAL_55:[0-9]+]] = vector.shape_cast %[[KACC_3_v0]] : vector<4xi32> to vector<2x2xi32>
425
445
// CHECK-NEXT: %[[VAL_56:[0-9]+]] = vector.insert_strided_slice %[[VAL_55]], %[[VAL_47]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
426
- // CHECK-NEXT: %[[VAL_57:[0-9]+]] = vector.extract_strided_slice %arg0 {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
427
- // CHECK-NEXT: %[[VAL_58:[0-9]+]] = vector.extract_strided_slice %arg1 {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
446
+
447
+ // Iteration: [1, 1, 1]
448
+ // CHECK-NEXT: %[[VAL_57:[0-9]+]] = vector.extract_strided_slice %[[LHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
449
+ // CHECK-NEXT: %[[VAL_58:[0-9]+]] = vector.extract_strided_slice %[[RHS]] {offsets = [2, 8], sizes = [2, 8], strides = [1, 1]} : vector<4x16xi8> to vector<2x8xi8>
428
450
// CHECK-NEXT: %[[VAL_59:[0-9]+]] = vector.shape_cast %[[VAL_57]] : vector<2x8xi8> to vector<16xi8>
429
451
// CHECK-NEXT: %[[VAL_60:[0-9]+]] = vector.shape_cast %[[VAL_58]] : vector<2x8xi8> to vector<16xi8>
430
452
// CHECK-NEXT: %[[KACC_3_v1:[0-9]+]] = arm_neon.intr.smmla %[[KACC_3_v0]], %[[VAL_59]], %[[VAL_60]] : vector<16xi8> to vector<4xi32>
431
453
// CHECK-NEXT: %[[VAL_62:[0-9]+]] = vector.shape_cast %[[KACC_3_v1]] : vector<4xi32> to vector<2x2xi32>
432
454
// CHECK-NEXT: %[[VAL_63:[0-9]+]] = vector.insert_strided_slice %[[VAL_62]], %[[VAL_56]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xi32> into vector<4x4xi32>
455
+
433
456
// CHECK-NEXT: return %[[VAL_63]] : vector<4x4xi32>
434
457
func.func @vector_arm_neon_mnk_unroll (%lhs: vector <4 x16 xi8 >, %rhs: vector <4 x16 xi8 >, %acc : vector <4 x4 xi32 >) -> vector <4 x4 xi32 > {
435
458
%lhs_extsi = arith.extsi %lhs : vector <4 x16 xi8 > to vector <4 x16 xi32 >
0 commit comments