1
- // RUN: mlir-opt %s -transform-interpreter -split-input-file | FileCheck %s
1
+ // RUN: mlir-opt -split-input-file \
2
+ // RUN: -transform-preload-library='transform-library-paths=%p/td/vectorize-with-patterns.mlir' \
3
+ // RUN: -transform-interpreter=entry-point=vectorize_with_patterns %s | FileCheck %s
2
4
3
5
#map0 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d2 )>
4
6
#map1 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d1 , d2 , d3 )>
@@ -27,15 +29,6 @@ func.func @vectorize_1d_tensor_extract(%arg0: tensor<3xf32>, %arg1: tensor<4x3xi
27
29
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]]] [%[[INDICES]]], %[[MASK]], %[[PASSTHRU]]
28
30
// CHECK: vector.transfer_write %[[GATHER]]
29
31
30
- module attributes {transform.with_named_sequence } {
31
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
32
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
33
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
34
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op ) -> !transform.any_op
35
- transform.yield
36
- }
37
- }
38
-
39
32
// -----
40
33
41
34
#map = affine_map <() -> ()>
@@ -59,15 +52,6 @@ func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>)
59
52
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
60
53
// CHECK: vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
61
54
62
- module attributes {transform.with_named_sequence } {
63
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
64
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
65
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
66
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
67
- transform.yield
68
- }
69
- }
70
-
71
55
// -----
72
56
73
57
#map = affine_map <(n ) -> (n )>
@@ -92,15 +76,6 @@ func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32
92
76
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
93
77
// CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
94
78
95
- module attributes {transform.with_named_sequence } {
96
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
97
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
98
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
99
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
100
- transform.yield
101
- }
102
- }
103
-
104
79
// -----
105
80
106
81
#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -130,14 +105,6 @@ func.func @vectorize_nd_tensor_extract_scalar_broadcast(%arg0: tensor<3x3xf32>,
130
105
// CHECK: %[[C0_2:.*]] = arith.constant 0 : index
131
106
// CHECK: vector.transfer_write %[[READ]], %[[ARG_1]]{{\[}}%[[C0_2]], %[[C0_2]], %[[C0_2]]] : vector<1x1x3xf32>, tensor<1x1x3xf32>
132
107
133
- module attributes {transform.with_named_sequence } {
134
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
135
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
136
- transform.structured.vectorize %0 { vectorize_nd_extract } : !transform.any_op
137
- transform.yield
138
- }
139
- }
140
-
141
108
// -----
142
109
143
110
#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -207,15 +174,6 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
207
174
// CHECK: %[[RES:.*]] = vector.transfer_write %[[BCAST]], %[[OUTPUT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32>
208
175
// CHECK: return %[[RES]] : tensor<3x1x1xf32>
209
176
210
- module attributes {transform.with_named_sequence } {
211
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
212
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
213
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
214
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
215
- transform.yield
216
- }
217
- }
218
-
219
177
// -----
220
178
221
179
func.func @vectorize_nd_tensor_extract_transfer_read_complex (%6: tensor <45 x80 x16 xf32 >, %arg0: index , %arg2: index , %arg1: index , %arg4: index , %extracted_slice : tensor <1 x4 xf32 >) -> tensor <1 x4 xf32 > {
@@ -259,15 +217,6 @@ func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16
259
217
// CHECK: return %[[VAL_21]] : tensor<1x4xf32>
260
218
// CHECK: }
261
219
262
- module attributes {transform.with_named_sequence } {
263
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
264
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
265
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
266
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
267
- transform.yield
268
- }
269
- }
270
-
271
220
// -----
272
221
273
222
#map0 = affine_map <(d0 , d1 , d2 , d3 ) -> (d0 , d2 )>
@@ -309,15 +258,6 @@ func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>,
309
258
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]]] [%[[T]]], %[[CST_1]], %[[PASSTHRU]] : tensor<3x3xf32>, vector<4x7x3x2xindex>, vector<4x7x3x2xi1>, vector<4x7x3x2xf32> into vector<4x7x3x2xf32>
310
259
// CHECK: vector.transfer_write %[[GATHER]], %[[ARG4]][%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true, true]} : vector<4x7x3x2xf32>, tensor<4x7x3x2xf32>
311
260
312
- module attributes {transform.with_named_sequence } {
313
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
314
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
315
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
316
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
317
- transform.yield
318
- }
319
- }
320
-
321
261
// -----
322
262
323
263
#map = affine_map <(d0 , d1 ) -> (d0 , d1 )>
@@ -339,15 +279,6 @@ func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load(%
339
279
return %1 : tensor <8 x1 xf32 >
340
280
}
341
281
342
- module attributes {transform.with_named_sequence } {
343
- transform.named_sequence @__transform_main (%arg0: !transform.any_op {transform.readonly }) {
344
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg0 : (!transform.any_op ) -> !transform.any_op
345
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
346
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
347
- transform.yield
348
- }
349
- }
350
-
351
282
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load
352
283
// CHECK-SAME: %[[ARG0:.*]]: tensor<8x128x768xf32>
353
284
// CHECK-SAME: %[[ARG1:.*]]: index
@@ -390,15 +321,6 @@ func.func @index_from_output_column_vector_gather_load(%src: tensor<8x128xf32>)
390
321
return %res : tensor <8 x1 xf32 >
391
322
}
392
323
393
- module attributes {transform.with_named_sequence } {
394
- transform.named_sequence @__transform_main (%arg2: !transform.any_op {transform.readonly }) {
395
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg2 : (!transform.any_op ) -> !transform.any_op
396
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
397
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
398
- transform.yield
399
- }
400
- }
401
-
402
324
// CHECK-LABEL: func.func @index_from_output_column_vector_gather_load(
403
325
// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
404
326
// CHECK: %[[C128:.*]] = arith.constant dense<128> : vector<1x8xindex>
@@ -437,15 +359,6 @@ func.func @index_from_output_column_vector_contiguous_load(%src: tensor<8x128xf3
437
359
return %res : tensor <8 x1 xf32 >
438
360
}
439
361
440
- module attributes {transform.with_named_sequence } {
441
- transform.named_sequence @__transform_main (%arg2: !transform.any_op {transform.readonly }) {
442
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg2 : (!transform.any_op ) -> !transform.any_op
443
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
444
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 {vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
445
- transform.yield
446
- }
447
- }
448
-
449
362
// CHECK-LABEL: func.func @index_from_output_column_vector_contiguous_load(
450
363
// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
451
364
// CHECK: %[[C0:.*]] = arith.constant 0 : index
@@ -497,15 +410,6 @@ func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32
497
410
// CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_8]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32>
498
411
// CHECK: return %[[VAL_14]] : tensor<5xf32>
499
412
500
- module attributes {transform.with_named_sequence } {
501
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
502
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
503
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
504
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
505
- transform.yield
506
- }
507
- }
508
-
509
413
// -----
510
414
511
415
// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Contiguous load.
@@ -540,15 +444,6 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<8
540
444
// CHECK: return %[[VAL_12]] : tensor<1x4xf32>
541
445
// CHECK: }
542
446
543
- module attributes {transform.with_named_sequence } {
544
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
545
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
546
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
547
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
548
- transform.yield
549
- }
550
- }
551
-
552
447
// -----
553
448
554
449
func.func @vectorize_nd_tensor_extract_with_tensor_extract (%input_1: tensor <1 x20 xi32 >, %input_2: tensor <257 x24 xf32 >, %arg0 : index , %arg1 : index , %arg2 : index , %arg3 : index ) -> tensor <1 x1 x4 xf32 > {
@@ -585,16 +480,6 @@ func.func @vectorize_nd_tensor_extract_with_tensor_extract(%input_1: tensor<1x20
585
480
// for address calculation also satisfy the required conditions).
586
481
// CHECK: vector.transfer_read %[[INPUT_2]][%{{.*}}, %{{.*}}, %{{.*}} {in_bounds = [true, true]} : tensor<257x24xf32>, vector<1x4xf32>
587
482
588
-
589
- module attributes {transform.with_named_sequence } {
590
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
591
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
592
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
593
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
594
- transform.yield
595
- }
596
- }
597
-
598
483
// -----
599
484
600
485
// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Gather load.
@@ -632,15 +517,6 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tensor<80x16
632
517
// CHECK: return %[[VAL_14]] : tensor<1x4xf32>
633
518
// CHECK: }
634
519
635
- module attributes {transform.with_named_sequence } {
636
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
637
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
638
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
639
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
640
- transform.yield
641
- }
642
- }
643
-
644
520
// -----
645
521
646
522
// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Gather load.
@@ -674,15 +550,6 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_gather(%arg0: tensor<80x16xf32
674
550
// CHECK: return %[[VAL_10]] : tensor<1x4xf32>
675
551
// CHECK: }
676
552
677
- module attributes {transform.with_named_sequence } {
678
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
679
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
680
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
681
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
682
- transform.yield
683
- }
684
- }
685
-
686
553
// -----
687
554
688
555
// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Contiguous load.
@@ -718,15 +585,6 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16
718
585
// CHECK: return %[[VAL_9]] : tensor<1x4xf32>
719
586
// CHECK: }
720
587
721
- module attributes {transform.with_named_sequence } {
722
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
723
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
724
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
725
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
726
- transform.yield
727
- }
728
- }
729
-
730
588
// -----
731
589
732
590
// The vectorizer assumes it's a gather load whenever using a block argument to calculate an index.
@@ -759,15 +617,6 @@ func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1:
759
617
// CHECK: return %[[VAL_12]] : tensor<5xf32>
760
618
// CHECK: }
761
619
762
- module attributes {transform.with_named_sequence } {
763
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
764
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
765
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
766
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
767
- transform.yield
768
- }
769
- }
770
-
771
620
// -----
772
621
773
622
#map1 = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -788,15 +637,6 @@ func.func @vectorize_0d_tensor_extract(%arg0: tensor<f32>, %arg2: tensor<1x1x3xf
788
637
// CHECK: %[[EXTRACT:.*]] = vector.transfer_read %[[ARG_0]][], %{{.+}} : tensor<f32>
789
638
// CHECK: vector.broadcast %[[EXTRACT]] : vector<f32> to vector<1x1x3xf32>
790
639
791
- module attributes {transform.with_named_sequence } {
792
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
793
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
794
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
795
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
796
- transform.yield
797
- }
798
- }
799
-
800
640
// -----
801
641
802
642
#map = affine_map <(d0 , d1 , d2 ) -> (d0 , d1 , d2 )>
@@ -833,16 +673,6 @@ func.func @vectorize_reverse_like_tensor_extract(%arg0: tensor<1x2x3xf32>, %arg1
833
673
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[T3]]], %[[MASK]], %[[PASSTHRU]]
834
674
// CHECK: vector.transfer_write %[[GATHER]]
835
675
836
- module attributes {transform.with_named_sequence } {
837
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
838
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
839
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
840
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
841
- transform.yield
842
- }
843
- }
844
-
845
-
846
676
// -----
847
677
848
678
func.func @vectorize_scalar_read_with_broadcast_from_column_tensor (%init: tensor <1 x1 x4 xi32 >) -> tensor <1 x1 x4 xi32 > {
@@ -874,12 +704,3 @@ func.func @vectorize_scalar_read_with_broadcast_from_column_tensor(%init: tensor
874
704
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[IDX_ELT]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
875
705
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<i32> to vector<1x1x4xi32>
876
706
// CHECK: %[[RES:.*]] = vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x4xi32>
877
-
878
- module attributes {transform.with_named_sequence } {
879
- transform.named_sequence @__transform_main (%arg1: !transform.any_op {transform.readonly }) {
880
- %0 = transform.structured.match ops {[" linalg.generic" ]} in %arg1 : (!transform.any_op ) -> !transform.any_op
881
- %1 = transform.get_parent_op %0 {isolated_from_above } : (!transform.any_op ) -> !transform.any_op
882
- %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op ) -> !transform.any_op
883
- transform.yield
884
- }
885
- }
0 commit comments