Skip to content

Commit d27701b

Browse files
LukacmaLukacma
committed
[AArch64] Model ZA array using inaccessible memory
1 parent e8e9868 commit d27701b

File tree

4 files changed

+483
-257
lines changed

4 files changed

+483
-257
lines changed

‎llvm/include/llvm/IR/IntrinsicsAArch64.td

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2940,7 +2940,7 @@ def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
29402940
let TargetPrefix = "aarch64" in {
29412941
class SME_Load_Store_Intrinsic<LLVMType pred_ty>
29422942
: DefaultAttrsIntrinsic<[],
2943-
[pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
2943+
[pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<2>>]>;
29442944

29452945
// Loads
29462946
def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>;
@@ -2968,18 +2968,18 @@ let TargetPrefix = "aarch64" in {
29682968

29692969
// Spill + fill
29702970
class SME_LDR_STR_ZA_Intrinsic
2971-
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty]>;
2971+
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOrArgMemOnly]>;
29722972
def int_aarch64_sme_ldr : SME_LDR_STR_ZA_Intrinsic;
29732973
def int_aarch64_sme_str : SME_LDR_STR_ZA_Intrinsic;
29742974

29752975
class SME_TileToVector_Intrinsic
29762976
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
29772977
[LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2978-
llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
2978+
llvm_i32_ty, llvm_i32_ty], [IntrReadMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<2>>]>;
29792979
class SME_VectorToTile_Intrinsic
29802980
: DefaultAttrsIntrinsic<[],
29812981
[llvm_i32_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2982-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
2982+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
29832983

29842984
def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic;
29852985
def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic;
@@ -2994,13 +2994,13 @@ let TargetPrefix = "aarch64" in {
29942994
class SME_MOVAZ_TileToVector_X2_Intrinsic
29952995
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
29962996
[llvm_i32_ty, llvm_i32_ty],
2997-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
2997+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
29982998

29992999
class SME_MOVAZ_TileToVector_X4_Intrinsic
30003000
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
30013001
LLVMMatchType<0>,LLVMMatchType<0>],
30023002
[llvm_i32_ty, llvm_i32_ty],
3003-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
3003+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30043004

30053005
def int_aarch64_sme_readz_horiz_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic;
30063006
def int_aarch64_sme_readz_vert_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic;
@@ -3011,7 +3011,7 @@ let TargetPrefix = "aarch64" in {
30113011
class SME_MOVAZ_TileToVector_Intrinsic
30123012
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
30133013
[llvm_i32_ty, llvm_i32_ty],
3014-
[IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
3014+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30153015

30163016
def int_aarch64_sme_readz_horiz : SME_MOVAZ_TileToVector_Intrinsic;
30173017
def int_aarch64_sme_readz_vert : SME_MOVAZ_TileToVector_Intrinsic;
@@ -3022,12 +3022,12 @@ let TargetPrefix = "aarch64" in {
30223022
def int_aarch64_sme_readz_x2
30233023
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
30243024
[llvm_i32_ty],
3025-
[IntrNoMem, IntrHasSideEffects]>;
3025+
[IntrInaccessibleMemOnly]>;
30263026

30273027
def int_aarch64_sme_readz_x4
30283028
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
30293029
[llvm_i32_ty],
3030-
[IntrNoMem, IntrHasSideEffects]>;
3030+
[IntrInaccessibleMemOnly]>;
30313031

30323032
def int_aarch64_sme_write_lane_zt
30333033
: DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_anyvector_ty, llvm_i32_ty],
@@ -3038,7 +3038,7 @@ let TargetPrefix = "aarch64" in {
30383038
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly, IntrWriteMem]>;
30393039

30403040

3041-
def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
3041+
def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30423042
def int_aarch64_sme_in_streaming_mode : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrNoMem]>, ClangBuiltin<"__builtin_arm_in_streaming_mode">;
30433043

30443044
class SME_OuterProduct_Intrinsic
@@ -3047,7 +3047,7 @@ let TargetPrefix = "aarch64" in {
30473047
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
30483048
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
30493049
LLVMMatchType<0>,
3050-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
3050+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
30513051

30523052
def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic;
30533053
def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic;
@@ -3112,7 +3112,7 @@ let TargetPrefix = "aarch64" in {
31123112
[llvm_i32_ty,
31133113
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
31143114
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
3115-
llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>;
3115+
llvm_anyvector_ty], [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
31163116

31173117
def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic;
31183118
def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic;
@@ -3232,56 +3232,56 @@ let TargetPrefix = "aarch64" in {
32323232
: DefaultAttrsIntrinsic<[],
32333233
[llvm_i32_ty,
32343234
llvm_anyvector_ty, LLVMMatchType<0>],
3235-
[]>;
3235+
[IntrInaccessibleMemOnly]>;
32363236

32373237
class SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic
32383238
: DefaultAttrsIntrinsic<[],
32393239
[llvm_i32_ty,
32403240
llvm_anyvector_ty, LLVMMatchType<0>,
32413241
LLVMMatchType<0>],
3242-
[]>;
3242+
[IntrInaccessibleMemOnly]>;
32433243

32443244
class SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic
32453245
: DefaultAttrsIntrinsic<[],
32463246
[llvm_i32_ty,
32473247
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32483248
LLVMMatchType<0>],
3249-
[]>;
3249+
[IntrInaccessibleMemOnly]>;
32503250

32513251
class SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic
32523252
: DefaultAttrsIntrinsic<[],
32533253
[llvm_i32_ty,
32543254
llvm_anyvector_ty, LLVMMatchType<0>,
32553255
LLVMMatchType<0>, LLVMMatchType<0>],
3256-
[]>;
3256+
[IntrInaccessibleMemOnly]>;
32573257

32583258
class SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic
32593259
: DefaultAttrsIntrinsic<[],
32603260
[llvm_i32_ty,
32613261
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32623262
LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
3263-
[]>;
3263+
[IntrInaccessibleMemOnly]>;
32643264

32653265
class SME2_Matrix_ArrayVector_Single_Index_Intrinsic
32663266
: DefaultAttrsIntrinsic<[],
32673267
[llvm_i32_ty,
32683268
llvm_anyvector_ty,
32693269
LLVMMatchType<0>, llvm_i32_ty],
3270-
[ImmArg<ArgIndex<3>>]>;
3270+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
32713271

32723272
class SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic
32733273
: DefaultAttrsIntrinsic<[],
32743274
[llvm_i32_ty,
32753275
llvm_anyvector_ty, LLVMMatchType<0>,
32763276
LLVMMatchType<0>, llvm_i32_ty],
3277-
[ImmArg<ArgIndex<4>>]>;
3277+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<4>>]>;
32783278

32793279
class SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic
32803280
: DefaultAttrsIntrinsic<[],
32813281
[llvm_i32_ty,
32823282
llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
32833283
LLVMMatchType<0>, llvm_i32_ty],
3284-
[ImmArg<ArgIndex<6>>]>;
3284+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<6>>]>;
32853285

32863286
class SME2_VG2_Multi_Imm_Intrinsic
32873287
: DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
@@ -3300,14 +3300,14 @@ let TargetPrefix = "aarch64" in {
33003300
: DefaultAttrsIntrinsic<[],
33013301
[llvm_i32_ty,
33023302
llvm_anyvector_ty, LLVMMatchType<0>],
3303-
[]>;
3303+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
33043304

33053305
class SME2_ZA_Write_VG4_Intrinsic
33063306
: DefaultAttrsIntrinsic<[],
33073307
[llvm_i32_ty,
33083308
llvm_anyvector_ty, LLVMMatchType<0>,
33093309
LLVMMatchType<0>, LLVMMatchType<0>],
3310-
[]>;
3310+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
33113311

33123312
class SME2_VG2_Multi_Single_Intrinsic
33133313
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
@@ -3396,50 +3396,50 @@ let TargetPrefix = "aarch64" in {
33963396
class SME2_ZA_ArrayVector_Read_VG2_Intrinsic
33973397
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
33983398
[llvm_i32_ty],
3399-
[]>;
3399+
[IntrReadMem, IntrInaccessibleMemOnly]>;
34003400

34013401
class SME2_ZA_ArrayVector_Read_VG4_Intrinsic
34023402
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
34033403
LLVMMatchType<0>, LLVMMatchType<0>],
34043404
[llvm_i32_ty],
3405-
[]>;
3405+
[IntrReadMem, IntrInaccessibleMemOnly]>;
34063406

34073407
class SME2_Matrix_TileVector_Read_VG2_Intrinsic
34083408
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
34093409
[llvm_i32_ty, llvm_i32_ty],
3410-
[]>;
3410+
[IntrReadMem, IntrInaccessibleMemOnly]>;
34113411

34123412
class SME2_Matrix_TileVector_Read_VG4_Intrinsic
34133413
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
34143414
LLVMMatchType<0>, LLVMMatchType<0>],
34153415
[llvm_i32_ty, llvm_i32_ty],
3416-
[]>;
3416+
[IntrReadMem, IntrInaccessibleMemOnly]>;
34173417

34183418
class SME2_ZA_ArrayVector_Write_VG2_Intrinsic
34193419
: DefaultAttrsIntrinsic<[],
34203420
[llvm_i32_ty,
34213421
llvm_anyvector_ty, LLVMMatchType<0>],
3422-
[]>;
3422+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
34233423

34243424
class SME2_ZA_ArrayVector_Write_VG4_Intrinsic
34253425
: DefaultAttrsIntrinsic<[],
34263426
[llvm_i32_ty,
34273427
llvm_anyvector_ty, LLVMMatchType<0>,
34283428
LLVMMatchType<0>, LLVMMatchType<0>],
3429-
[]>;
3429+
[IntrWriteMem, IntrInaccessibleMemOnly]>;
34303430

34313431
class SME2_Matrix_TileVector_Write_VG2_Intrinsic
34323432
: DefaultAttrsIntrinsic<[],
34333433
[llvm_i32_ty, llvm_i32_ty,
34343434
llvm_anyvector_ty, LLVMMatchType<0>],
3435-
[ImmArg<ArgIndex<0>>]>;
3435+
[IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
34363436

34373437
class SME2_Matrix_TileVector_Write_VG4_Intrinsic
34383438
: DefaultAttrsIntrinsic<[],
34393439
[llvm_i32_ty, llvm_i32_ty,
34403440
llvm_anyvector_ty, LLVMMatchType<0>,
34413441
LLVMMatchType<0>, LLVMMatchType<0>],
3442-
[ImmArg<ArgIndex<0>>]>;
3442+
[IntrWriteMem, IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>]>;
34433443

34443444
class SME2_VG2_Multi_Single_Single_Intrinsic
34453445
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
@@ -3605,7 +3605,7 @@ let TargetPrefix = "aarch64" in {
36053605
// Multi-vector zeroing
36063606

36073607
foreach vg = ["vg1x2", "vg1x4", "vg2x1", "vg2x2", "vg2x4", "vg4x1", "vg4x2", "vg4x4"] in {
3608-
def int_aarch64_sme_zero_za64_ # vg : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects]>;
3608+
def int_aarch64_sme_zero_za64_ # vg : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;
36093609
}
36103610

36113611
// Multi-vector signed saturating doubling multiply high
@@ -4045,57 +4045,57 @@ let TargetPrefix = "aarch64" in {
40454045
[llvm_i32_ty,
40464046
llvm_nxv16i1_ty, llvm_nxv16i1_ty,
40474047
llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4048-
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4048+
[ImmArg<ArgIndex<0>>, IntrInaccessibleMemOnly]>;
40494049

40504050
class SME_FP8_ZA_LANE_VGx1_Intrinsic
40514051
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40524052
llvm_nxv16i8_ty,
40534053
llvm_nxv16i8_ty,
40544054
llvm_i32_ty],
4055-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<3>>]>;
4055+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<3>>]>;
40564056

40574057
class SME_FP8_ZA_LANE_VGx2_Intrinsic
40584058
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40594059
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40604060
llvm_nxv16i8_ty,
40614061
llvm_i32_ty],
4062-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<4>>]>;
4062+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<4>>]>;
40634063

40644064
class SME_FP8_ZA_LANE_VGx4_Intrinsic
40654065
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40664066
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40674067
llvm_nxv16i8_ty,
40684068
llvm_i32_ty],
4069-
[IntrInaccessibleMemOnly, IntrHasSideEffects, ImmArg<ArgIndex<6>>]>;
4069+
[IntrInaccessibleMemOnly, ImmArg<ArgIndex<6>>]>;
40704070
class SME_FP8_ZA_SINGLE_VGx1_Intrinsic
40714071
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40724072
llvm_nxv16i8_ty,
40734073
llvm_nxv16i8_ty],
4074-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4074+
[IntrInaccessibleMemOnly]>;
40754075

40764076
class SME_FP8_ZA_SINGLE_VGx2_Intrinsic
40774077
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40784078
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40794079
llvm_nxv16i8_ty],
4080-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4080+
[IntrInaccessibleMemOnly]>;
40814081

40824082
class SME_FP8_ZA_SINGLE_VGx4_Intrinsic
40834083
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40844084
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40854085
llvm_nxv16i8_ty],
4086-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4086+
[IntrInaccessibleMemOnly]>;
40874087

40884088
class SME_FP8_ZA_MULTI_VGx2_Intrinsic
40894089
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40904090
llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40914091
llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4092-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4092+
[IntrInaccessibleMemOnly]>;
40934093

40944094
class SME_FP8_ZA_MULTI_VGx4_Intrinsic
40954095
: DefaultAttrsIntrinsic<[], [llvm_i32_ty,
40964096
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty,
40974097
llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty, llvm_nxv16i8_ty],
4098-
[IntrInaccessibleMemOnly, IntrHasSideEffects]>;
4098+
[IntrInaccessibleMemOnly]>;
40994099
//
41004100
// CVT from FP8 to half-precision/BFloat16 multi-vector
41014101
//

0 commit comments

Comments
 (0)