[AArch64] Assembly support for the Armv9.5-A Memory System Extensions (#76237)

pratlucas · web-flow · commit e4f1c528326f · 2023-12-22T14:40:29.000Z
This implements assembly support for the Memory Systems Extensions
introduced as part of the Armv9.5-A architecture version.
The changes include:
* New subtarget feature for FEAT_TLBIW.
* New system registers for FEAT_HDBSS:
  * HDBSSBR_EL2 and HDBSSPROD_EL2.
* New system registers for FEAT_HACDBS:
  * HACDBSBR_EL2 and HACDBSCONS_EL2.
* New TLBI instructions for FEAT_TLBIW:
  * VMALLWS2E1(nXS), VMALLWS2E1IS(nXS) and VMALLWS2E1OS(nXS).
* New system register for FEAT_FGWTE3:
  * FGWTE3_EL3.
diff --git a/clang/test/Driver/aarch64-v95a.c b/clang/test/Driver/aarch64-v95a.c
@@ -25,3 +25,7 @@
 // RUN: %clang -target aarch64 -march=armv9.5a+pauth-lr -### -c %s 2>&1 | FileCheck -check-prefix=V95A-PAUTHLR %s
 // RUN: %clang -target aarch64 -march=armv9.5-a+pauth-lr -### -c %s 2>&1 | FileCheck -check-prefix=V95A-PAUTHLR %s
 // V95A-PAUTHLR: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v9.5a" "-target-feature" "+pauth-lr"
+
+// RUN: %clang -target aarch64 -march=armv9.5a+tlbiw -### -c %s 2>&1 | FileCheck -check-prefix=V95A-TLBIW %s
+// RUN: %clang -target aarch64 -march=armv9.5-a+tlbiw -### -c %s 2>&1 | FileCheck -check-prefix=V95A-TLBIW %s
+// V95A-TLBIW: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v9.5a" "-target-feature" "+tlbiw"
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -175,6 +175,7 @@ enum ArchExtKind : unsigned {
   AEK_SMEFA64 =       71, // FEAT_SME_FA64
   AEK_CPA =           72, // FEAT_CPA
   AEK_PAUTHLR =       73, // FEAT_PAuth_LR
+  AEK_TLBIW =         74, // FEAT_TLBIW
   AEK_NUM_EXTENSIONS
 };
 using ExtensionBitset = Bitset<AEK_NUM_EXTENSIONS>;
@@ -299,6 +300,7 @@ inline constexpr ExtensionInfo Extensions[] = {
     {"sme-fa64",  AArch64::AEK_SMEFA64,  "+sme-fa64", "-sme-fa64",  FEAT_INIT, "", 0},
     {"cpa", AArch64::AEK_CPA, "+cpa", "-cpa", FEAT_INIT, "", 0},
     {"pauth-lr", AArch64::AEK_PAUTHLR, "+pauth-lr", "-pauth-lr", FEAT_INIT, "", 0},
+    {"tlbiw", AArch64::AEK_TLBIW, "+tlbiw", "-tlbiw", FEAT_INIT, "", 0},
     // Special cases
     {"none", AArch64::AEK_NONE, {}, {}, FEAT_INIT, "", ExtensionInfo::MaxFMVPriority},
 };
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
@@ -630,6 +630,9 @@ def FeatureCPA : SubtargetFeature<"cpa", "HasCPA", "true",
 def FeaturePAuthLR : SubtargetFeature<"pauth-lr", "HasPAuthLR",
     "true", "Enable Armv9.5-A PAC enhancements (FEAT_PAuth_LR)">;
 
+def FeatureTLBIW : SubtargetFeature<"tlbiw", "HasTLBIW", "true",
+  "Enable ARMv9.5-A TLBI VMALL for Dirty State (FEAT_TLBIW)">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -643,6 +643,14 @@ defm : TLBI<"PAALLOS",      0b110, 0b1000, 0b0001, 0b100, 0>;
 defm : TLBI<"PAALL",        0b110, 0b1000, 0b0111, 0b100, 0>;
 }
 
+// Armv9.5-A TLBI VMALL for Dirty State
+let Requires = ["AArch64::FeatureTLBIW"] in {
+//                           op1,   CRn,    CRm,    op2,   needsreg
+defm : TLBI<"VMALLWS2E1",    0b100, 0b1000, 0b0110, 0b010, 0>;
+defm : TLBI<"VMALLWS2E1IS",  0b100, 0b1000, 0b0010, 0b010, 0>;
+defm : TLBI<"VMALLWS2E1OS",  0b100, 0b1000, 0b0101, 0b010, 0>;
+}
+
 //===----------------------------------------------------------------------===//
 // MRS/MSR (system register read/write) instruction options.
 //===----------------------------------------------------------------------===//
@@ -1951,3 +1959,17 @@ def : WOSysReg<"SPMZR_EL0",         0b10, 0b011, 0b1001, 0b1100, 0b100>;
 //                                  Op0   Op1    CRn     CRm     Op2
 def : RWSysReg<"VDISR_EL3",         0b11, 0b110, 0b1100, 0b0001, 0b001>;
 def : RWSysReg<"VSESR_EL3",         0b11, 0b110, 0b0101, 0b0010, 0b011>;
+
+// v9.5a Hardware Dirty State Tracking Structure (FEAT_HDBSS)
+//                                  Op0   Op1    CRn     CRm     Op2
+def : RWSysReg<"HDBSSBR_EL2",       0b11, 0b100, 0b0010, 0b0011, 0b010>;
+def : RWSysReg<"HDBSSPROD_EL2",     0b11, 0b100, 0b0010, 0b0011, 0b011>;
+
+// v9.5a Hardware Accelerator for Cleaning Dirty State (FEAT_HACDBS)
+//                                  Op0   Op1    CRn     CRm     Op2
+def : RWSysReg<"HACDBSBR_EL2",      0b11, 0b100, 0b0010, 0b0011, 0b100>;
+def : RWSysReg<"HACDBSCONS_EL2",    0b11, 0b100, 0b0010, 0b0011, 0b101>;
+
+// v9.5a Fine Grained Write Trap EL3 (FEAT_FGWTE3)
+//                                  Op0   Op1    CRn     CRm     Op2
+def : RWSysReg<"FGWTE3_EL3",        0b11, 0b110, 0b0001, 0b0001, 0b101>;
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3706,6 +3706,7 @@ static const struct Extension {
     {"sme-f8f32", {AArch64::FeatureSMEF8F32}},
     {"sme-fa64",  {AArch64::FeatureSMEFA64}},
     {"cpa", {AArch64::FeatureCPA}},
+    {"tlbiw", {AArch64::FeatureTLBIW}},
 };
 
 static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
diff --git a/llvm/test/MC/AArch64/armv9.5a-fgwte3.s b/llvm/test/MC/AArch64/armv9.5a-fgwte3.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+
+mrs x0, FGWTE3_EL3
+// CHECK: mrs x0, FGWTE3_EL3                  // encoding: [0xa0,0x11,0x3e,0xd5]
+msr FGWTE3_EL3, x0
+// CHECK: msr FGWTE3_EL3, x0                  // encoding: [0xa0,0x11,0x1e,0xd5]
diff --git a/llvm/test/MC/AArch64/armv9.5a-hacdbs.s b/llvm/test/MC/AArch64/armv9.5a-hacdbs.s
@@ -0,0 +1,12 @@
+// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+
+mrs x0, HACDBSBR_EL2
+// CHECK: mrs x0, HACDBSBR_EL2                  // encoding: [0x80,0x23,0x3c,0xd5]
+msr HACDBSBR_EL2, x0
+// CHECK: msr HACDBSBR_EL2, x0                  // encoding: [0x80,0x23,0x1c,0xd5]
+
+mrs x0, HACDBSCONS_EL2
+// CHECK: mrs x0, HACDBSCONS_EL2                  // encoding: [0xa0,0x23,0x3c,0xd5]
+msr HACDBSCONS_EL2, x0
+// CHECK: msr HACDBSCONS_EL2, x0                  // encoding: [0xa0,0x23,0x1c,0xd5]
+
diff --git a/llvm/test/MC/AArch64/armv9.5a-hdbss.s b/llvm/test/MC/AArch64/armv9.5a-hdbss.s
@@ -0,0 +1,12 @@
+// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+
+mrs x0, HDBSSBR_EL2
+// CHECK: mrs x0, HDBSSBR_EL2                  // encoding: [0x40,0x23,0x3c,0xd5]
+msr HDBSSBR_EL2, x0
+// CHECK: msr HDBSSBR_EL2, x0                  // encoding: [0x40,0x23,0x1c,0xd5]
+
+mrs x0, HDBSSPROD_EL2
+// CHECK: mrs x0, HDBSSPROD_EL2                  // encoding: [0x60,0x23,0x3c,0xd5]
+msr HDBSSPROD_EL2, x0
+// CHECK: msr HDBSSPROD_EL2, x0                  // encoding: [0x60,0x23,0x1c,0xd5]
+
diff --git a/llvm/test/MC/AArch64/armv9.5a-tlbiw.s b/llvm/test/MC/AArch64/armv9.5a-tlbiw.s
@@ -0,0 +1,27 @@
+// RUN: llvm-mc -triple aarch64 -show-encoding -mattr=+tlbiw -mattr=+xs < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-XS %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+tlbiw < %s 2> %t | FileCheck --check-prefix=CHECK-TLBIW %s && FileCheck --check-prefix=ERROR-NO-XS-TLBIW %s < %t
+// RUN: not llvm-mc -triple aarch64 < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-TLBIW --check-prefix=ERROR-NO-XS-TLBIW %s
+
+tlbi VMALLWS2E1
+// CHECK-TLBIW: tlbi vmallws2e1                  // encoding: [0x5f,0x86,0x0c,0xd5]
+// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1 requires: tlbiw
+
+tlbi VMALLWS2E1IS
+// CHECK-TLBIW: tlbi vmallws2e1is                // encoding: [0x5f,0x82,0x0c,0xd5]
+// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1IS requires: tlbiw
+
+tlbi VMALLWS2E1OS
+// CHECK-TLBIW: tlbi vmallws2e1os                // encoding: [0x5f,0x85,0x0c,0xd5]
+// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1OS requires: tlbiw
+
+tlbi VMALLWS2E1nXS
+// CHECK-XS: tlbi vmallws2e1nxs                  // encoding: [0x5f,0x96,0x0c,0xd5]
+// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1nXS requires: xs, tlbiw
+
+tlbi VMALLWS2E1ISnXS
+// CHECK-XS: tlbi vmallws2e1isnxs                // encoding: [0x5f,0x92,0x0c,0xd5]
+// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1ISnXS requires: xs, tlbiw
+
+tlbi VMALLWS2E1OSnXS
+// CHECK-XS: tlbi vmallws2e1osnxs                // encoding: [0x5f,0x95,0x0c,0xd5]
+// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1OSnXS requires: xs, tlbiw
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-fgwte3.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-fgwte3.txt
@@ -0,0 +1,7 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+[0xa0,0x11,0x3e,0xd5]
+# CHECK: mrs x0, FGWTE3_EL3
+
+[0xa0,0x11,0x1e,0xd5]
+# CHECK: msr FGWTE3_EL3, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hacdbs.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-hacdbs.txt
@@ -0,0 +1,14 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+[0x80,0x23,0x3c,0xd5]
+# CHECK: mrs x0, HACDBSBR_EL2
+
+[0x80,0x23,0x1c,0xd5]
+# CHECK: msr HACDBSBR_EL2, x0
+
+[0xa0,0x23,0x3c,0xd5]
+# CHECK: mrs x0, HACDBSCONS_EL2
+
+[0xa0,0x23,0x1c,0xd5]
+# CHECK: msr HACDBSCONS_EL2, x0
+
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hdbss.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-hdbss.txt
@@ -0,0 +1,14 @@
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
+
+[0x40,0x23,0x3c,0xd5]
+# CHECK: mrs x0, HDBSSBR_EL2
+
+[0x40,0x23,0x1c,0xd5]
+# CHECK: msr HDBSSBR_EL2, x0
+
+[0x60,0x23,0x3c,0xd5]
+# CHECK: mrs x0, HDBSSPROD_EL2
+
+[0x60,0x23,0x1c,0xd5]
+# CHECK: msr HDBSSPROD_EL2, x0
+
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-tlbiw.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-tlbiw.txt
@@ -0,0 +1,27 @@
+# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+tlbiw -mattr=+xs < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-XS %s
+# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+tlbiw < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-NO-XS-TLBIW %s
+# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck --check-prefix=CHECK-NO-TLBIW --check-prefix=CHECK-NO-XS-TLBIW %s
+
+[0x5f,0x86,0x0c,0xd5]
+# CHECK-TLBIW: tlbi vmallws2e1
+# CHECK-NO-TLBIW: sys #4, c8, c6, #2
+
+[0x5f,0x82,0x0c,0xd5]
+# CHECK-TLBIW: tlbi vmallws2e1is
+# CHECK-NO-TLBIW: sys #4, c8, c2, #2
+
+[0x5f,0x85,0x0c,0xd5]
+# CHECK-TLBIW: tlbi vmallws2e1os
+# CHECK-NO-TLBIW: sys #4, c8, c5, #2
+
+[0x5f,0x96,0x0c,0xd5]
+# CHECK-XS: tlbi vmallws2e1nxs
+# CHECK-NO-XS-TLBIW: sys #4, c9, c6, #2
+
+[0x5f,0x92,0x0c,0xd5]
+# CHECK-XS: tlbi vmallws2e1isnxs
+# CHECK-NO-XS-TLBIW: sys #4, c9, c2, #2
+
+[0x5f,0x95,0x0c,0xd5]
+# CHECK-XS: tlbi vmallws2e1osnxs
+# CHECK-NO-XS-TLBIW: sys #4, c9, c5, #2
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1813,6 +1813,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_SME_LUTv2,    AArch64::AEK_SMEF8F16,
       AArch64::AEK_SMEF8F32,     AArch64::AEK_SMEFA64,
       AArch64::AEK_CPA,          AArch64::AEK_PAUTHLR,
+      AArch64::AEK_TLBIW,
   };
 
   std::vector<StringRef> Features;
@@ -1901,6 +1902,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+sme-fa64"));
   EXPECT_TRUE(llvm::is_contained(Features, "+cpa"));
   EXPECT_TRUE(llvm::is_contained(Features, "+pauth-lr"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+tlbiw"));
 
   // Assuming we listed every extension above, this should produce the same
   // result. (note that AEK_NONE doesn't have a name so it won't be in the
