[AMDGPU] gfx11 LDSDIR instructions MC support

Contributors:
Carl Ritson <[email protected]>

Patch 8/N for upstreaming of AMDGPU gfx11 architecture.

Depends on D125498

Reviewed By: critson, rampitec, #amdgpu

Differential Revision: https://reviews.llvm.org/D125820

Author: Sisyph

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

@@ -161,6 +161,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     ImmTyCBSZ,
     ImmTyABID,
     ImmTyEndpgm,
+    ImmTyWaitVDST,
   };
 
   enum ImmKindTy {
@@ -835,6 +836,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
   bool isS16Imm() const;
   bool isU16Imm() const;
   bool isEndpgm() const;
+  bool isWaitVDST() const;
 
   StringRef getExpressionAsToken() const {
     assert(isExpr());
@@ -1042,6 +1044,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     case ImmTyCBSZ: OS << "CBSZ"; break;
     case ImmTyABID: OS << "ABID"; break;
     case ImmTyEndpgm: OS << "Endpgm"; break;
+    case ImmTyWaitVDST: OS << "WaitVDST"; break;
     }
   }
 
@@ -1758,6 +1761,9 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
 
   OperandMatchResultTy parseEndpgmOp(OperandVector &Operands);
   AMDGPUOperand::Ptr defaultEndpgmImmOperands() const;
+
+  OperandMatchResultTy parseWaitVDST(OperandVector &Operands);
+  AMDGPUOperand::Ptr defaultWaitVDST() const;
 };
 
 struct OptionalOperand {
@@ -3969,7 +3975,7 @@ Optional<StringRef> AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
     const auto &Src = Inst.getOperand(SrcIdx);
     if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
 
-      if (isGFX90A())
+      if (isGFX90A() || isGFX11Plus())
         return StringRef("lds_direct is not supported on this GPU");
 
       if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA))
@@ -7834,7 +7840,8 @@ static const OptionalOperand AMDGPUOptionalOperandTable[] = {
   {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr},
   {"blgp", AMDGPUOperand::ImmTyBLGP, false, nullptr},
   {"cbsz", AMDGPUOperand::ImmTyCBSZ, false, nullptr},
-  {"abid", AMDGPUOperand::ImmTyABID, false, nullptr}
+  {"abid", AMDGPUOperand::ImmTyABID, false, nullptr},
+  {"wait_vdst", AMDGPUOperand::ImmTyWaitVDST, false, nullptr}
 };
 
 void AMDGPUAsmParser::onBeginOfFile() {
@@ -8839,3 +8846,15 @@ OperandMatchResultTy AMDGPUAsmParser::parseEndpgmOp(OperandVector &Operands) {
 }
 
 bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmTyEndpgm); }
+
+//===----------------------------------------------------------------------===//
+// LDSDIR
+//===----------------------------------------------------------------------===//
+
+AMDGPUOperand::Ptr AMDGPUAsmParser::defaultWaitVDST() const {
+  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyWaitVDST);
+}
+
+bool AMDGPUOperand::isWaitVDST() const {
+  return isImmTy(ImmTyWaitVDST) && isUInt<4>(getImm());
+}

llvm/lib/Target/AMDGPU/LDSDIRInstructions.td

@@ -0,0 +1,106 @@
+//===-- LDSDIRInstructions.td - LDS Direct Instruction Definitions --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// LDSDIR encoding
+//===----------------------------------------------------------------------===//
+
+class LDSDIRe<bits<2> op, bit is_direct> : Enc32 {
+  // encoding fields
+  bits<2> attrchan;
+  bits<6> attr;
+  bits<4> waitvdst;
+  bits<8> vdst;
+
+  // encoding
+  let Inst{31-24} = 0xce; // encoding
+  let Inst{23-22} = 0x0; // reserved
+  let Inst{21-20} = op;
+  let Inst{19-16} = waitvdst;
+  let Inst{15-10} = !if(is_direct, ?, attr);
+  let Inst{9-8} = !if(is_direct, ?, attrchan);
+  let Inst{7-0} = vdst;
+}
+
+//===----------------------------------------------------------------------===//
+// LDSDIR Classes
+//===----------------------------------------------------------------------===//
+
+class LDSDIR_getIns<bit direct> {
+  dag ret = !if(direct,
+    (ins wait_vdst:$waitvdst),
+    (ins Attr:$attr, AttrChan:$attrchan, wait_vdst:$waitvdst)
+  );
+}
+
+class LDSDIR_Common<string opName, string asm = "", bit direct> : InstSI<
+    (outs VGPR_32:$vdst),
+    LDSDIR_getIns<direct>.ret,
+    asm> {
+  let LDSDIR = 1;
+  let EXP_CNT = 1;
+
+  let hasSideEffects = 0;
+  let mayLoad = 1;
+  let mayStore = 0;
+
+  string Mnemonic = opName;
+  let UseNamedOperandTable = 1;
+
+  let Uses = [M0, EXEC];
+  let DisableWQM = 0;
+  let SchedRW = [WriteLDS];
+
+  bit is_direct;
+  let is_direct = direct;
+}
+
+class LDSDIR_Pseudo<string opName, bit direct> :
+  LDSDIR_Common<opName, "", direct>,
+  SIMCInstr<opName, SIEncodingFamily.NONE> {
+  let isPseudo = 1;
+  let isCodeGenOnly = 1;
+}
+
+class LDSDIR_getAsm<bit direct> {
+  string ret = !if(direct,
+    " $vdst$waitvdst",
+    " $vdst, $attr$attrchan$waitvdst"
+  );
+}
+
+class LDSDIR_Real<bits<2> op, LDSDIR_Pseudo lds, int subtarget> :
+  LDSDIR_Common<lds.Mnemonic,
+                lds.Mnemonic # LDSDIR_getAsm<lds.is_direct>.ret,
+                lds.is_direct>,
+  SIMCInstr <lds.Mnemonic, subtarget>,
+  LDSDIRe<op, lds.is_direct> {
+  let isPseudo = 0;
+  let isCodeGenOnly = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// LDS Direct Instructions
+//===----------------------------------------------------------------------===//
+
+def LDS_DIRECT_LOAD : LDSDIR_Pseudo<"lds_direct_load", 1>;
+def LDS_PARAM_LOAD : LDSDIR_Pseudo<"lds_param_load", 0>;
+
+//===----------------------------------------------------------------------===//
+// GFX11+
+//===----------------------------------------------------------------------===//
+
+multiclass LDSDIR_Real_gfx11<bits<2> op, LDSDIR_Pseudo lds = !cast<LDSDIR_Pseudo>(NAME)> {
+  def _gfx11 : LDSDIR_Real<op, lds, SIEncodingFamily.GFX11> {
+    let AssemblerPredicate = isGFX11Plus;
+    let DecoderNamespace = "GFX11";
+  }
+}
+
+defm LDS_PARAM_LOAD : LDSDIR_Real_gfx11<0x0>;
+defm LDS_DIRECT_LOAD : LDSDIR_Real_gfx11<0x1>;

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp

@@ -595,6 +595,16 @@ void AMDGPUInstPrinter::printDefaultVccOperand(unsigned OpNo,
     O << ", ";
 }
 
+void AMDGPUInstPrinter::printWaitVDST(const MCInst *MI, unsigned OpNo,
+                                      const MCSubtargetInfo &STI,
+                                      raw_ostream &O) {
+  uint8_t Imm = MI->getOperand(OpNo).getImm();
+  if (Imm != 0) {
+    O << " wait_vdst:";
+    printU4ImmDecOperand(MI, OpNo, O);
+  }
+}
+
 void AMDGPUInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                      const MCSubtargetInfo &STI,
                                      raw_ostream &O) {

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h

@@ -175,6 +175,8 @@ class AMDGPUInstPrinter : public MCInstPrinter {
                  raw_ostream &O);
   void printDefaultVccOperand(unsigned OpNo, const MCSubtargetInfo &STI,
                               raw_ostream &O);
+  void printWaitVDST(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
+                    raw_ostream &O);
 
   void printExpSrcN(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O, unsigned N);

llvm/lib/Target/AMDGPU/SIDefines.h

@@ -63,6 +63,9 @@ enum : uint64_t {
   VGPRSpill = 1 << 24,
   SGPRSpill = 1 << 25,
 
+  // LDSDIR instruction format.
+  LDSDIR = 1 << 26,
+
   // High bits - other information.
   VM_CNT = UINT64_C(1) << 32,
   EXP_CNT = UINT64_C(1) << 33,

llvm/lib/Target/AMDGPU/SIInstrFormats.td

@@ -48,6 +48,9 @@ class InstSI <dag outs, dag ins, string asm = "",
   field bit VGPRSpill = 0;
   field bit SGPRSpill = 0;
 
+  // LDSDIR instruction format.
+  field bit LDSDIR = 0;
+
   // High bits - other information.
   field bit VM_CNT = 0;
   field bit EXP_CNT = 0;
@@ -173,6 +176,8 @@ class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{24} = VGPRSpill;
   let TSFlags{25} = SGPRSpill;
 
+  let TSFlags{26} = LDSDIR;
+
   let TSFlags{32} = VM_CNT;
   let TSFlags{33} = EXP_CNT;
   let TSFlags{34} = LGKM_CNT;

llvm/lib/Target/AMDGPU/SIInstrInfo.h

@@ -666,6 +666,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
   }
 
+  static bool isLDSDIR(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
+  }
+
+  bool isLDSDIR(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::LDSDIR;
+  }
+
   static bool isScalarUnit(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
   }

llvm/lib/Target/AMDGPU/SIInstrInfo.td

@@ -1291,6 +1291,8 @@ def exp_tgt : NamedOperandU32<"ExpTgt", NamedMatchClass<"ExpTgt", 0>> {
 
 }
 
+def wait_vdst : NamedOperandU8<"WaitVDST", NamedMatchClass<"WaitVDST">>;
+
 } // End OperandType = "OPERAND_IMMEDIATE"
 
 class KImmMatchClass<int size> : AsmOperandClass {

llvm/lib/Target/AMDGPU/SIInstructions.td

@@ -30,6 +30,7 @@ include "SMInstructions.td"
 include "FLATInstructions.td"
 include "BUFInstructions.td"
 include "EXPInstructions.td"
+include "LDSDIRInstructions.td"
 
 //===----------------------------------------------------------------------===//
 // VINTRP Instructions

llvm/test/MC/AMDGPU/gfx11_err.s

@@ -20,3 +20,9 @@ s_delay_alu instid0(VALU_DEP_1) | (SALU_CYCLE_1)
 
 s_delay_alu instid0(VALU_DEP_1) | SALU_CYCLE_1)
 // GFX11: [[@LINE-1]]:{{[0-9]+}}: error: expected a left parenthesis
+
+lds_direct_load v15 wait_vdst:16
+// GFX11: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+lds_direct_load v15 wait_vdst
+// GFX11: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction

llvm/test/MC/AMDGPU/ldsdir.s

@@ -0,0 +1,103 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX11 %s
+
+lds_direct_load v1 wait_vdst:15
+// GFX11: lds_direct_load v1 wait_vdst:15  ; encoding: [0x01,0x00,0x1f,0xce]
+
+lds_direct_load v2 wait_vdst:14
+// GFX11: lds_direct_load v2 wait_vdst:14  ; encoding: [0x02,0x00,0x1e,0xce]
+
+lds_direct_load v3 wait_vdst:13
+// GFX11: lds_direct_load v3 wait_vdst:13  ; encoding: [0x03,0x00,0x1d,0xce]
+
+lds_direct_load v4 wait_vdst:12
+// GFX11: lds_direct_load v4 wait_vdst:12  ; encoding: [0x04,0x00,0x1c,0xce]
+
+lds_direct_load v5 wait_vdst:11
+// GFX11: lds_direct_load v5 wait_vdst:11  ; encoding: [0x05,0x00,0x1b,0xce]
+
+lds_direct_load v6 wait_vdst:10
+// GFX11: lds_direct_load v6 wait_vdst:10  ; encoding: [0x06,0x00,0x1a,0xce]
+
+lds_direct_load v7 wait_vdst:9
+// GFX11: lds_direct_load v7 wait_vdst:9   ; encoding: [0x07,0x00,0x19,0xce]
+
+lds_direct_load v8 wait_vdst:8
+// GFX11: lds_direct_load v8 wait_vdst:8   ; encoding: [0x08,0x00,0x18,0xce]
+
+lds_direct_load v9 wait_vdst:7
+// GFX11: lds_direct_load v9 wait_vdst:7   ; encoding: [0x09,0x00,0x17,0xce]
+
+lds_direct_load v10 wait_vdst:6
+// GFX11: lds_direct_load v10 wait_vdst:6  ; encoding: [0x0a,0x00,0x16,0xce]
+
+lds_direct_load v11 wait_vdst:5
+// GFX11: lds_direct_load v11 wait_vdst:5  ; encoding: [0x0b,0x00,0x15,0xce]
+
+lds_direct_load v12 wait_vdst:4
+// GFX11: lds_direct_load v12 wait_vdst:4  ; encoding: [0x0c,0x00,0x14,0xce]
+
+lds_direct_load v13 wait_vdst:3
+// GFX11: lds_direct_load v13 wait_vdst:3  ; encoding: [0x0d,0x00,0x13,0xce]
+
+lds_direct_load v14 wait_vdst:2
+// GFX11: lds_direct_load v14 wait_vdst:2  ; encoding: [0x0e,0x00,0x12,0xce]
+
+lds_direct_load v15 wait_vdst:1
+// GFX11: lds_direct_load v15 wait_vdst:1  ; encoding: [0x0f,0x00,0x11,0xce]
+
+lds_direct_load v16 wait_vdst:0
+// GFX11: lds_direct_load v16  ; encoding: [0x10,0x00,0x10,0xce]
+
+lds_direct_load v17
+// GFX11: lds_direct_load v17  ; encoding: [0x11,0x00,0x10,0xce]
+
+lds_param_load v1, attr0.x wait_vdst:15
+// GFX11: lds_param_load v1, attr0.x wait_vdst:15   ; encoding: [0x01,0x00,0x0f,0xce]
+
+lds_param_load v2, attr0.y wait_vdst:14
+// GFX11: lds_param_load v2, attr0.y wait_vdst:14   ; encoding: [0x02,0x01,0x0e,0xce]
+
+lds_param_load v3, attr0.z wait_vdst:13
+// GFX11: lds_param_load v3, attr0.z wait_vdst:13   ; encoding: [0x03,0x02,0x0d,0xce]
+
+lds_param_load v4, attr0.w wait_vdst:12
+// GFX11: lds_param_load v4, attr0.w wait_vdst:12   ; encoding: [0x04,0x03,0x0c,0xce]
+
+lds_param_load v5, attr0.x wait_vdst:11
+// GFX11: lds_param_load v5, attr0.x wait_vdst:11   ; encoding: [0x05,0x00,0x0b,0xce]
+
+lds_param_load v6, attr1.x wait_vdst:10
+// GFX11: lds_param_load v6, attr1.x wait_vdst:10   ; encoding: [0x06,0x04,0x0a,0xce]
+
+lds_param_load v7, attr2.y wait_vdst:9
+// GFX11: lds_param_load v7, attr2.y wait_vdst:9    ; encoding: [0x07,0x09,0x09,0xce]
+
+lds_param_load v8, attr3.z wait_vdst:8
+// GFX11: lds_param_load v8, attr3.z wait_vdst:8    ; encoding: [0x08,0x0e,0x08,0xce]
+
+lds_param_load v9, attr4.w wait_vdst:7
+// GFX11: lds_param_load v9, attr4.w wait_vdst:7    ; encoding: [0x09,0x13,0x07,0xce]
+
+lds_param_load v10, attr11.x wait_vdst:6
+// GFX11: lds_param_load v10, attr11.x wait_vdst:6  ; encoding: [0x0a,0x2c,0x06,0xce]
+
+lds_param_load v11, attr22.y wait_vdst:5
+// GFX11: lds_param_load v11, attr22.y wait_vdst:5  ; encoding: [0x0b,0x59,0x05,0xce]
+
+lds_param_load v12, attr33.z wait_vdst:4
+// GFX11: lds_param_load v12, attr33.z wait_vdst:4  ; encoding: [0x0c,0x86,0x04,0xce]
+
+lds_param_load v13, attr63.x wait_vdst:3
+// GFX11: lds_param_load v13, attr63.x wait_vdst:3  ; encoding: [0x0d,0xfc,0x03,0xce]
+
+lds_param_load v14, attr63.y wait_vdst:2
+// GFX11: lds_param_load v14, attr63.y wait_vdst:2  ; encoding: [0x0e,0xfd,0x02,0xce]
+
+lds_param_load v15, attr63.z wait_vdst:1
+// GFX11: lds_param_load v15, attr63.z wait_vdst:1  ; encoding: [0x0f,0xfe,0x01,0xce]
+
+lds_param_load v16, attr63.w wait_vdst:0
+// GFX11: lds_param_load v16, attr63.w  ; encoding: [0x10,0xff,0x00,0xce]
+
+lds_param_load v17, attr63.w
+// GFX11: lds_param_load v17, attr63.w  ; encoding: [0x11,0xff,0x00,0xce]