RegMem patch.

Author: JonPsson1

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -1700,7 +1700,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// instruction that defines FoldAsLoadDefReg, and the function returns
   /// the machine instruction generated due to folding.
   virtual MachineInstr *optimizeLoadInstr(MachineInstr &MI,
-                                          const MachineRegisterInfo *MRI,
+                                          MachineRegisterInfo *MRI,
                                           Register &FoldAsLoadDefReg,
                                           MachineInstr *&DefMI) const {
     return nullptr;

llvm/lib/CodeGen/PeepholeOptimizer.cpp

@@ -1868,6 +1868,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
       // If we run into an instruction we can't fold across, discard
       // the load candidates.  Note: We might be able to fold *into* this
       // instruction, so this needs to be after the folding logic.
+      // TODO: Try AA for a store?
       if (MI->isLoadFoldBarrier()) {
         LLVM_DEBUG(dbgs() << "Encountered load fold barrier on " << *MI);
         FoldAsLoadDefCandidates.clear();

llvm/lib/Target/SystemZ/CMakeLists.txt

@@ -20,6 +20,7 @@ add_llvm_target(SystemZCodeGen
   SystemZConstantPoolValue.cpp
   SystemZCopyPhysRegs.cpp
   SystemZElimCompare.cpp
+  SystemZFinalizeRegMem.cpp
   SystemZFrameLowering.cpp
   SystemZHazardRecognizer.cpp
   SystemZISelDAGToDAG.cpp

llvm/lib/Target/SystemZ/SystemZ.h

@@ -195,12 +195,14 @@ FunctionPass *createSystemZShortenInstPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLongBranchPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLDCleanupPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZCopyPhysRegsPass(SystemZTargetMachine &TM);
+FunctionPass *createSystemZFinalizeRegMemPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZPostRewritePass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZTDCPass();
 
 void initializeSystemZCopyPhysRegsPass(PassRegistry &);
 void initializeSystemZDAGToDAGISelPass(PassRegistry &);
 void initializeSystemZElimComparePass(PassRegistry &);
+void initializeSystemZFinalizeRegMemPass(PassRegistry &);
 void initializeSystemZLDCleanupPass(PassRegistry &);
 void initializeSystemZLongBranchPass(PassRegistry &);
 void initializeSystemZPostRewritePass(PassRegistry &);

llvm/lib/Target/SystemZ/SystemZFinalizeRegMem.cpp

@@ -0,0 +1,94 @@
+//===------- SystemZFinalizeRegMem.cpp - Finalize FP reg/mem folding ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass converts any remaining reg/reg pseudos into the real target
+// instruction in cases where the peephole optimizer did not fold a load into
+// a reg/mem instruction.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SystemZMachineFunctionInfo.h"
+#include "SystemZTargetMachine.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+namespace {
+
+class SystemZFinalizeRegMem : public MachineFunctionPass {
+public:
+  static char ID;
+  SystemZFinalizeRegMem()
+    : MachineFunctionPass(ID), TII(nullptr), MRI(nullptr) {
+    initializeSystemZFinalizeRegMemPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+private:
+
+  bool visitMBB(MachineBasicBlock &MBB);
+
+  const SystemZInstrInfo *TII;
+  MachineRegisterInfo *MRI;
+};
+
+char SystemZFinalizeRegMem::ID = 0;
+
+} // end anonymous namespace
+
+INITIALIZE_PASS(SystemZFinalizeRegMem, "systemz-finalize-regmem",
+                "SystemZ Finalize RegMem", false, false)
+
+FunctionPass *llvm::
+createSystemZFinalizeRegMemPass(SystemZTargetMachine &TM) {
+  return new SystemZFinalizeRegMem();
+}
+
+void SystemZFinalizeRegMem::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesCFG();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool SystemZFinalizeRegMem::visitMBB(MachineBasicBlock &MBB) {
+  bool Changed = false;
+  for (MachineInstr &MI : MBB) {
+    unsigned PseudoOpcode = MI.getOpcode();
+    unsigned TargetOpcode =
+      PseudoOpcode == SystemZ::WFADB_CCPseudo    ? SystemZ::WFADB
+      : PseudoOpcode == SystemZ::WFASB_CCPseudo  ? SystemZ::WFASB
+      : PseudoOpcode == SystemZ::WFSDB_CCPseudo  ? SystemZ::WFSDB
+      : PseudoOpcode == SystemZ::WFSSB_CCPseudo  ? SystemZ::WFSSB
+      : 0;
+    if (TargetOpcode) {
+        MI.setDesc(TII->get(TargetOpcode));
+        int CCIdx = MI.findRegisterDefOperandIdx(SystemZ::CC);
+        MI.removeOperand(CCIdx);
+        Changed = true;
+    }
+  }
+  return Changed;
+}
+
+bool SystemZFinalizeRegMem::runOnMachineFunction(MachineFunction &F) {
+  TII = F.getSubtarget<SystemZSubtarget>().getInstrInfo();
+  MRI = &F.getRegInfo();
+
+  bool Modified = false;
+  for (auto &MBB : F)
+    Modified |= visitMBB(MBB);
+
+  return Modified;
+}

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

@@ -692,6 +692,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
   }
 
+  // Don't select reg/mem LDEB if WLDEB is available.
+  if (Subtarget.hasVector())
+    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
+
   // Floating-point truncation and stores need to be done separately.
   setTruncStoreAction(MVT::f64,  MVT::f32, Expand);
   setTruncStoreAction(MVT::f128, MVT::f32, Expand);

llvm/lib/Target/SystemZ/SystemZInstrFP.td

@@ -201,6 +201,7 @@ let Predicates = [FeatureNoVectorEnhancements1] in {
 // Extend memory floating-point values to wider representations.
 let Uses = [FPC], mayRaiseFPException = 1 in {
   def LDEB : UnaryRXE<"ldeb", 0xED04, z_any_extloadf32, FP64, 4>;
+  def LDEB : UnaryRXE<"ldeb", 0xED04, z_fpr_any_extloadf32, FP64, 4>;
   def LXEB : UnaryRXE<"lxeb", 0xED06, null_frag, FP128, 4>;
   def LXDB : UnaryRXE<"lxdb", 0xED05, null_frag, FP128, 8>;
 }
@@ -362,8 +363,8 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
   def SQDBR : UnaryRRE<"sqdbr", 0xB315, any_fsqrt, FP64,  FP64>;
   def SQXBR : UnaryRRE<"sqxbr", 0xB316, any_fsqrt, FP128, FP128>;
 
-  def SQEB : UnaryRXE<"sqeb", 0xED14, loadu<any_fsqrt>, FP32, 4>;
-  def SQDB : UnaryRXE<"sqdb", 0xED15, loadu<any_fsqrt>, FP64, 8>;
+  def SQEB : UnaryRXE<"sqeb", 0xED14, loadu<any_fsqrt, z_fprload>, FP32, 4>;
+  def SQDB : UnaryRXE<"sqdb", 0xED15, loadu<any_fsqrt, z_fprload>, FP64, 8>;
 }
 
 // Round to an integer, with the second operand (modifier M3) specifying
@@ -432,6 +433,8 @@ let Uses = [FPC], mayRaiseFPException = 1,
   }
   defm AEB : BinaryRXEAndPseudo<"aeb", 0xED0A, any_fadd, FP32, z_load, 4>;
   defm ADB : BinaryRXEAndPseudo<"adb", 0xED1A, any_fadd, FP64, z_load, 8>;
+  defm AEB : BinaryRXEAndPseudo<"aeb", 0xED0A, any_fadd, FP32, z_fprload, 4>;
+  defm ADB : BinaryRXEAndPseudo<"adb", 0xED1A, any_fadd, FP64, z_fprload, 8>;
 }
 
 // Subtraction.
@@ -443,6 +446,8 @@ let Uses = [FPC], mayRaiseFPException = 1,
 
   defm SEB : BinaryRXEAndPseudo<"seb",  0xED0B, any_fsub, FP32, z_load, 4>;
   defm SDB : BinaryRXEAndPseudo<"sdb",  0xED1B, any_fsub, FP64, z_load, 8>;
+  defm SEB : BinaryRXEAndPseudo<"seb",  0xED0B, any_fsub, FP32, z_fprload, 4>;
+  defm SDB : BinaryRXEAndPseudo<"sdb",  0xED1B, any_fsub, FP64, z_fprload, 8>;
 }
 
 // Multiplication.
@@ -454,6 +459,8 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
   }
   defm MEEB : BinaryRXEAndPseudo<"meeb", 0xED17, any_fmul, FP32, z_load, 4>;
   defm MDB  : BinaryRXEAndPseudo<"mdb",  0xED1C, any_fmul, FP64, z_load, 8>;
+  defm MEEB : BinaryRXEAndPseudo<"meeb", 0xED17, any_fmul, FP32, z_fprload, 4>;
+  defm MDB  : BinaryRXEAndPseudo<"mdb",  0xED1C, any_fmul, FP64, z_fprload, 8>;
 }
 
 // f64 multiplication of two FP32 registers.
@@ -497,6 +504,10 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
 
   defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32, z_load, 4>;
   defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64, z_load, 8>;
+  defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32,
+                                  z_fprload, 4>;
+  defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64,
+                                  z_fprload, 8>;
 }
 
 // Fused multiply-subtract.
@@ -506,6 +517,10 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
 
   defm MSEB : TernaryRXFAndPseudo<"mseb", 0xED0F, z_any_fms, FP32, FP32, z_load, 4>;
   defm MSDB : TernaryRXFAndPseudo<"msdb", 0xED1F, z_any_fms, FP64, FP64, z_load, 8>;
+  defm MSEB : TernaryRXFAndPseudo<"mseb", 0xED0F, z_any_fms, FP32, FP32,
+                                  z_fprload, 4>;
+  defm MSDB : TernaryRXFAndPseudo<"msdb", 0xED1F, z_any_fms, FP64, FP64,
+                                  z_fprload, 8>;
 }
 
 // Division.
@@ -516,6 +531,8 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
 
   defm DEB : BinaryRXEAndPseudo<"deb", 0xED0D, any_fdiv, FP32, z_load, 4>;
   defm DDB : BinaryRXEAndPseudo<"ddb", 0xED1D, any_fdiv, FP64, z_load, 8>;
+  defm DEB : BinaryRXEAndPseudo<"deb", 0xED0D, any_fdiv, FP32, z_fprload, 4>;
+  defm DDB : BinaryRXEAndPseudo<"ddb", 0xED1D, any_fdiv, FP64, z_fprload, 8>;
 }
 
 // Divide to integer.
@@ -535,6 +552,8 @@ let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC], CCValues = 0xF in {
 
   def CEB : CompareRXE<"ceb", 0xED09, z_any_fcmp, FP32, z_load, 4>;
   def CDB : CompareRXE<"cdb", 0xED19, z_any_fcmp, FP64, z_load, 8>;
+  def CEB : CompareRXE<"ceb", 0xED09, z_any_fcmp, FP32, z_fprload, 4>;
+  def CDB : CompareRXE<"cdb", 0xED19, z_any_fcmp, FP64, z_fprload, 8>;
 
   def KEBR : CompareRRE<"kebr", 0xB308, z_strict_fcmps, FP32,  FP32>;
   def KDBR : CompareRRE<"kdbr", 0xB318, z_strict_fcmps, FP64,  FP64>;

llvm/lib/Target/SystemZ/SystemZInstrFormats.td

@@ -5536,3 +5536,17 @@ multiclass StringRRE<string mnemonic, bits<16> opcode,
                       [(set GR64:$end, (operator GR64:$start1, GR64:$start2,
                                                  GR32:$char))]>;
 }
+
+multiclass BinaryVRRcAndCCPseudo<string mnemonic, bits<16> opcode,
+                                 SDPatternOperator operator,
+                                 TypedReg tr1, TypedReg tr2, bits<4> type = 0,
+                                 bits<4> m5 = 0, bits<4> m6 = 0,
+                                 string fp_mnemonic = ""> {
+  def "" : BinaryVRRc<mnemonic, opcode, null_frag, tr1, tr2, type, m5, m6,
+                      fp_mnemonic>;
+  let Defs = [CC] in
+    def _CCPseudo : Pseudo<(outs tr1.op:$V1), (ins tr2.op:$V2, tr2.op:$V3),
+                           [(set (tr1.vt tr1.op:$V1),
+                                 (operator (tr2.vt tr2.op:$V2),
+                                           (tr2.vt tr2.op:$V3)))]>;
+}