[ScheduleDAGInstrs::buildSchedGraph()] Handling of memory dependecies rewritten.

Recommited, after some fixing with test cases.

Updated test cases:
test/CodeGen/AArch64/arm64-misched-memdep-bug.ll
test/CodeGen/AArch64/tailcall_misched_graph.ll

Temporarily disabled test cases:
test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll
test/CodeGen/PowerPC/ppc64-fastcc.ll (partially updated)
test/CodeGen/PowerPC/vsx-fma-m.ll
test/CodeGen/PowerPC/vsx-fma-sp.ll

http://reviews.llvm.org/D8705
Reviewers: Hal Finkel, Andy Trick.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@259673 91177308-0d34-0410-b5e6-96231b3b80d8

Author: JonPsson

include/llvm/CodeGen/PseudoSourceValue.h

@@ -27,6 +27,8 @@ class MachineMemOperand;
 class raw_ostream;
 
 raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO);
+class PseudoSourceValue;
+raw_ostream &operator<<(raw_ostream &OS, const PseudoSourceValue* PSV);
 
 /// Special value supplied for machine level alias analysis. It indicates that
 /// a memory access references the functions stack frame (e.g., a spill slot),
@@ -45,6 +47,8 @@ class PseudoSourceValue {
 
 private:
   PSVKind Kind;
+  friend raw_ostream &llvm::operator<<(raw_ostream &OS,
+                                       const PseudoSourceValue* PSV);
 
   friend class MachineMemOperand; // For printCustom().
 

include/llvm/CodeGen/ScheduleDAG.h

@@ -396,6 +396,17 @@ namespace llvm {
     /// specified node.
     bool addPred(const SDep &D, bool Required = true);
 
+    /// addPredBarrier - This adds a barrier edge to SU by calling
+    /// addPred(), with latency 0 generally or latency 1 for a store
+    /// followed by a load.
+    bool addPredBarrier(SUnit *SU) {
+      SDep Dep(SU, SDep::Barrier);
+      unsigned TrueMemOrderLatency =
+        ((SU->getInstr()->mayStore() && this->getInstr()->mayLoad()) ? 1 : 0);
+      Dep.setLatency(TrueMemOrderLatency);
+      return addPred(Dep);
+    }
+
     /// removePred - This removes the specified edge as a pred of the current
     /// node if it exists.  It also removes the current node as a successor of
     /// the specified node.

include/llvm/CodeGen/ScheduleDAGInstrs.h

@@ -15,12 +15,14 @@
 #ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
 #define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SparseMultiSet.h"
 #include "llvm/ADT/SparseSet.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include <list>
 
 namespace llvm {
   class MachineFrameInfo;
@@ -84,6 +86,10 @@ namespace llvm {
   typedef SparseMultiSet<VReg2SUnitOperIdx, VirtReg2IndexFunctor>
     VReg2SUnitOperIdxMultiMap;
 
+  typedef PointerUnion<const Value *, const PseudoSourceValue *> ValueType;
+  typedef SmallVector<PointerIntPair<ValueType, 1, bool>, 4>
+          UnderlyingObjectsVector;
+
   /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of
   /// MachineInstrs.
   class ScheduleDAGInstrs : public ScheduleDAG {
@@ -149,10 +155,66 @@ namespace llvm {
     /// Tracks the last instructions in this region using each virtual register.
     VReg2SUnitOperIdxMultiMap CurrentVRegUses;
 
-    /// PendingLoads - Remember where unknown loads are after the most recent
-    /// unknown store, as we iterate. As with Defs and Uses, this is here
-    /// to minimize construction/destruction.
-    std::vector<SUnit *> PendingLoads;
+    AliasAnalysis *AAForDep;
+
+    /// Remember a generic side-effecting instruction as we proceed.
+    /// No other SU ever gets scheduled around it (except in the special
+    /// case of a huge region that gets reduced).
+    SUnit *BarrierChain;
+
+  public:
+
+    /// A list of SUnits, used in Value2SUsMap, during DAG construction.
+    /// Note: to gain speed it might be worth investigating an optimized
+    /// implementation of this data structure, such as a singly linked list
+    /// with a memory pool (SmallVector was tried but slow and SparseSet is not
+    /// applicable).
+    typedef std::list<SUnit *> SUList;
+  protected:
+    /// A map from ValueType to SUList, used during DAG construction,
+    /// as a means of remembering which SUs depend on which memory
+    /// locations.
+    class Value2SUsMap;
+
+    /// Remove in FIFO order some SUs from huge maps.
+    void reduceHugeMemNodeMaps(Value2SUsMap &stores,
+                               Value2SUsMap &loads, unsigned N);
+
+    /// Add a chain edge between SUa and SUb, but only if both AliasAnalysis
+    /// and Target fail to deny the dependency.
+    void addChainDependency(SUnit *SUa, SUnit *SUb,
+                            unsigned Latency = 0);
+
+    /// Add dependencies as needed from all SUs in list to SU.
+    void addChainDependencies(SUnit *SU, SUList &sus, unsigned Latency) {
+      for (auto *su : sus)
+        addChainDependency(SU, su, Latency);
+    }
+
+    /// Add dependencies as needed from all SUs in map, to SU.
+    void addChainDependencies(SUnit *SU, Value2SUsMap &Val2SUsMap);
+
+    /// Add dependencies as needed to SU, from all SUs mapped to V.
+    void addChainDependencies(SUnit *SU, Value2SUsMap &Val2SUsMap,
+                              ValueType V);
+
+    /// Add barrier chain edges from all SUs in map, and then clear
+    /// the map. This is equivalent to insertBarrierChain(), but
+    /// optimized for the common case where the new BarrierChain (a
+    /// global memory object) has a higher NodeNum than all SUs in
+    /// map. It is assumed BarrierChain has been set before calling
+    /// this.
+    void addBarrierChain(Value2SUsMap &map);
+
+    /// Insert a barrier chain in a huge region, far below current
+    /// SU. Add barrier chain edges from all SUs in map with higher
+    /// NodeNums than this new BarrierChain, and remove them from
+    /// map. It is assumed BarrierChain has been set before calling
+    /// this.
+    void insertBarrierChain(Value2SUsMap &map);
+
+    /// For an unanalyzable memory access, this Value is used in maps.
+    UndefValue *UnknownValue;
 
     /// DbgValues - Remember instruction that precedes DBG_VALUE.
     /// These are generated by buildSchedGraph but persist so they can be