Skip to content

Commit 04a8bff

Browse files
vporpovporpo
authored
[SandboxVec][DAG] Build actual dependencies (#111094)
This patch implements actual dependencies checking using BatchAA. This adds memory dep edges between MemDGNodes.
1 parent 0e86e52 commit 04a8bff

File tree

5 files changed

+485
-24
lines changed

5 files changed

+485
-24
lines changed

‎llvm/include/llvm/SandboxIR/Utils.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#ifndef LLVM_SANDBOXIR_UTILS_H
1313
#define LLVM_SANDBOXIR_UTILS_H
1414

15+
#include "llvm/Analysis/AliasAnalysis.h"
1516
#include "llvm/Analysis/LoopAccessAnalysis.h"
1617
#include "llvm/Analysis/MemoryLocation.h"
1718
#include "llvm/Analysis/ScalarEvolution.h"
@@ -99,6 +100,13 @@ class Utils {
99100
return false;
100101
return *Diff > 0;
101102
}
103+
104+
/// Equivalent to BatchAA::getModRefInfo().
105+
static ModRefInfo
106+
aliasAnalysisGetModRefInfo(BatchAAResults &BatchAA, const Instruction *I,
107+
const std::optional<MemoryLocation> &OptLoc) {
108+
return BatchAA.getModRefInfo(cast<llvm::Instruction>(I->Val), OptLoc);
109+
}
102110
};
103111

104112
} // namespace llvm::sandboxir

‎llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
#include "llvm/ADT/DenseMap.h"
2626
#include "llvm/ADT/iterator_range.h"
27+
#include "llvm/Analysis/AliasAnalysis.h"
2728
#include "llvm/SandboxIR/Instruction.h"
2829
#include "llvm/SandboxIR/IntrinsicInst.h"
2930
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
@@ -47,6 +48,7 @@ class DGNode {
4748
// TODO: Use a PointerIntPair for SubclassID and I.
4849
/// For isa/dyn_cast etc.
4950
DGNodeID SubclassID;
51+
// TODO: Move MemPreds to MemDGNode.
5052
/// Memory predecessors.
5153
DenseSet<MemDGNode *> MemPreds;
5254

@@ -86,13 +88,20 @@ class DGNode {
8688
(!(II = dyn_cast<IntrinsicInst>(I)) || isMemIntrinsic(II));
8789
}
8890

91+
/// \Returns true if \p I is fence like. It excludes non-mem intrinsics.
92+
static bool isFenceLike(Instruction *I) {
93+
IntrinsicInst *II;
94+
return I->isFenceLike() &&
95+
(!(II = dyn_cast<IntrinsicInst>(I)) || isMemIntrinsic(II));
96+
}
97+
8998
/// \Returns true if \p I is a memory dependency candidate instruction.
9099
static bool isMemDepNodeCandidate(Instruction *I) {
91100
AllocaInst *Alloca;
92101
return isMemDepCandidate(I) ||
93102
((Alloca = dyn_cast<AllocaInst>(I)) &&
94103
Alloca->isUsedWithInAlloca()) ||
95-
isStackSaveOrRestoreIntrinsic(I);
104+
isStackSaveOrRestoreIntrinsic(I) || isFenceLike(I);
96105
}
97106

98107
Instruction *getInstruction() const { return I; }
@@ -159,8 +168,37 @@ class DependencyGraph {
159168
/// The DAG spans across all instructions in this interval.
160169
Interval<Instruction> DAGInterval;
161170

171+
std::unique_ptr<BatchAAResults> BatchAA;
172+
173+
enum class DependencyType {
174+
RAW, ///> Read After Write
175+
WAW, ///> Write After Write
176+
RAR, ///> Read After Read
177+
WAR, ///> Write After Read
178+
CTRL, ///> Control-related dependencies, like with PHIs/Terminators
179+
OTHER, ///> Currently used for stack related instrs
180+
NONE, ///> No memory/other dependency
181+
};
182+
/// \Returns the dependency type depending on whether instructions may
183+
/// read/write memory or whether they are some specific opcode-related
184+
/// restrictions.
185+
/// Note: It does not check whether a memory dependency is actually correct,
186+
/// as it won't call AA. Therefore it returns the worst-case dep type.
187+
static DependencyType getRoughDepType(Instruction *FromI, Instruction *ToI);
188+
189+
// TODO: Implement AABudget.
190+
/// \Returns true if there is a memory/other dependency \p SrcI->DstI.
191+
bool alias(Instruction *SrcI, Instruction *DstI, DependencyType DepType);
192+
193+
bool hasDep(sandboxir::Instruction *SrcI, sandboxir::Instruction *DstI);
194+
195+
/// Go through all mem nodes in \p SrcScanRange and try to add dependencies to
196+
/// \p DstN.
197+
void scanAndAddDeps(DGNode &DstN, const Interval<MemDGNode> &SrcScanRange);
198+
162199
public:
163-
DependencyGraph() {}
200+
DependencyGraph(AAResults &AA)
201+
: BatchAA(std::make_unique<BatchAAResults>(AA)) {}
164202

165203
DGNode *getNode(Instruction *I) const {
166204
auto It = InstrToNodeMap.find(I);

‎llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ template <typename T, typename IntervalType> class IntervalIterator {
5858
}
5959
IntervalIterator &operator--() {
6060
// `I` is nullptr for end() when To is the BB terminator.
61-
I = I != nullptr ? I->getPrevNode() : R.To;
61+
I = I != nullptr ? I->getPrevNode() : R.bottom();
6262
return *this;
6363
}
6464
IntervalIterator operator--(int) {
@@ -110,14 +110,16 @@ template <typename T> class Interval {
110110
T *bottom() const { return To; }
111111

112112
using iterator = IntervalIterator<T, Interval>;
113-
using const_iterator = IntervalIterator<const T, const Interval>;
114113
iterator begin() { return iterator(From, *this); }
115114
iterator end() {
116115
return iterator(To != nullptr ? To->getNextNode() : nullptr, *this);
117116
}
118-
const_iterator begin() const { return const_iterator(From, *this); }
119-
const_iterator end() const {
120-
return const_iterator(To != nullptr ? To->getNextNode() : nullptr, *this);
117+
iterator begin() const {
118+
return iterator(From, const_cast<Interval &>(*this));
119+
}
120+
iterator end() const {
121+
return iterator(To != nullptr ? To->getNextNode() : nullptr,
122+
const_cast<Interval &>(*this));
121123
}
122124
/// Equality.
123125
bool operator==(const Interval &Other) const {

‎llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp

Lines changed: 121 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88

99
#include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
1010
#include "llvm/ADT/ArrayRef.h"
11+
#include "llvm/SandboxIR/Utils.h"
1112

12-
using namespace llvm::sandboxir;
13+
namespace llvm::sandboxir {
1314

1415
#ifndef NDEBUG
1516
void DGNode::print(raw_ostream &OS, bool PrintDeps) const {
@@ -50,29 +51,135 @@ MemDGNodeIntervalBuilder::make(const Interval<Instruction> &Instrs,
5051
cast<MemDGNode>(DAG.getNode(MemBotI)));
5152
}
5253

54+
DependencyGraph::DependencyType
55+
DependencyGraph::getRoughDepType(Instruction *FromI, Instruction *ToI) {
56+
// TODO: Perhaps compile-time improvement by skipping if neither is mem?
57+
if (FromI->mayWriteToMemory()) {
58+
if (ToI->mayReadFromMemory())
59+
return DependencyType::RAW;
60+
if (ToI->mayWriteToMemory())
61+
return DependencyType::WAW;
62+
} else if (FromI->mayReadFromMemory()) {
63+
if (ToI->mayWriteToMemory())
64+
return DependencyType::WAR;
65+
if (ToI->mayReadFromMemory())
66+
return DependencyType::RAR;
67+
}
68+
if (isa<sandboxir::PHINode>(FromI) || isa<sandboxir::PHINode>(ToI))
69+
return DependencyType::CTRL;
70+
if (ToI->isTerminator())
71+
return DependencyType::CTRL;
72+
if (DGNode::isStackSaveOrRestoreIntrinsic(FromI) ||
73+
DGNode::isStackSaveOrRestoreIntrinsic(ToI))
74+
return DependencyType::OTHER;
75+
return DependencyType::NONE;
76+
}
77+
78+
static bool isOrdered(Instruction *I) {
79+
auto IsOrdered = [](Instruction *I) {
80+
if (auto *LI = dyn_cast<LoadInst>(I))
81+
return !LI->isUnordered();
82+
if (auto *SI = dyn_cast<StoreInst>(I))
83+
return !SI->isUnordered();
84+
if (DGNode::isFenceLike(I))
85+
return true;
86+
return false;
87+
};
88+
bool Is = IsOrdered(I);
89+
assert((!Is || DGNode::isMemDepCandidate(I)) &&
90+
"An ordered instruction must be a MemDepCandidate!");
91+
return Is;
92+
}
93+
94+
bool DependencyGraph::alias(Instruction *SrcI, Instruction *DstI,
95+
DependencyType DepType) {
96+
std::optional<MemoryLocation> DstLocOpt =
97+
Utils::memoryLocationGetOrNone(DstI);
98+
if (!DstLocOpt)
99+
return true;
100+
// Check aliasing.
101+
assert((SrcI->mayReadFromMemory() || SrcI->mayWriteToMemory()) &&
102+
"Expected a mem instr");
103+
// TODO: Check AABudget
104+
ModRefInfo SrcModRef =
105+
isOrdered(SrcI)
106+
? ModRefInfo::Mod
107+
: Utils::aliasAnalysisGetModRefInfo(*BatchAA, SrcI, *DstLocOpt);
108+
switch (DepType) {
109+
case DependencyType::RAW:
110+
case DependencyType::WAW:
111+
return isModSet(SrcModRef);
112+
case DependencyType::WAR:
113+
return isRefSet(SrcModRef);
114+
default:
115+
llvm_unreachable("Expected only RAW, WAW and WAR!");
116+
}
117+
}
118+
119+
bool DependencyGraph::hasDep(Instruction *SrcI, Instruction *DstI) {
120+
DependencyType RoughDepType = getRoughDepType(SrcI, DstI);
121+
switch (RoughDepType) {
122+
case DependencyType::RAR:
123+
return false;
124+
case DependencyType::RAW:
125+
case DependencyType::WAW:
126+
case DependencyType::WAR:
127+
return alias(SrcI, DstI, RoughDepType);
128+
case DependencyType::CTRL:
129+
// Adding actual dep edges from PHIs/to terminator would just create too
130+
// many edges, which would be bad for compile-time.
131+
// So we ignore them in the DAG formation but handle them in the
132+
// scheduler, while sorting the ready list.
133+
return false;
134+
case DependencyType::OTHER:
135+
return true;
136+
case DependencyType::NONE:
137+
return false;
138+
}
139+
}
140+
141+
void DependencyGraph::scanAndAddDeps(DGNode &DstN,
142+
const Interval<MemDGNode> &SrcScanRange) {
143+
assert(isa<MemDGNode>(DstN) &&
144+
"DstN is the mem dep destination, so it must be mem");
145+
Instruction *DstI = DstN.getInstruction();
146+
// Walk up the instruction chain from ScanRange bottom to top, looking for
147+
// memory instrs that may alias.
148+
for (MemDGNode &SrcN : reverse(SrcScanRange)) {
149+
Instruction *SrcI = SrcN.getInstruction();
150+
if (hasDep(SrcI, DstI))
151+
DstN.addMemPred(&SrcN);
152+
}
153+
}
154+
53155
Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
54156
if (Instrs.empty())
55157
return {};
56-
// TODO: For now create a chain of dependencies.
57-
Interval<Instruction> Interval(Instrs);
58-
auto *TopI = Interval.top();
59-
auto *BotI = Interval.bottom();
60-
DGNode *LastN = getOrCreateNode(TopI);
158+
159+
Interval<Instruction> InstrInterval(Instrs);
160+
161+
DGNode *LastN = getOrCreateNode(InstrInterval.top());
162+
// Create DGNodes for all instrs in Interval to avoid future Instruction to
163+
// DGNode lookups.
61164
MemDGNode *LastMemN = dyn_cast<MemDGNode>(LastN);
62-
for (Instruction *I = TopI->getNextNode(), *E = BotI->getNextNode(); I != E;
63-
I = I->getNextNode()) {
64-
auto *N = getOrCreateNode(I);
65-
N->addMemPred(LastMemN);
165+
for (Instruction &I : drop_begin(InstrInterval)) {
166+
auto *N = getOrCreateNode(&I);
66167
// Build the Mem node chain.
67168
if (auto *MemN = dyn_cast<MemDGNode>(N)) {
68169
MemN->setPrevNode(LastMemN);
69170
if (LastMemN != nullptr)
70171
LastMemN->setNextNode(MemN);
71172
LastMemN = MemN;
72173
}
73-
LastN = N;
74174
}
75-
return Interval;
175+
// Create the dependencies.
176+
auto DstRange = MemDGNodeIntervalBuilder::make(InstrInterval, *this);
177+
for (MemDGNode &DstN : drop_begin(DstRange)) {
178+
auto SrcRange = Interval<MemDGNode>(DstRange.top(), DstN.getPrevNode());
179+
scanAndAddDeps(DstN, SrcRange);
180+
}
181+
182+
return InstrInterval;
76183
}
77184

78185
#ifndef NDEBUG
@@ -95,3 +202,5 @@ void DependencyGraph::dump() const {
95202
dbgs() << "\n";
96203
}
97204
#endif // NDEBUG
205+
206+
} // namespace llvm::sandboxir

0 commit comments

Comments
 (0)