Skip to content

Commit e2fc6a3

Browse files
fhahnfhahn
committed
[MemCpyOpt] Preserve MemorySSA.
This patch updates MemCpyOpt to preserve MemorySSA. It uses the MemoryDef at the insertion point of the builder and inserts the new def after that def. In some cases, we just modify a memory instruction. In that case, get the defining access, then remove the memory access and add a new one. If the defining access is in a different block, insert a new def at the beginning of the current block, otherwise after the defining access. Reviewed By: asbirlea Differential Revision: https://reviews.llvm.org/D86651
1 parent aec9e20 commit e2fc6a3

File tree

3 files changed

+285
-16
lines changed

3 files changed

+285
-16
lines changed

‎llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ class Instruction;
3030
class MemCpyInst;
3131
class MemMoveInst;
3232
class MemoryDependenceResults;
33+
class MemorySSA;
34+
class MemorySSAUpdater;
3335
class MemSetInst;
3436
class StoreInst;
3537
class TargetLibraryInfo;
@@ -41,6 +43,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
4143
AliasAnalysis *AA = nullptr;
4244
AssumptionCache *AC = nullptr;
4345
DominatorTree *DT = nullptr;
46+
MemorySSAUpdater *MSSAU = nullptr;
4447

4548
public:
4649
MemCpyOptPass() = default;
@@ -50,7 +53,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
5053
// Glue for the old PM.
5154
bool runImpl(Function &F, MemoryDependenceResults *MD_,
5255
TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
53-
AssumptionCache *AC_, DominatorTree *DT_);
56+
AssumptionCache *AC_, DominatorTree *DT_, MemorySSA *MSSA_);
5457

5558
private:
5659
// Helper functions

‎llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

Lines changed: 142 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
#include "llvm/Analysis/GlobalsModRef.h"
2424
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
2525
#include "llvm/Analysis/MemoryLocation.h"
26+
#include "llvm/Analysis/MemorySSA.h"
27+
#include "llvm/Analysis/MemorySSAUpdater.h"
2628
#include "llvm/Analysis/TargetLibraryInfo.h"
2729
#include "llvm/Analysis/ValueTracking.h"
2830
#include "llvm/IR/Argument.h"
@@ -278,6 +280,7 @@ class MemCpyOptLegacyPass : public FunctionPass {
278280
AU.addPreserved<MemoryDependenceWrapperPass>();
279281
AU.addRequired<AAResultsWrapperPass>();
280282
AU.addPreserved<AAResultsWrapperPass>();
283+
AU.addPreserved<MemorySSAWrapperPass>();
281284
}
282285
};
283286

@@ -315,7 +318,27 @@ Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
315318
MemsetRanges Ranges(DL);
316319

317320
BasicBlock::iterator BI(StartInst);
321+
322+
// Keeps track of the last memory use or def before the insertion point for
323+
// the new memset. The new MemoryDef for the inserted memsets will be inserted
324+
// after MemInsertPoint. It points to either LastMemDef or to the last user
325+
// before the insertion point of the memset, if there are any such users.
326+
MemoryUseOrDef *MemInsertPoint = nullptr;
327+
// Keeps track of the last MemoryDef between StartInst and the insertion point
328+
// for the new memset. This will become the defining access of the inserted
329+
// memsets.
330+
MemoryDef *LastMemDef = nullptr;
318331
for (++BI; !BI->isTerminator(); ++BI) {
332+
if (MSSAU) {
333+
auto *CurrentAcc = cast_or_null<MemoryUseOrDef>(
334+
MSSAU->getMemorySSA()->getMemoryAccess(&*BI));
335+
if (CurrentAcc) {
336+
MemInsertPoint = CurrentAcc;
337+
if (auto *CurrentDef = dyn_cast<MemoryDef>(CurrentAcc))
338+
LastMemDef = CurrentDef;
339+
}
340+
}
341+
319342
if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
320343
// If the instruction is readnone, ignore it, otherwise bail out. We
321344
// don't even allow readonly here because we don't want something like:
@@ -394,15 +417,27 @@ Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
394417
: Range.TheStores) dbgs()
395418
<< *SI << '\n';
396419
dbgs() << "With: " << *AMemSet << '\n');
397-
398420
if (!Range.TheStores.empty())
399421
AMemSet->setDebugLoc(Range.TheStores[0]->getDebugLoc());
400422

423+
if (MSSAU) {
424+
assert(LastMemDef && MemInsertPoint &&
425+
"Both LastMemDef and MemInsertPoint need to be set");
426+
auto *NewDef = cast<MemoryDef>(
427+
MSSAU->createMemoryAccessAfter(AMemSet, LastMemDef, MemInsertPoint));
428+
MSSAU->insertDef(NewDef, /*RenameUses=*/true);
429+
LastMemDef = NewDef;
430+
MemInsertPoint = NewDef;
431+
}
432+
401433
// Zap all the stores.
402434
for (Instruction *SI : Range.TheStores) {
435+
if (MSSAU)
436+
MSSAU->removeMemoryAccess(SI);
403437
MD->removeInstruction(SI);
404438
SI->eraseFromParent();
405439
}
440+
406441
++NumMemSetInfer;
407442
}
408443

@@ -573,6 +608,17 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
573608
LLVM_DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI << " => "
574609
<< *M << "\n");
575610

611+
if (MSSAU) {
612+
assert(isa<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(P)));
613+
auto *LastDef =
614+
cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(P));
615+
auto *NewAccess =
616+
MSSAU->createMemoryAccessAfter(M, LastDef, LastDef);
617+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
618+
MSSAU->removeMemoryAccess(SI);
619+
MSSAU->removeMemoryAccess(LI);
620+
}
621+
576622
MD->removeInstruction(SI);
577623
SI->eraseFromParent();
578624
MD->removeInstruction(LI);
@@ -621,6 +667,11 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
621667
DL.getTypeStoreSize(SI->getOperand(0)->getType()),
622668
commonAlignment(SI->getAlign(), LI->getAlign()), C);
623669
if (changed) {
670+
if (MSSAU) {
671+
MSSAU->removeMemoryAccess(SI);
672+
MSSAU->removeMemoryAccess(LI);
673+
}
674+
624675
MD->removeInstruction(SI);
625676
SI->eraseFromParent();
626677
MD->removeInstruction(LI);
@@ -658,6 +709,15 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
658709

659710
LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
660711

712+
if (MSSAU) {
713+
assert(isa<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(SI)));
714+
auto *LastDef =
715+
cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(SI));
716+
auto *NewAccess = MSSAU->createMemoryAccessAfter(M, LastDef, LastDef);
717+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
718+
MSSAU->removeMemoryAccess(SI);
719+
}
720+
661721
MD->removeInstruction(SI);
662722
SI->eraseFromParent();
663723
NumMemSetInfer++;
@@ -939,14 +999,23 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
939999
// TODO: Is this worth it if we're creating a less aligned memcpy? For
9401000
// example we could be moving from movaps -> movq on x86.
9411001
IRBuilder<> Builder(M);
1002+
Instruction *NewM;
9421003
if (UseMemMove)
943-
Builder.CreateMemMove(M->getRawDest(), M->getDestAlign(),
944-
MDep->getRawSource(), MDep->getSourceAlign(),
945-
M->getLength(), M->isVolatile());
1004+
NewM = Builder.CreateMemMove(M->getRawDest(), M->getDestAlign(),
1005+
MDep->getRawSource(), MDep->getSourceAlign(),
1006+
M->getLength(), M->isVolatile());
9461007
else
947-
Builder.CreateMemCpy(M->getRawDest(), M->getDestAlign(),
948-
MDep->getRawSource(), MDep->getSourceAlign(),
949-
M->getLength(), M->isVolatile());
1008+
NewM = Builder.CreateMemCpy(M->getRawDest(), M->getDestAlign(),
1009+
MDep->getRawSource(), MDep->getSourceAlign(),
1010+
M->getLength(), M->isVolatile());
1011+
1012+
if (MSSAU) {
1013+
assert(isa<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(M)));
1014+
auto *LastDef = cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(M));
1015+
auto *NewAccess = MSSAU->createMemoryAccessAfter(NewM, LastDef, LastDef);
1016+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
1017+
MSSAU->removeMemoryAccess(M);
1018+
}
9501019

9511020
// Remove the instruction we're replacing.
9521021
MD->removeInstruction(M);
@@ -1012,11 +1081,25 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
10121081
Value *SizeDiff = Builder.CreateSub(DestSize, SrcSize);
10131082
Value *MemsetLen = Builder.CreateSelect(
10141083
Ule, ConstantInt::getNullValue(DestSize->getType()), SizeDiff);
1015-
Builder.CreateMemSet(
1084+
Instruction *NewMemSet = Builder.CreateMemSet(
10161085
Builder.CreateGEP(Dest->getType()->getPointerElementType(), Dest,
10171086
SrcSize),
10181087
MemSet->getOperand(1), MemsetLen, MaybeAlign(Align));
10191088

1089+
if (MSSAU) {
1090+
assert(isa<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(MemCpy)) &&
1091+
"MemCpy must be a MemoryDef");
1092+
// The new memset is inserted after the memcpy, but it is known that its
1093+
// defining access is the memset about to be removed which immediately
1094+
// precedes the memcpy.
1095+
auto *LastDef =
1096+
cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(MemCpy));
1097+
auto *NewAccess = MSSAU->createMemoryAccessBefore(
1098+
NewMemSet, LastDef->getDefiningAccess(), LastDef);
1099+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
1100+
MSSAU->removeMemoryAccess(MemSet);
1101+
}
1102+
10201103
MD->removeInstruction(MemSet);
10211104
MemSet->eraseFromParent();
10221105
return true;
@@ -1081,8 +1164,16 @@ bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
10811164
}
10821165

10831166
IRBuilder<> Builder(MemCpy);
1084-
Builder.CreateMemSet(MemCpy->getRawDest(), MemSet->getOperand(1), CopySize,
1085-
MaybeAlign(MemCpy->getDestAlignment()));
1167+
Instruction *NewM =
1168+
Builder.CreateMemSet(MemCpy->getRawDest(), MemSet->getOperand(1),
1169+
CopySize, MaybeAlign(MemCpy->getDestAlignment()));
1170+
if (MSSAU) {
1171+
auto *LastDef =
1172+
cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(MemCpy));
1173+
auto *NewAccess = MSSAU->createMemoryAccessAfter(NewM, LastDef, LastDef);
1174+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
1175+
}
1176+
10861177
return true;
10871178
}
10881179

@@ -1098,6 +1189,9 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
10981189
// If the source and destination of the memcpy are the same, then zap it.
10991190
if (M->getSource() == M->getDest()) {
11001191
++BBI;
1192+
if (MSSAU)
1193+
MSSAU->removeMemoryAccess(M);
1194+
11011195
MD->removeInstruction(M);
11021196
M->eraseFromParent();
11031197
return true;
@@ -1109,8 +1203,18 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
11091203
if (Value *ByteVal = isBytewiseValue(GV->getInitializer(),
11101204
M->getModule()->getDataLayout())) {
11111205
IRBuilder<> Builder(M);
1112-
Builder.CreateMemSet(M->getRawDest(), ByteVal, M->getLength(),
1113-
MaybeAlign(M->getDestAlignment()), false);
1206+
Instruction *NewM =
1207+
Builder.CreateMemSet(M->getRawDest(), ByteVal, M->getLength(),
1208+
MaybeAlign(M->getDestAlignment()), false);
1209+
if (MSSAU) {
1210+
auto *LastDef =
1211+
cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(M));
1212+
auto *NewAccess =
1213+
MSSAU->createMemoryAccessAfter(NewM, LastDef, LastDef);
1214+
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
1215+
MSSAU->removeMemoryAccess(M);
1216+
}
1217+
11141218
MD->removeInstruction(M);
11151219
M->eraseFromParent();
11161220
++NumCpyToSet;
@@ -1145,6 +1249,9 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
11451249
M->getSourceAlign().valueOrOne());
11461250
if (performCallSlotOptzn(M, M->getDest(), M->getSource(),
11471251
CopySize->getZExtValue(), Alignment, C)) {
1252+
if (MSSAU)
1253+
MSSAU->removeMemoryAccess(M);
1254+
11481255
MD->removeInstruction(M);
11491256
M->eraseFromParent();
11501257
return true;
@@ -1161,6 +1268,9 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
11611268
return processMemCpyMemCpyDependence(M, MDep);
11621269
} else if (SrcDepInfo.isDef()) {
11631270
if (hasUndefContents(SrcDepInfo.getInst(), CopySize)) {
1271+
if (MSSAU)
1272+
MSSAU->removeMemoryAccess(M);
1273+
11641274
MD->removeInstruction(M);
11651275
M->eraseFromParent();
11661276
++NumMemCpyInstr;
@@ -1171,6 +1281,8 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
11711281
if (SrcDepInfo.isClobber())
11721282
if (MemSetInst *MDep = dyn_cast<MemSetInst>(SrcDepInfo.getInst()))
11731283
if (performMemCpyToMemSetOptzn(M, MDep)) {
1284+
if (MSSAU)
1285+
MSSAU->removeMemoryAccess(M);
11741286
MD->removeInstruction(M);
11751287
M->eraseFromParent();
11761288
++NumCpyToSet;
@@ -1201,6 +1313,9 @@ bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
12011313
M->setCalledFunction(Intrinsic::getDeclaration(M->getModule(),
12021314
Intrinsic::memcpy, ArgTys));
12031315

1316+
// For MemorySSA nothing really changes (except that memcpy may imply stricter
1317+
// aliasing guarantees).
1318+
12041319
// MemDep may have over conservative information about this instruction, just
12051320
// conservatively flush it from the cache.
12061321
MD->removeInstruction(M);
@@ -1338,27 +1453,34 @@ PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
13381453
auto *AA = &AM.getResult<AAManager>(F);
13391454
auto *AC = &AM.getResult<AssumptionAnalysis>(F);
13401455
auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
1456+
auto *MSSA = AM.getCachedResult<MemorySSAAnalysis>(F);
13411457

1342-
bool MadeChange = runImpl(F, &MD, &TLI, AA, AC, DT);
1458+
bool MadeChange =
1459+
runImpl(F, &MD, &TLI, AA, AC, DT, MSSA ? &MSSA->getMSSA() : nullptr);
13431460
if (!MadeChange)
13441461
return PreservedAnalyses::all();
13451462

13461463
PreservedAnalyses PA;
13471464
PA.preserveSet<CFGAnalyses>();
13481465
PA.preserve<GlobalsAA>();
13491466
PA.preserve<MemoryDependenceAnalysis>();
1467+
if (MSSA)
1468+
PA.preserve<MemorySSAAnalysis>();
13501469
return PA;
13511470
}
13521471

13531472
bool MemCpyOptPass::runImpl(Function &F, MemoryDependenceResults *MD_,
13541473
TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
1355-
AssumptionCache *AC_, DominatorTree *DT_) {
1474+
AssumptionCache *AC_, DominatorTree *DT_,
1475+
MemorySSA *MSSA_) {
13561476
bool MadeChange = false;
13571477
MD = MD_;
13581478
TLI = TLI_;
13591479
AA = AA_;
13601480
AC = AC_;
13611481
DT = DT_;
1482+
MemorySSAUpdater MSSAU_(MSSA_);
1483+
MSSAU = MSSA_ ? &MSSAU_ : nullptr;
13621484
// If we don't have at least memset and memcpy, there is little point of doing
13631485
// anything here. These are required by a freestanding implementation, so if
13641486
// even they are disabled, there is no point in trying hard.
@@ -1371,6 +1493,9 @@ bool MemCpyOptPass::runImpl(Function &F, MemoryDependenceResults *MD_,
13711493
MadeChange = true;
13721494
}
13731495

1496+
if (MSSA_ && VerifyMemorySSA)
1497+
MSSA_->verifyMemorySSA();
1498+
13741499
MD = nullptr;
13751500
return MadeChange;
13761501
}
@@ -1385,6 +1510,8 @@ bool MemCpyOptLegacyPass::runOnFunction(Function &F) {
13851510
auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
13861511
auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
13871512
auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1513+
auto *MSSAWP = getAnalysisIfAvailable<MemorySSAWrapperPass>();
13881514

1389-
return Impl.runImpl(F, MD, TLI, AA, AC, DT);
1515+
return Impl.runImpl(F, MD, TLI, AA, AC, DT,
1516+
MSSAWP ? &MSSAWP->getMSSA() : nullptr);
13901517
}

0 commit comments

Comments
 (0)