Skip to content

Commit 9b1760b

Browse files
gottesmmgottesmm
committed
[PMOpt] Restructure aggregateAvailableValues as a class.
This is in preparation for adding the distinction of destructive vs non-destructive extracts. NFC. rdar://31521023
1 parent c459a8e commit 9b1760b

File tree

1 file changed

+182
-103
lines changed

1 file changed

+182
-103
lines changed

‎lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp

Lines changed: 182 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,15 @@ static SILValue nonDestructivelyExtractSubElement(SILValue Val,
197197
// Available Value Aggregation
198198
//===----------------------------------------------------------------------===//
199199

200+
namespace {
201+
202+
/// Our available value representation. Will become a struct later.
203+
using AvailableValue = std::pair<SILValue, unsigned>;
204+
205+
} // end anonymous namespace
206+
200207
static bool anyMissing(unsigned StartSubElt, unsigned NumSubElts,
201-
ArrayRef<std::pair<SILValue, unsigned>> &Values) {
208+
ArrayRef<AvailableValue> &Values) {
202209
while (NumSubElts) {
203210
if (!Values[StartSubElt].first)
204211
return true;
@@ -208,93 +215,168 @@ static bool anyMissing(unsigned StartSubElt, unsigned NumSubElts,
208215
return false;
209216
}
210217

218+
namespace {
219+
220+
/// A class that aggregates available values, loading them if they are not
221+
/// available.
222+
class AvailableValueAggregator {
223+
SILModule &M;
224+
SILBuilderWithScope B;
225+
SILLocation Loc;
226+
MutableArrayRef<AvailableValue> AvailableValueList;
227+
228+
public:
229+
AvailableValueAggregator(SILInstruction *Inst,
230+
MutableArrayRef<AvailableValue> AvailableValueList)
231+
: M(Inst->getModule()), B(Inst), Loc(Inst->getLoc()),
232+
AvailableValueList(AvailableValueList) {}
233+
234+
// This is intended to be passed by reference only once constructed.
235+
AvailableValueAggregator(const AvailableValueAggregator &) = delete;
236+
AvailableValueAggregator(AvailableValueAggregator &&) = delete;
237+
AvailableValueAggregator &
238+
operator=(const AvailableValueAggregator &) = delete;
239+
AvailableValueAggregator &operator=(AvailableValueAggregator &&) = delete;
240+
241+
SILValue aggregateValues(SILType LoadTy, SILValue Address, unsigned FirstElt);
242+
243+
private:
244+
SILValue aggregateFullyAvailableValue(SILType LoadTy, unsigned FirstElt);
245+
SILValue aggregateTupleSubElts(TupleType *TT, SILType LoadTy,
246+
SILValue Address, unsigned FirstElt);
247+
SILValue aggregateStructSubElts(StructDecl *SD, SILType LoadTy,
248+
SILValue Address, unsigned FirstElt);
249+
SILValue handlePrimitiveValue(SILType LoadTy, SILValue Address,
250+
unsigned FirstElt);
251+
};
252+
253+
} // end anonymous namespace
254+
211255
/// Given a bunch of primitive subelement values, build out the right aggregate
212256
/// type (LoadTy) by emitting tuple and struct instructions as necessary.
213-
static SILValue aggregateAvailableValues(
214-
SILInstruction *Inst, SILType LoadTy, SILValue Address,
215-
ArrayRef<std::pair<SILValue, unsigned>> AvailableValues,
216-
unsigned FirstElt) {
217-
assert(LoadTy.isObject());
218-
SILModule &M = Inst->getModule();
219-
257+
SILValue AvailableValueAggregator::aggregateValues(SILType LoadTy,
258+
SILValue Address,
259+
unsigned FirstElt) {
220260
// Check to see if the requested value is fully available, as an aggregate.
221261
// This is a super-common case for single-element structs, but is also a
222262
// general answer for arbitrary structs and tuples as well.
223-
if (FirstElt < AvailableValues.size()) { // #Elements may be zero.
224-
SILValue FirstVal = AvailableValues[FirstElt].first;
225-
if (FirstVal && AvailableValues[FirstElt].second == 0 &&
226-
FirstVal->getType() == LoadTy) {
227-
// If the first element of this value is available, check any extra ones
228-
// before declaring success.
229-
bool AllMatch = true;
230-
for (unsigned i = 0, e = getNumSubElements(LoadTy, M); i != e; ++i)
231-
if (AvailableValues[FirstElt + i].first != FirstVal ||
232-
AvailableValues[FirstElt + i].second != i) {
233-
AllMatch = false;
234-
break;
235-
}
236-
237-
if (AllMatch)
238-
return FirstVal;
239-
}
240-
}
241-
242-
SILBuilderWithScope B(Inst);
263+
if (SILValue Result = aggregateFullyAvailableValue(LoadTy, FirstElt))
264+
return Result;
243265

244-
if (TupleType *TT = LoadTy.getAs<TupleType>()) {
245-
SmallVector<SILValue, 4> ResultElts;
266+
// If we have a tuple type, then aggregate the tuple's elements into a full
267+
// tuple value.
268+
if (TupleType *TT = LoadTy.getAs<TupleType>())
269+
return aggregateTupleSubElts(TT, LoadTy, Address, FirstElt);
246270

247-
for (unsigned EltNo : indices(TT->getElements())) {
248-
SILType EltTy = LoadTy.getTupleElementType(EltNo);
249-
unsigned NumSubElt = getNumSubElements(EltTy, M);
271+
// If we have a struct type, then aggregate the struct's elements into a full
272+
// struct value.
273+
if (auto *SD = getFullyReferenceableStruct(LoadTy))
274+
return aggregateStructSubElts(SD, LoadTy, Address, FirstElt);
250275

251-
// If we are missing any of the available values in this struct element,
252-
// compute an address to load from.
253-
SILValue EltAddr;
254-
if (anyMissing(FirstElt, NumSubElt, AvailableValues))
255-
EltAddr = B.createTupleElementAddr(Inst->getLoc(), Address, EltNo,
256-
EltTy.getAddressType());
276+
// Otherwise, we have a non-aggregate primitive. Load or extract the value.
277+
return handlePrimitiveValue(LoadTy, Address, FirstElt);
278+
}
257279

258-
ResultElts.push_back(aggregateAvailableValues(Inst, EltTy, EltAddr,
259-
AvailableValues, FirstElt));
260-
FirstElt += NumSubElt;
261-
}
280+
// See if we have this value is fully available. In such a case, return it as an
281+
// aggregate. This is a super-common case for single-element structs, but is
282+
// also a general answer for arbitrary structs and tuples as well.
283+
SILValue
284+
AvailableValueAggregator::aggregateFullyAvailableValue(SILType LoadTy,
285+
unsigned FirstElt) {
286+
if (FirstElt >= AvailableValueList.size()) { // #Elements may be zero.
287+
return SILValue();
288+
}
262289

263-
return B.createTuple(Inst->getLoc(), LoadTy, ResultElts);
290+
SILValue FirstVal = AvailableValueList[FirstElt].first;
291+
// Make sure that the first element is available.
292+
if (!FirstVal || AvailableValueList[FirstElt].second != 0 ||
293+
FirstVal->getType() != LoadTy) {
294+
return SILValue();
264295
}
265296

266-
// Extract struct elements from fully referenceable structs.
267-
if (auto *SD = getFullyReferenceableStruct(LoadTy)) {
268-
SmallVector<SILValue, 4> ResultElts;
297+
// If the first element of this value is available, check that any extra
298+
// available values match our first value.
299+
if (llvm::any_of(
300+
range(getNumSubElements(LoadTy, M)), [&](unsigned Index) -> bool {
301+
return AvailableValueList[FirstElt + Index].first != FirstVal ||
302+
AvailableValueList[FirstElt + Index].second != Index;
303+
}))
304+
return SILValue();
269305

270-
for (auto *FD : SD->getStoredProperties()) {
271-
SILType EltTy = LoadTy.getFieldType(FD, M);
272-
unsigned NumSubElt = getNumSubElements(EltTy, M);
306+
return FirstVal;
307+
}
273308

274-
// If we are missing any of the available values in this struct element,
275-
// compute an address to load from.
276-
SILValue EltAddr;
277-
if (anyMissing(FirstElt, NumSubElt, AvailableValues))
278-
EltAddr = B.createStructElementAddr(Inst->getLoc(), Address, FD,
279-
EltTy.getAddressType());
309+
SILValue AvailableValueAggregator::aggregateTupleSubElts(TupleType *TT,
310+
SILType LoadTy,
311+
SILValue Address,
312+
unsigned FirstElt) {
313+
SmallVector<SILValue, 4> ResultElts;
314+
315+
for (unsigned EltNo : indices(TT->getElements())) {
316+
SILType EltTy = LoadTy.getTupleElementType(EltNo);
317+
unsigned NumSubElt = getNumSubElements(EltTy, M);
318+
319+
// If we are missing any of the available values in this struct element,
320+
// compute an address to load from.
321+
SILValue EltAddr;
322+
if (anyMissing(FirstElt, NumSubElt, AvailableValueList))
323+
EltAddr =
324+
B.createTupleElementAddr(Loc, Address, EltNo, EltTy.getAddressType());
325+
326+
ResultElts.push_back(aggregateValues(EltTy, EltAddr, FirstElt));
327+
FirstElt += NumSubElt;
328+
}
280329

281-
ResultElts.push_back(aggregateAvailableValues(Inst, EltTy, EltAddr,
282-
AvailableValues, FirstElt));
283-
FirstElt += NumSubElt;
284-
}
285-
return B.createStruct(Inst->getLoc(), LoadTy, ResultElts);
330+
return B.createTuple(Loc, LoadTy, ResultElts);
331+
}
332+
333+
SILValue AvailableValueAggregator::aggregateStructSubElts(StructDecl *SD,
334+
SILType LoadTy,
335+
SILValue Address,
336+
unsigned FirstElt) {
337+
SmallVector<SILValue, 4> ResultElts;
338+
339+
for (auto *FD : SD->getStoredProperties()) {
340+
SILType EltTy = LoadTy.getFieldType(FD, M);
341+
unsigned NumSubElt = getNumSubElements(EltTy, M);
342+
343+
// If we are missing any of the available values in this struct element,
344+
// compute an address to load from.
345+
SILValue EltAddr;
346+
if (anyMissing(FirstElt, NumSubElt, AvailableValueList))
347+
EltAddr =
348+
B.createStructElementAddr(Loc, Address, FD, EltTy.getAddressType());
349+
350+
ResultElts.push_back(aggregateValues(EltTy, EltAddr, FirstElt));
351+
FirstElt += NumSubElt;
286352
}
353+
return B.createStruct(Loc, LoadTy, ResultElts);
354+
}
287355

288-
// Otherwise, we have a simple primitive. If the value is available, use it,
289-
// otherwise emit a load of the value.
290-
auto Val = AvailableValues[FirstElt];
291-
if (!Val.first)
292-
return B.createLoad(Inst->getLoc(), Address,
293-
LoadOwnershipQualifier::Unqualified);
356+
// We have looked through all of the aggregate values and finally found a
357+
// "primitive value". If the value is available, use it (extracting if we need
358+
// to), otherwise emit a load of the value with the appropriate qualifier.
359+
SILValue AvailableValueAggregator::handlePrimitiveValue(SILType LoadTy,
360+
SILValue Address,
361+
unsigned FirstElt) {
362+
auto &Val = AvailableValueList[FirstElt];
363+
364+
// If the value is not available, load the value.
365+
if (!Val.first) {
366+
return B.createLoad(Loc, Address, LoadOwnershipQualifier::Unqualified);
367+
}
294368

295-
SILValue EltVal = nonDestructivelyExtractSubElement(Val.first, Val.second, B,
296-
Inst->getLoc());
297-
// It must be the same type as LoadTy if available.
369+
// If we have an available value, we know that we know that the available
370+
// value is already being consumed by the store. This means that we must
371+
// insert a copy of EltVal after we extract it if we do not have a trivial
372+
// value. We use SILBuilder::emit*Operation to handle both trivial/non-trivial
373+
// cases without needing to introduce control flow here.
374+
SILValue Aggregate = Val.first;
375+
unsigned AggregateSubElementNumber = Val.second;
376+
377+
// Then extract the subelement from the borrowed aggregate.
378+
SILValue EltVal = nonDestructivelyExtractSubElement(
379+
Aggregate, AggregateSubElementNumber, B, Loc);
298380
assert(EltVal->getType() == LoadTy && "Subelement types mismatch");
299381
return EltVal;
300382
}
@@ -345,18 +427,17 @@ class AllocOptimize {
345427

346428
// Load promotion.
347429
bool hasEscapedAt(SILInstruction *I);
348-
void
349-
updateAvailableValues(SILInstruction *Inst,
350-
llvm::SmallBitVector &RequiredElts,
351-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result,
352-
llvm::SmallBitVector &ConflictingValues);
353-
void computeAvailableValues(
354-
SILInstruction *StartingFrom, llvm::SmallBitVector &RequiredElts,
355-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result);
430+
void updateAvailableValues(SILInstruction *Inst,
431+
llvm::SmallBitVector &RequiredElts,
432+
SmallVectorImpl<AvailableValue> &Result,
433+
llvm::SmallBitVector &ConflictingValues);
434+
void computeAvailableValues(SILInstruction *StartingFrom,
435+
llvm::SmallBitVector &RequiredElts,
436+
SmallVectorImpl<AvailableValue> &Result);
356437
void computeAvailableValuesFrom(
357438
SILBasicBlock::iterator StartingFrom, SILBasicBlock *BB,
358439
llvm::SmallBitVector &RequiredElts,
359-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result,
440+
SmallVectorImpl<AvailableValue> &Result,
360441
llvm::SmallDenseMap<SILBasicBlock *, llvm::SmallBitVector, 32>
361442
&VisitedBlocks,
362443
llvm::SmallBitVector &ConflictingValues);
@@ -431,10 +512,10 @@ bool AllocOptimize::hasEscapedAt(SILInstruction *I) {
431512
/// The specified instruction is a non-load access of the element being
432513
/// promoted. See if it provides a value or refines the demanded element mask
433514
/// used for load promotion.
434-
void AllocOptimize::
435-
updateAvailableValues(SILInstruction *Inst, llvm::SmallBitVector &RequiredElts,
436-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result,
437-
llvm::SmallBitVector &ConflictingValues) {
515+
void AllocOptimize::updateAvailableValues(
516+
SILInstruction *Inst, llvm::SmallBitVector &RequiredElts,
517+
SmallVectorImpl<AvailableValue> &Result,
518+
llvm::SmallBitVector &ConflictingValues) {
438519
// Handle store and assign.
439520
if (isa<StoreInst>(Inst)) {
440521
unsigned StartSubElt = computeSubelement(Inst->getOperand(1), TheMemory);
@@ -514,10 +595,9 @@ updateAvailableValues(SILInstruction *Inst, llvm::SmallBitVector &RequiredElts,
514595
/// captures the available value (plus an indicator of which subelement of that
515596
/// value is needed).
516597
///
517-
void AllocOptimize::
518-
computeAvailableValues(SILInstruction *StartingFrom,
519-
llvm::SmallBitVector &RequiredElts,
520-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result) {
598+
void AllocOptimize::computeAvailableValues(
599+
SILInstruction *StartingFrom, llvm::SmallBitVector &RequiredElts,
600+
SmallVectorImpl<AvailableValue> &Result) {
521601
llvm::SmallDenseMap<SILBasicBlock*, llvm::SmallBitVector, 32> VisitedBlocks;
522602
llvm::SmallBitVector ConflictingValues(Result.size());
523603

@@ -535,13 +615,12 @@ computeAvailableValues(SILInstruction *StartingFrom,
535615
return;
536616
}
537617

538-
void AllocOptimize::
539-
computeAvailableValuesFrom(SILBasicBlock::iterator StartingFrom,
540-
SILBasicBlock *BB,
541-
llvm::SmallBitVector &RequiredElts,
542-
SmallVectorImpl<std::pair<SILValue, unsigned>> &Result,
543-
llvm::SmallDenseMap<SILBasicBlock*, llvm::SmallBitVector, 32> &VisitedBlocks,
544-
llvm::SmallBitVector &ConflictingValues) {
618+
void AllocOptimize::computeAvailableValuesFrom(
619+
SILBasicBlock::iterator StartingFrom, SILBasicBlock *BB,
620+
llvm::SmallBitVector &RequiredElts, SmallVectorImpl<AvailableValue> &Result,
621+
llvm::SmallDenseMap<SILBasicBlock *, llvm::SmallBitVector, 32>
622+
&VisitedBlocks,
623+
llvm::SmallBitVector &ConflictingValues) {
545624
assert(!RequiredElts.none() && "Scanning with a goal of finding nothing?");
546625

547626
// If there is a potential modification in the current block, scan the block
@@ -668,8 +747,8 @@ bool AllocOptimize::promoteLoad(SILInstruction *Inst) {
668747
// Set up the bitvector of elements being demanded by the load.
669748
llvm::SmallBitVector RequiredElts(NumMemorySubElements);
670749
RequiredElts.set(FirstElt, FirstElt+NumLoadSubElements);
671-
672-
SmallVector<std::pair<SILValue, unsigned>, 8> AvailableValues;
750+
751+
SmallVector<AvailableValue, 8> AvailableValues;
673752
AvailableValues.resize(NumMemorySubElements);
674753

675754
// Find out if we have any available values. If no bits are demanded, we
@@ -704,9 +783,9 @@ bool AllocOptimize::promoteLoad(SILInstruction *Inst) {
704783
// Aggregate together all of the subelements into something that has the same
705784
// type as the load did, and emit smaller) loads for any subelements that were
706785
// not available.
707-
auto Load = cast<LoadInst>(Inst);
708-
auto NewVal = aggregateAvailableValues(Load, LoadTy, Load->getOperand(),
709-
AvailableValues, FirstElt);
786+
auto *Load = cast<LoadInst>(Inst);
787+
AvailableValueAggregator Agg(Load, AvailableValues);
788+
SILValue NewVal = Agg.aggregateValues(LoadTy, Load->getOperand(), FirstElt);
710789

711790
++NumLoadPromoted;
712791

@@ -751,8 +830,8 @@ bool AllocOptimize::promoteDestroyAddr(DestroyAddrInst *DAI) {
751830
// Set up the bitvector of elements being demanded by the load.
752831
llvm::SmallBitVector RequiredElts(NumMemorySubElements);
753832
RequiredElts.set(FirstElt, FirstElt+NumLoadSubElements);
754-
755-
SmallVector<std::pair<SILValue, unsigned>, 8> AvailableValues;
833+
834+
SmallVector<AvailableValue, 8> AvailableValues;
756835
AvailableValues.resize(NumMemorySubElements);
757836

758837
// Find out if we have any available values. If no bits are demanded, we
@@ -769,8 +848,8 @@ bool AllocOptimize::promoteDestroyAddr(DestroyAddrInst *DAI) {
769848
// Aggregate together all of the subelements into something that has the same
770849
// type as the load did, and emit smaller) loads for any subelements that were
771850
// not available.
772-
auto NewVal =
773-
aggregateAvailableValues(DAI, LoadTy, Address, AvailableValues, FirstElt);
851+
AvailableValueAggregator Agg(DAI, AvailableValues);
852+
SILValue NewVal = Agg.aggregateValues(LoadTy, Address, FirstElt);
774853

775854
++NumDestroyAddrPromoted;
776855

0 commit comments

Comments
 (0)