@@ -201,9 +201,7 @@ struct ComplexDeinterleavingCompositeNode {
201
201
}
202
202
}
203
203
204
- bool AreOperandsValid () {
205
- return OperandsValid;
206
- }
204
+ bool AreOperandsValid () { return OperandsValid; }
207
205
};
208
206
209
207
class ComplexDeinterleavingGraph {
@@ -918,11 +916,11 @@ ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I, bool &FromCache) {
918
916
return It->second ;
919
917
}
920
918
921
- if (NodePtr CN = identifyPartialReduction (R, I))
919
+ if (NodePtr CN = identifyPartialReduction (R, I))
922
920
return CN;
923
921
924
922
bool IsReduction = RealPHI == R && (!ImagPHI || ImagPHI == I);
925
- if (!IsReduction && R->getType () != I->getType ())
923
+ if (!IsReduction && R->getType () != I->getType ())
926
924
return nullptr ;
927
925
928
926
if (NodePtr CN = identifySplat (R, I))
@@ -1450,18 +1448,20 @@ bool ComplexDeinterleavingGraph::identifyNodes(Instruction *RootI) {
1450
1448
if (It != RootToNode.end ()) {
1451
1449
auto RootNode = It->second ;
1452
1450
assert (RootNode->Operation ==
1453
- ComplexDeinterleavingOperation::ReductionOperation || RootNode->Operation == ComplexDeinterleavingOperation::ReductionSingle);
1451
+ ComplexDeinterleavingOperation::ReductionOperation ||
1452
+ RootNode->Operation ==
1453
+ ComplexDeinterleavingOperation::ReductionSingle);
1454
1454
// Find out which part, Real or Imag, comes later, and only if we come to
1455
1455
// the latest part, add it to OrderedRoots.
1456
1456
auto *R = cast<Instruction>(RootNode->Real );
1457
1457
auto *I = RootNode->Imag ? cast<Instruction>(RootNode->Imag ) : nullptr ;
1458
1458
1459
1459
Instruction *ReplacementAnchor;
1460
- if (I)
1460
+ if (I)
1461
1461
ReplacementAnchor = R->comesBefore (I) ? I : R;
1462
- else
1462
+ else
1463
1463
ReplacementAnchor = R;
1464
-
1464
+
1465
1465
if (ReplacementAnchor != RootI)
1466
1466
return false ;
1467
1467
OrderedRoots.push_back (RootI);
@@ -1553,7 +1553,7 @@ void ComplexDeinterleavingGraph::identifyReductionNodes() {
1553
1553
for (size_t j = i + 1 ; j < OperationInstruction.size (); ++j) {
1554
1554
if (Processed[j])
1555
1555
continue ;
1556
-
1556
+
1557
1557
auto *Imag = OperationInstruction[j];
1558
1558
if (Real->getType () != Imag->getType ())
1559
1559
continue ;
@@ -1588,18 +1588,20 @@ void ComplexDeinterleavingGraph::identifyReductionNodes() {
1588
1588
1589
1589
// We want to check that we have 2 operands, but the function attributes
1590
1590
// being counted as operands bloats this value.
1591
- if (Real->getNumOperands () < 2 )
1591
+ if (Real->getNumOperands () < 2 )
1592
1592
continue ;
1593
1593
1594
1594
RealPHI = ReductionInfo[Real].first ;
1595
1595
ImagPHI = nullptr ;
1596
1596
PHIsFound = false ;
1597
1597
auto Node = identifyNode (Real->getOperand (0 ), Real->getOperand (1 ));
1598
- if (Node && PHIsFound) {
1599
- LLVM_DEBUG (dbgs () << " Identified single reduction starting from instruction: "
1600
- << *Real << " /" << *ReductionInfo[Real].second << " \n " );
1598
+ if (Node && PHIsFound) {
1599
+ LLVM_DEBUG (
1600
+ dbgs () << " Identified single reduction starting from instruction: "
1601
+ << *Real << " /" << *ReductionInfo[Real].second << " \n " );
1601
1602
Processed[i] = true ;
1602
- auto RootNode = prepareCompositeNode (ComplexDeinterleavingOperation::ReductionSingle, Real, nullptr );
1603
+ auto RootNode = prepareCompositeNode (
1604
+ ComplexDeinterleavingOperation::ReductionSingle, Real, nullptr );
1603
1605
RootNode->addOperand (Node);
1604
1606
RootToNode[Real] = RootNode;
1605
1607
submitCompositeNode (RootNode);
@@ -2059,7 +2061,8 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
2059
2061
return ReplacementNode;
2060
2062
}
2061
2063
2062
- void ComplexDeinterleavingGraph::processReductionSingle (Value *OperationReplacement, RawNodePtr Node) {
2064
+ void ComplexDeinterleavingGraph::processReductionSingle (
2065
+ Value *OperationReplacement, RawNodePtr Node) {
2063
2066
auto *Real = cast<Instruction>(Node->Real );
2064
2067
auto *OldPHI = ReductionInfo[Real].first ;
2065
2068
auto *NewPHI = OldToNewPHI[OldPHI];
@@ -2071,21 +2074,21 @@ void ComplexDeinterleavingGraph::processReductionSingle(Value *OperationReplacem
2071
2074
IRBuilder<> Builder (Incoming->getTerminator ());
2072
2075
2073
2076
Value *NewInit = nullptr ;
2074
- if (auto *C = dyn_cast<Constant>(Init)) {
2075
- if (C->isZeroValue ())
2077
+ if (auto *C = dyn_cast<Constant>(Init)) {
2078
+ if (C->isZeroValue ())
2076
2079
NewInit = Constant::getNullValue (NewVTy);
2077
2080
}
2078
2081
2079
2082
if (!NewInit)
2080
2083
NewInit = Builder.CreateIntrinsic (Intrinsic::vector_interleave2, NewVTy,
2081
- {Init, Constant::getNullValue (VTy)});
2084
+ {Init, Constant::getNullValue (VTy)});
2082
2085
2083
2086
NewPHI->addIncoming (NewInit, Incoming);
2084
2087
NewPHI->addIncoming (OperationReplacement, BackEdge);
2085
2088
2086
2089
auto *FinalReduction = ReductionInfo[Real].second ;
2087
2090
Builder.SetInsertPoint (&*FinalReduction->getParent ()->getFirstInsertionPt ());
2088
- // TODO Ensure that the `AddReduce` here matches the original, found in `FinalReduction`
2091
+
2089
2092
auto *AddReduce = Builder.CreateAddReduce (OperationReplacement);
2090
2093
FinalReduction->replaceAllUsesWith (AddReduce);
2091
2094
}
@@ -2151,7 +2154,8 @@ void ComplexDeinterleavingGraph::replaceNodes() {
2151
2154
ReductionInfo[RootImag].first ->removeIncomingValue (BackEdge);
2152
2155
DeadInstrRoots.push_back (RootReal);
2153
2156
DeadInstrRoots.push_back (RootImag);
2154
- } else if (RootNode->Operation == ComplexDeinterleavingOperation::ReductionSingle) {
2157
+ } else if (RootNode->Operation ==
2158
+ ComplexDeinterleavingOperation::ReductionSingle) {
2155
2159
auto *RootInst = cast<Instruction>(RootNode->Real );
2156
2160
ReductionInfo[RootInst].first ->removeIncomingValue (BackEdge);
2157
2161
DeadInstrRoots.push_back (ReductionInfo[RootInst].second );
0 commit comments