@@ -906,75 +906,73 @@ ComplexDeinterleavingGraph::NodePtr
906
906
ComplexDeinterleavingGraph::identifyDotProduct (Value *V) {
907
907
auto *Inst = cast<Instruction>(V);
908
908
909
- if (!TL->isComplexDeinterleavingOperationSupported (ComplexDeinterleavingOperation::CDot, Inst->getType ())) {
910
- LLVM_DEBUG (dbgs () << " Target doesn't support complex deinterleaving operation CDot with the type " << *Inst->getType () << " \n " );
909
+ if (!TL->isComplexDeinterleavingOperationSupported (
910
+ ComplexDeinterleavingOperation::CDot, Inst->getType ())) {
911
+ LLVM_DEBUG (dbgs () << " Target doesn't support complex deinterleaving "
912
+ " operation CDot with the type "
913
+ << *Inst->getType () << " \n " );
911
914
return nullptr ;
912
915
}
913
916
914
917
auto *RealUser = cast<Instruction>(*Inst->user_begin ());
915
918
916
- NodePtr CN = prepareCompositeNode (ComplexDeinterleavingOperation::CDot, Inst, nullptr );
919
+ NodePtr CN =
920
+ prepareCompositeNode (ComplexDeinterleavingOperation::CDot, Inst, nullptr );
917
921
918
922
NodePtr ANode;
919
923
920
- const Intrinsic::ID PartialReduceInt = Intrinsic::experimental_vector_partial_reduce_add;
924
+ const Intrinsic::ID PartialReduceInt =
925
+ Intrinsic::experimental_vector_partial_reduce_add;
921
926
922
927
Value *AReal = nullptr ;
923
928
Value *AImag = nullptr ;
924
929
Value *BReal = nullptr ;
925
930
Value *BImag = nullptr ;
926
931
Value *Phi = nullptr ;
927
932
928
- auto UnwrapCast = [](Value *V) -> Value* {
929
- if (auto *CI = dyn_cast<CastInst>(V))
933
+ auto UnwrapCast = [](Value *V) -> Value * {
934
+ if (auto *CI = dyn_cast<CastInst>(V))
930
935
return CI->getOperand (0 );
931
936
return V;
932
937
};
933
-
934
- auto PatternRot0 =
935
- m_Intrinsic<PartialReduceInt>(
936
- m_Intrinsic<PartialReduceInt>(
937
- m_Value (Phi),
938
- m_Mul (m_Value (BReal), m_Value (AReal))),
939
- m_Neg (m_Mul (m_Value (BImag), m_Value (AImag))));
940
-
941
- auto PatternRot270 =
942
- m_Intrinsic<PartialReduceInt>(
943
- m_Intrinsic<PartialReduceInt>(
944
- m_Value (Phi),
945
- m_Neg (m_Mul (m_Value (BReal), m_Value (AImag)))),
946
- m_Mul (m_Value (BImag), m_Value (AReal)));
947
-
948
- if (match (Inst, PatternRot0)) {
938
+
939
+ auto PatternRot0 = m_Intrinsic<PartialReduceInt>(
940
+ m_Intrinsic<PartialReduceInt>(m_Value (Phi),
941
+ m_Mul (m_Value (BReal), m_Value (AReal))),
942
+ m_Neg (m_Mul (m_Value (BImag), m_Value (AImag))));
943
+
944
+ auto PatternRot270 = m_Intrinsic<PartialReduceInt>(
945
+ m_Intrinsic<PartialReduceInt>(
946
+ m_Value (Phi), m_Neg (m_Mul (m_Value (BReal), m_Value (AImag)))),
947
+ m_Mul (m_Value (BImag), m_Value (AReal)));
948
+
949
+ if (match (Inst, PatternRot0)) {
949
950
CN->Rotation = ComplexDeinterleavingRotation::Rotation_0;
950
- }else if (match (Inst, PatternRot270)) {
951
+ } else if (match (Inst, PatternRot270)) {
951
952
CN->Rotation = ComplexDeinterleavingRotation::Rotation_270;
952
- }else {
953
+ } else {
953
954
Value *A0, *A1;
954
955
// The rotations 90 and 180 share the same operation pattern, so inspect the
955
- // order of the operands, identifying where the real and imaginary components
956
- // of A go, to discern between the aforementioned rotations.
957
- auto PatternRot90Rot180 =
958
- m_Intrinsic<PartialReduceInt>(
959
- m_Intrinsic<PartialReduceInt>(
960
- m_Value (Phi),
961
- m_Mul (m_Value (BReal), m_Value (A0))
962
- ),
963
- m_Mul (m_Value (BImag), m_Value (A1)));
964
-
965
- if (!match (Inst, PatternRot90Rot180))
956
+ // order of the operands, identifying where the real and imaginary
957
+ // components of A go, to discern between the aforementioned rotations.
958
+ auto PatternRot90Rot180 = m_Intrinsic<PartialReduceInt>(
959
+ m_Intrinsic<PartialReduceInt>(m_Value (Phi),
960
+ m_Mul (m_Value (BReal), m_Value (A0))),
961
+ m_Mul (m_Value (BImag), m_Value (A1)));
962
+
963
+ if (!match (Inst, PatternRot90Rot180))
966
964
return nullptr ;
967
-
965
+
968
966
A0 = UnwrapCast (A0);
969
967
A1 = UnwrapCast (A1);
970
968
971
969
// Test if A0 is real/A1 is imag
972
970
ANode = identifyNode (A0, A1);
973
- if (!ANode) {
971
+ if (!ANode) {
974
972
// Test if A0 is imag/A1 is real
975
973
ANode = identifyNode (A1, A0);
976
974
// Unable to identify operand components, thus unable to identify rotation
977
- if (!ANode)
975
+ if (!ANode)
978
976
return nullptr ;
979
977
CN->Rotation = ComplexDeinterleavingRotation::Rotation_90;
980
978
AReal = A1;
@@ -994,10 +992,14 @@ ComplexDeinterleavingGraph::identifyDotProduct(Value *V) {
994
992
bool WasANodeFromCache = false ;
995
993
NodePtr Node = identifyNode (AReal, AImag, WasANodeFromCache);
996
994
997
- // In the case that a node was identified to figure out the rotation, ensure that trying to
998
- // identify a node with AReal and AImag post-unwrap results in the same node
999
- if (Node && ANode && !WasANodeFromCache) {
1000
- LLVM_DEBUG (dbgs () << " Identified node is different from previously identified node. Unable to confidently generate a complex operation node\n " );
995
+ // In the case that a node was identified to figure out the rotation, ensure
996
+ // that trying to identify a node with AReal and AImag post-unwrap results in
997
+ // the same node
998
+ if (Node && ANode && !WasANodeFromCache) {
999
+ LLVM_DEBUG (
1000
+ dbgs ()
1001
+ << " Identified node is different from previously identified node. "
1002
+ " Unable to confidently generate a complex operation node\n " );
1001
1003
return nullptr ;
1002
1004
}
1003
1005
@@ -1014,18 +1016,19 @@ ComplexDeinterleavingGraph::identifyPartialReduction(Value *R, Value *I) {
1014
1016
return nullptr ;
1015
1017
1016
1018
VectorType *RealTy = dyn_cast<VectorType>(R->getType ());
1017
- if (!RealTy)
1019
+ if (!RealTy)
1018
1020
return nullptr ;
1019
1021
VectorType *ImagTy = dyn_cast<VectorType>(I->getType ());
1020
- if (!ImagTy)
1022
+ if (!ImagTy)
1021
1023
return nullptr ;
1022
1024
1023
- if (RealTy->isScalableTy () != ImagTy->isScalableTy ())
1025
+ if (RealTy->isScalableTy () != ImagTy->isScalableTy ())
1024
1026
return nullptr ;
1025
- if (RealTy->getElementType () != ImagTy->getElementType ())
1027
+ if (RealTy->getElementType () != ImagTy->getElementType ())
1026
1028
return nullptr ;
1027
1029
1028
- // `I` is known to only have one user, so iterate over the Phi (R) users to find the common user between R and I
1030
+ // `I` is known to only have one user, so iterate over the Phi (R) users to
1031
+ // find the common user between R and I
1029
1032
auto *CommonUser = *I->user_begin ();
1030
1033
bool CommonUserFound = false ;
1031
1034
for (auto *User : R->users ()) {
@@ -1039,10 +1042,11 @@ ComplexDeinterleavingGraph::identifyPartialReduction(Value *R, Value *I) {
1039
1042
return nullptr ;
1040
1043
1041
1044
auto *IInst = dyn_cast<IntrinsicInst>(CommonUser);
1042
- if (!IInst || IInst->getIntrinsicID () != Intrinsic::experimental_vector_partial_reduce_add)
1045
+ if (!IInst || IInst->getIntrinsicID () !=
1046
+ Intrinsic::experimental_vector_partial_reduce_add)
1043
1047
return nullptr ;
1044
1048
1045
- if (NodePtr CN = identifyDotProduct (IInst))
1049
+ if (NodePtr CN = identifyDotProduct (IInst))
1046
1050
return CN;
1047
1051
1048
1052
return nullptr ;
@@ -1063,7 +1067,7 @@ ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I, bool &FromCache) {
1063
1067
return It->second ;
1064
1068
}
1065
1069
1066
- if (NodePtr CN = identifyPartialReduction (R, I))
1070
+ if (NodePtr CN = identifyPartialReduction (R, I))
1067
1071
return CN;
1068
1072
1069
1073
bool IsReduction = RealPHI == R && (!ImagPHI || ImagPHI == I);
@@ -2138,8 +2142,7 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
2138
2142
assert (!Input1 || (Input0->getType () == Input1->getType () &&
2139
2143
" Node inputs need to be of the same type" ));
2140
2144
ReplacementNode = TL->createComplexDeinterleavingIR (
2141
- Builder, Node->Operation , Node->Rotation , Input0, Input1,
2142
- Accumulator);
2145
+ Builder, Node->Operation , Node->Rotation , Input0, Input1, Accumulator);
2143
2146
break ;
2144
2147
}
2145
2148
case ComplexDeinterleavingOperation::CAdd:
0 commit comments