Skip to content

Commit 010f329

Browse files
zakk0610zakk0610
committed
[RISCV][Clang] Support policy function for all vector segment load.
We will switch all UndefValue to PoisonValue in follow up patches. Reviewed By: kito-cheng Differential Revision: https://reviews.llvm.org/D126750
1 parent 8611a77 commit 010f329

File tree

14 files changed

+845
-85
lines changed

14 files changed

+845
-85
lines changed

‎clang/include/clang/Basic/riscv_vector.td

Lines changed: 145 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -843,22 +843,29 @@ multiclass RVVUnitStridedSegLoad<string op> {
843843
IRName = op # nf,
844844
MaskedIRName = op # nf # "_mask",
845845
NF = nf,
846-
SupportOverloading = false,
847846
ManualCodegen = [{
848847
{
849-
// builtin: (val0 address, val1 address, ..., ptr, vl)
850848
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
851-
IntrinsicTypes = {ResultType, Ops[NF + 1]->getType()};
849+
// TA builtin: (val0 address, val1 address, ..., ptr, vl)
850+
// TU builtin: (val0 address, ..., passthru0, ..., ptr, vl)
851+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
852852
// intrinsic: (passthru0, passthru1, ..., ptr, vl)
853853
SmallVector<llvm::Value*, 10> Operands;
854-
for (unsigned I = 0; I < NF; ++I)
855-
Operands.push_back(llvm::UndefValue::get(ResultType));
856-
Operands.push_back(Ops[NF]);
857-
Operands.push_back(Ops[NF + 1]);
854+
if (DefaultPolicy == TAIL_AGNOSTIC) {
855+
for (unsigned I = 0; I < NF; ++I)
856+
Operands.push_back(llvm::UndefValue::get(ResultType));
857+
Operands.push_back(Ops[NF]);
858+
Operands.push_back(Ops[NF + 1]);
859+
} else {
860+
for (unsigned I = 0; I < NF; ++I)
861+
Operands.push_back(Ops[NF + I]);
862+
Operands.push_back(Ops[2 * NF]);
863+
Operands.push_back(Ops[2 * NF + 1]);
864+
}
858865
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
859866
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
860867
clang::CharUnits Align =
861-
CGM.getNaturalPointeeTypeAlignment(E->getArg(NF)->getType());
868+
CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
862869
llvm::Value *V;
863870
for (unsigned I = 0; I < NF; ++I) {
864871
llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I});
@@ -869,17 +876,26 @@ multiclass RVVUnitStridedSegLoad<string op> {
869876
}],
870877
MaskedManualCodegen = [{
871878
{
879+
// TAMA builtin: (val0 address, ..., mask, ptr, vl)
872880
// builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl)
873881
// intrinsic: (maskedoff0, ..., ptr, mask, vl)
874-
IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
875-
Ops[2 * NF + 2]->getType()};
882+
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
883+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
876884
SmallVector<llvm::Value*, 12> Operands;
877-
for (unsigned I = 0; I < NF; ++I)
878-
Operands.push_back(Ops[NF + I + 1]);
879-
Operands.push_back(Ops[2 * NF + 1]);
880-
Operands.push_back(Ops[NF]);
881-
Operands.push_back(Ops[2 * NF + 2]);
882-
Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
885+
if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
886+
for (unsigned I = 0; I < NF; ++I)
887+
Operands.push_back(llvm::UndefValue::get(ResultType));
888+
Operands.push_back(Ops[NF + 1]);
889+
Operands.push_back(Ops[NF]);
890+
Operands.push_back(Ops[NF + 2]);
891+
} else {
892+
for (unsigned I = 0; I < NF; ++I)
893+
Operands.push_back(Ops[NF + I + 1]);
894+
Operands.push_back(Ops[2 * NF + 1]);
895+
Operands.push_back(Ops[NF]);
896+
Operands.push_back(Ops[2 * NF + 2]);
897+
}
898+
Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
883899
assert(Operands.size() == NF + 4);
884900
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
885901
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -918,19 +934,28 @@ multiclass RVVUnitStridedSegLoadFF<string op> {
918934
IRName = op # nf # "ff",
919935
MaskedIRName = op # nf # "ff_mask",
920936
NF = nf,
921-
SupportOverloading = false,
922937
ManualCodegen = [{
923938
{
924-
// builtin: (val0 address, val1 address, ..., ptr, new_vl, vl)
939+
// TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl)
940+
// TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl)
925941
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
926-
IntrinsicTypes = {ResultType, Ops[NF + 2]->getType()};
942+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
927943
// intrinsic: (passthru0, passthru1, ..., ptr, vl)
928944
SmallVector<llvm::Value*, 12> Operands;
929-
for (unsigned I = 0; I < NF; ++I)
930-
Operands.push_back(llvm::UndefValue::get(ResultType));
931-
Operands.push_back(Ops[NF]);
932-
Operands.push_back(Ops[NF + 2]);
933-
Value *NewVL = Ops[NF + 1];
945+
Value *NewVL;
946+
if (DefaultPolicy == TAIL_AGNOSTIC) {
947+
for (unsigned I = 0; I < NF; ++I)
948+
Operands.push_back(llvm::UndefValue::get(ResultType));
949+
Operands.push_back(Ops[NF]);
950+
Operands.push_back(Ops[NF + 2]);
951+
NewVL = Ops[NF + 1];
952+
} else {
953+
for (unsigned I = 0; I < NF; ++I)
954+
Operands.push_back(Ops[NF + I]);
955+
Operands.push_back(Ops[2 * NF]);
956+
Operands.push_back(Ops[2 * NF + 2]);
957+
NewVL = Ops[2 * NF + 1];
958+
}
934959
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
935960
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
936961
clang::CharUnits Align =
@@ -946,18 +971,29 @@ multiclass RVVUnitStridedSegLoadFF<string op> {
946971
}],
947972
MaskedManualCodegen = [{
948973
{
974+
// TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl)
949975
// builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl)
950976
// intrinsic: (maskedoff0, ..., ptr, mask, vl)
951-
IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
952-
Ops[2 * NF + 3]->getType()};
977+
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
978+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
953979
SmallVector<llvm::Value*, 12> Operands;
954-
for (unsigned I = 0; I < NF; ++I)
955-
Operands.push_back(Ops[NF + I + 1]);
956-
Operands.push_back(Ops[2 * NF + 1]);
957-
Operands.push_back(Ops[NF]);
958-
Operands.push_back(Ops[2 * NF + 3]);
959-
Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
960-
Value *NewVL = Ops[2 * NF + 2];
980+
Value *NewVL;
981+
if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
982+
for (unsigned I = 0; I < NF; ++I)
983+
Operands.push_back(llvm::UndefValue::get(ResultType));
984+
Operands.push_back(Ops[NF + 1]);
985+
Operands.push_back(Ops[NF]);
986+
Operands.push_back(Ops[NF + 3]);
987+
NewVL = Ops[NF + 2];
988+
} else {
989+
for (unsigned I = 0; I < NF; ++I)
990+
Operands.push_back(Ops[NF + I + 1]);
991+
Operands.push_back(Ops[2 * NF + 1]);
992+
Operands.push_back(Ops[NF]);
993+
Operands.push_back(Ops[2 * NF + 3]);
994+
NewVL = Ops[2 * NF + 2];
995+
}
996+
Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
961997
assert(Operands.size() == NF + 4);
962998
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
963999
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -997,19 +1033,27 @@ multiclass RVVStridedSegLoad<string op> {
9971033
IRName = op # nf,
9981034
MaskedIRName = op # nf # "_mask",
9991035
NF = nf,
1000-
SupportOverloading = false,
10011036
ManualCodegen = [{
10021037
{
1003-
// builtin: (val0 address, val1 address, ..., ptr, stride, vl)
1038+
// TA builtin: (val0 address, val1 address, ..., ptr, stride, vl)
1039+
// TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl)
10041040
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
1005-
IntrinsicTypes = {ResultType, Ops[NF + 2]->getType()};
1041+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
10061042
// intrinsic: (passthru0, passthru1, ..., ptr, stride, vl)
10071043
SmallVector<llvm::Value*, 12> Operands;
1008-
for (unsigned I = 0; I < NF; ++I)
1009-
Operands.push_back(llvm::UndefValue::get(ResultType));
1010-
Operands.push_back(Ops[NF]);
1011-
Operands.push_back(Ops[NF + 1]);
1012-
Operands.push_back(Ops[NF + 2]);
1044+
if (DefaultPolicy == TAIL_AGNOSTIC) {
1045+
for (unsigned I = 0; I < NF; ++I)
1046+
Operands.push_back(llvm::UndefValue::get(ResultType));
1047+
Operands.push_back(Ops[NF]);
1048+
Operands.push_back(Ops[NF + 1]);
1049+
Operands.push_back(Ops[NF + 2]);
1050+
} else {
1051+
for (unsigned I = 0; I < NF; ++I)
1052+
Operands.push_back(Ops[NF + I]);
1053+
Operands.push_back(Ops[2 * NF]);
1054+
Operands.push_back(Ops[2 * NF + 1]);
1055+
Operands.push_back(Ops[2 * NF + 2]);
1056+
}
10131057
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
10141058
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
10151059
clang::CharUnits Align =
@@ -1024,18 +1068,28 @@ multiclass RVVStridedSegLoad<string op> {
10241068
}],
10251069
MaskedManualCodegen = [{
10261070
{
1071+
//TAMA builtin: (val0 address, ..., mask, ptr, stride, vl)
10271072
// builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl)
10281073
// intrinsic: (maskedoff0, ..., ptr, stride, mask, vl)
1029-
IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
1030-
Ops[2 * NF + 3]->getType()};
1074+
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
1075+
IntrinsicTypes = {ResultType, Ops.back()->getType()};
10311076
SmallVector<llvm::Value*, 12> Operands;
1032-
for (unsigned I = 0; I < NF; ++I)
1033-
Operands.push_back(Ops[NF + I + 1]);
1034-
Operands.push_back(Ops[2 * NF + 1]);
1035-
Operands.push_back(Ops[2 * NF + 2]);
1036-
Operands.push_back(Ops[NF]);
1037-
Operands.push_back(Ops[2 * NF + 3]);
1038-
Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
1077+
if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
1078+
for (unsigned I = 0; I < NF; ++I)
1079+
Operands.push_back(llvm::UndefValue::get(ResultType));
1080+
Operands.push_back(Ops[NF + 1]);
1081+
Operands.push_back(Ops[NF + 2]);
1082+
Operands.push_back(Ops[NF]);
1083+
Operands.push_back(Ops[NF + 3]);
1084+
} else {
1085+
for (unsigned I = 0; I < NF; ++I)
1086+
Operands.push_back(Ops[NF + I + 1]);
1087+
Operands.push_back(Ops[2 * NF + 1]);
1088+
Operands.push_back(Ops[2 * NF + 2]);
1089+
Operands.push_back(Ops[NF]);
1090+
Operands.push_back(Ops[2 * NF + 3]);
1091+
}
1092+
Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
10391093
assert(Operands.size() == NF + 5);
10401094
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
10411095
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -1072,16 +1126,26 @@ multiclass RVVIndexedSegLoad<string op> {
10721126
NF = nf,
10731127
ManualCodegen = [{
10741128
{
1075-
// builtin: (val0 address, val1 address, ..., ptr, index, vl)
1129+
// TA builtin: (val0 address, val1 address, ..., ptr, index, vl)
1130+
// TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl)
10761131
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
1077-
IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops[NF + 2]->getType()};
10781132
// intrinsic: (passthru0, passthru1, ..., ptr, index, vl)
10791133
SmallVector<llvm::Value*, 12> Operands;
1080-
for (unsigned I = 0; I < NF; ++I)
1081-
Operands.push_back(llvm::UndefValue::get(ResultType));
1082-
Operands.push_back(Ops[NF]);
1083-
Operands.push_back(Ops[NF + 1]);
1084-
Operands.push_back(Ops[NF + 2]);
1134+
if (DefaultPolicy == TAIL_AGNOSTIC) {
1135+
for (unsigned I = 0; I < NF; ++I)
1136+
Operands.push_back(llvm::UndefValue::get(ResultType));
1137+
Operands.push_back(Ops[NF]);
1138+
Operands.push_back(Ops[NF + 1]);
1139+
Operands.push_back(Ops[NF + 2]);
1140+
IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()};
1141+
} else {
1142+
for (unsigned I = 0; I < NF; ++I)
1143+
Operands.push_back(Ops[NF + I]);
1144+
Operands.push_back(Ops[2 * NF]);
1145+
Operands.push_back(Ops[2 * NF + 1]);
1146+
Operands.push_back(Ops[2 * NF + 2]);
1147+
IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()};
1148+
}
10851149
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
10861150
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
10871151
clang::CharUnits Align =
@@ -1096,18 +1160,29 @@ multiclass RVVIndexedSegLoad<string op> {
10961160
}],
10971161
MaskedManualCodegen = [{
10981162
{
1163+
// TAMA builtin: (val0 address, ..., mask, ptr, index, vl)
10991164
// builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl)
1100-
IntrinsicTypes = {ConvertType(E->getArg(0)->getType()->getPointeeType()),
1101-
Ops[2 * NF + 2]->getType(), Ops[2 * NF + 3]->getType()};
1165+
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
11021166
// intrinsic: (maskedoff0, ..., ptr, index, mask, vl)
11031167
SmallVector<llvm::Value*, 12> Operands;
1104-
for (unsigned I = 0; I < NF; ++I)
1105-
Operands.push_back(Ops[NF + I + 1]);
1106-
Operands.push_back(Ops[2 * NF + 1]);
1107-
Operands.push_back(Ops[2 * NF + 2]);
1108-
Operands.push_back(Ops[NF]);
1109-
Operands.push_back(Ops[2 * NF + 3]);
1110-
Operands.push_back(ConstantInt::get(Ops.back()->getType(), TAIL_UNDISTURBED));
1168+
if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
1169+
for (unsigned I = 0; I < NF; ++I)
1170+
Operands.push_back(llvm::UndefValue::get(ResultType));
1171+
Operands.push_back(Ops[NF + 1]);
1172+
Operands.push_back(Ops[NF + 2]);
1173+
Operands.push_back(Ops[NF]);
1174+
Operands.push_back(Ops[NF + 3]);
1175+
IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()};
1176+
} else {
1177+
for (unsigned I = 0; I < NF; ++I)
1178+
Operands.push_back(Ops[NF + I + 1]);
1179+
Operands.push_back(Ops[2 * NF + 1]);
1180+
Operands.push_back(Ops[2 * NF + 2]);
1181+
Operands.push_back(Ops[NF]);
1182+
Operands.push_back(Ops[2 * NF + 3]);
1183+
IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()};
1184+
}
1185+
Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
11111186
assert(Operands.size() == NF + 5);
11121187
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
11131188
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -1649,14 +1724,15 @@ defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>;
16491724
defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>;
16501725

16511726
// 7.8 Vector Load/Store Segment Instructions
1652-
// TODO: Support policy function for segment load.
1653-
let UnMaskedPolicyScheme = NonePolicy,
1654-
MaskedPolicyScheme = NonePolicy in {
1727+
let UnMaskedPolicyScheme = HasPassthruOperand in {
16551728
defm : RVVUnitStridedSegLoad<"vlseg">;
16561729
defm : RVVUnitStridedSegLoadFF<"vlseg">;
16571730
defm : RVVStridedSegLoad<"vlsseg">;
16581731
defm : RVVIndexedSegLoad<"vluxseg">;
16591732
defm : RVVIndexedSegLoad<"vloxseg">;
1733+
}
1734+
let UnMaskedPolicyScheme = NonePolicy,
1735+
MaskedPolicyScheme = NonePolicy in {
16601736
defm : RVVUnitStridedSegStore<"vsseg">;
16611737
defm : RVVStridedSegStore<"vssseg">;
16621738
defm : RVVIndexedSegStore<"vsuxseg">;

‎clang/lib/Support/RISCVVIntrinsicUtils.cpp

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -941,8 +941,8 @@ llvm::SmallVector<PrototypeDescriptor> RVVIntrinsic::computeBuiltinTypes(
941941
if (IsMasked) {
942942
// If HasMaskedOffOperand, insert result type as first input operand if
943943
// need.
944-
if (HasMaskedOffOperand) {
945-
if (NF == 1 && DefaultPolicy != Policy::TAMA) {
944+
if (HasMaskedOffOperand && DefaultPolicy != Policy::TAMA) {
945+
if (NF == 1) {
946946
NewPrototype.insert(NewPrototype.begin() + 1, NewPrototype[0]);
947947
} else if (NF > 1) {
948948
// Convert
@@ -971,22 +971,34 @@ llvm::SmallVector<PrototypeDescriptor> RVVIntrinsic::computeBuiltinTypes(
971971
// If IsMasked, insert PrototypeDescriptor:Mask as first input operand.
972972
NewPrototype.insert(NewPrototype.begin() + 1, PrototypeDescriptor::Mask);
973973
}
974-
} else if (NF == 1) {
975-
if (DefaultPolicy == Policy::TU && HasPassthruOp && !IsPrototypeDefaultTU)
976-
NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]);
977-
else if (DefaultPolicy == Policy::TA && HasPassthruOp &&
978-
IsPrototypeDefaultTU)
979-
NewPrototype.erase(NewPrototype.begin() + 1);
980-
if (DefaultScheme == PolicyScheme::HasPassthruOperandAtIdx1) {
981-
if (DefaultPolicy == Policy::TU && !IsPrototypeDefaultTU) {
982-
// Insert undisturbed output to index 1
983-
NewPrototype.insert(NewPrototype.begin() + 2, NewPrototype[0]);
984-
} else if (DefaultPolicy == Policy::TA && IsPrototypeDefaultTU) {
985-
// Erase passthru for TA policy
986-
NewPrototype.erase(NewPrototype.begin() + 2);
974+
} else {
975+
if (NF == 1) {
976+
if (DefaultPolicy == Policy::TU && HasPassthruOp && !IsPrototypeDefaultTU)
977+
NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]);
978+
else if (DefaultPolicy == Policy::TA && HasPassthruOp &&
979+
IsPrototypeDefaultTU)
980+
NewPrototype.erase(NewPrototype.begin() + 1);
981+
if (DefaultScheme == PolicyScheme::HasPassthruOperandAtIdx1) {
982+
if (DefaultPolicy == Policy::TU && !IsPrototypeDefaultTU) {
983+
// Insert undisturbed output to index 1
984+
NewPrototype.insert(NewPrototype.begin() + 2, NewPrototype[0]);
985+
} else if (DefaultPolicy == Policy::TA && IsPrototypeDefaultTU) {
986+
// Erase passthru for TA policy
987+
NewPrototype.erase(NewPrototype.begin() + 2);
988+
}
987989
}
990+
} else if (DefaultPolicy == Policy::TU && HasPassthruOp) {
991+
// NF > 1 cases for segment load operations.
992+
// Convert
993+
// (void, op0 address, op1 address, ...)
994+
// to
995+
// (void, op0 address, op1 address, maskedoff0, maskedoff1, ...)
996+
PrototypeDescriptor MaskoffType = Prototype[1];
997+
MaskoffType.TM &= ~static_cast<uint8_t>(TypeModifier::Pointer);
998+
for (unsigned I = 0; I < NF; ++I)
999+
NewPrototype.insert(NewPrototype.begin() + NF + 1, MaskoffType);
9881000
}
989-
}
1001+
}
9901002

9911003
// If HasVL, append PrototypeDescriptor:VL to last operand
9921004
if (HasVL)

0 commit comments

Comments
 (0)