Skip to content

Commit 7afaa75

Browse files
MacDueMacDue
committed
[IR] Add llvm.sincospi intrinsic
This adds the `llvm.sincospi` intrinsic, legalization, and lowering (mostly reusing the lowering for sincos and frexp). The `llvm.sincospi` intrinsic takes a floating-point value and returns both the sine and cosine of the value multiplied by pi. It computes the result more accurately than the naive approach of doing the multiplication ahead of time, especially for large input values. ``` declare { float, float } @llvm.sincospi.f32(float %Val) declare { double, double } @llvm.sincospi.f64(double %Val) declare { x86_fp80, x86_fp80 } @llvm.sincospi.f80(x86_fp80 %Val) declare { fp128, fp128 } @llvm.sincospi.f128(fp128 %Val) declare { ppc_fp128, ppc_fp128 } @llvm.sincospi.ppcf128(ppc_fp128 %Val) declare { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float> %Val) ``` Currently, the default lowering of this intrinsic relies on the `sincospi[f|l]` functions being available in the target's runtime (e.g. libc).
1 parent a91d188 commit 7afaa75

16 files changed

+442
-24
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16122,6 +16122,52 @@ of the argument.
1612216122
When specified with the fast-math-flag 'afn', the result may be approximated
1612316123
using a less accurate calculation.
1612416124

16125+
'``llvm.sincospi.*``' Intrinsic
16126+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
16127+
16128+
Syntax:
16129+
"""""""
16130+
16131+
This is an overloaded intrinsic. You can use ``llvm.sincospi`` on any
16132+
floating-point or vector of floating-point type. Not all targets support
16133+
all types however.
16134+
16135+
::
16136+
16137+
declare { float, float } @llvm.sincospi.f32(float %Val)
16138+
declare { double, double } @llvm.sincospi.f64(double %Val)
16139+
declare { x86_fp80, x86_fp80 } @llvm.sincospi.f80(x86_fp80 %Val)
16140+
declare { fp128, fp128 } @llvm.sincospi.f128(fp128 %Val)
16141+
declare { ppc_fp128, ppc_fp128 } @llvm.sincospi.ppcf128(ppc_fp128 %Val)
16142+
declare { <4 x float>, <4 x float> } @llvm.sincospi.v4f32(<4 x float> %Val)
16143+
16144+
Overview:
16145+
"""""""""
16146+
16147+
The '``llvm.sincospi.*``' intrinsics returns the sine and cosine of pi*operand.
16148+
16149+
Arguments:
16150+
""""""""""
16151+
16152+
The argument is a :ref:`floating-point <t_floating>` value or
16153+
:ref:`vector <t_vector>` of floating-point values. Returns two values matching
16154+
the argument type in a struct.
16155+
16156+
Semantics:
16157+
""""""""""
16158+
16159+
This is equivalent to the `llvm.sincos.*` intrinsic where the argument has been
16160+
multiplied by pi, however, it computes the result more accurately especially
16161+
for large input values.
16162+
16163+
.. note::
16164+
16165+
Currently, the default lowering of this intrinsic relies on the `sincospi[f|l]`
16166+
functions being available in the target's runtime (e.g. libc).
16167+
16168+
When specified with the fast-math-flag 'afn', the result may be approximated
16169+
using a less accurate calculation.
16170+
1612516171
'``llvm.modf.*``' Intrinsic
1612616172
^^^^^^^^^^^^^^^^^^^^^^^^^^^
1612716173

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
21012101
case Intrinsic::sincos:
21022102
ISD = ISD::FSINCOS;
21032103
break;
2104+
case Intrinsic::sincospi:
2105+
ISD = ISD::FSINCOSPI;
2106+
break;
21042107
case Intrinsic::modf:
21052108
ISD = ISD::FMODF;
21062109
break;

‎llvm/include/llvm/CodeGen/ISDOpcodes.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1058,6 +1058,10 @@ enum NodeType {
10581058
/// FSINCOS - Compute both fsin and fcos as a single operation.
10591059
FSINCOS,
10601060

1061+
/// FSINCOSPI - Compute both the sine and cosine times pi more accurately
1062+
/// than FSINCOS(pi*x), especially for large x.
1063+
FSINCOSPI,
1064+
10611065
/// FMODF - Decomposes the given arg in integral and fractional parts, each
10621066
/// having the same type and sign as the arg.
10631067
FMODF,

‎llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ Libcall getFREXP(EVT RetVT);
6666
/// UNKNOWN_LIBCALL if there is none.
6767
Libcall getFSINCOS(EVT RetVT);
6868

69+
/// getSINCOSPI - Return the SINCOSPI_* value for the given types, or
70+
/// UNKNOWN_LIBCALL if there is none.
71+
Libcall getSINCOSPI(EVT RetVT);
72+
6973
/// getMODF - Return the MODF_* value for the given types, or
7074
/// UNKNOWN_LIBCALL if there is none.
7175
Libcall getMODF(EVT RetVT);

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,6 +1063,8 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
10631063
def int_roundeven : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
10641064
def int_sincos : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
10651065
[llvm_anyfloat_ty]>;
1066+
def int_sincospi : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
1067+
[llvm_anyfloat_ty]>;
10661068
def int_modf : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
10671069
[llvm_anyfloat_ty]>;
10681070

‎llvm/include/llvm/IR/RuntimeLibcalls.def

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,11 @@ HANDLE_LIBCALL(FREXP_F64, "frexp")
354354
HANDLE_LIBCALL(FREXP_F80, "frexpl")
355355
HANDLE_LIBCALL(FREXP_F128, "frexpl")
356356
HANDLE_LIBCALL(FREXP_PPCF128, "frexpl")
357+
HANDLE_LIBCALL(SINCOSPI_F32, "sincospif")
358+
HANDLE_LIBCALL(SINCOSPI_F64, "sincospi")
359+
HANDLE_LIBCALL(SINCOSPI_F80, "sincospil")
360+
HANDLE_LIBCALL(SINCOSPI_F128, "sincospil")
361+
HANDLE_LIBCALL(SINCOSPI_PPCF128, "sincospil")
357362
HANDLE_LIBCALL(MODF_F32, "modff")
358363
HANDLE_LIBCALL(MODF_F64, "modf")
359364
HANDLE_LIBCALL(MODF_F80, "modfl")

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4520,11 +4520,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
45204520
ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80,
45214521
RTLIB::TANH_F128, RTLIB::TANH_PPCF128, Results);
45224522
break;
4523-
case ISD::FSINCOS: {
4524-
RTLIB::Libcall LC = RTLIB::getFSINCOS(Node->getValueType(0));
4523+
case ISD::FSINCOS:
4524+
case ISD::FSINCOSPI: {
4525+
EVT VT = Node->getValueType(0);
4526+
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4527+
? RTLIB::getFSINCOS(VT)
4528+
: RTLIB::getSINCOSPI(VT);
45254529
bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
45264530
if (!Expanded)
4527-
llvm_unreachable("Expected scalar FSINCOS to expand to libcall!");
4531+
llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
45284532
break;
45294533
}
45304534
case ISD::FLOG:
@@ -5507,7 +5511,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
55075511
break;
55085512
}
55095513
case ISD::FMODF:
5510-
case ISD::FSINCOS: {
5514+
case ISD::FSINCOS:
5515+
case ISD::FSINCOSPI: {
55115516
Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
55125517
Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1,
55135518
Node->getFlags());

‎llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2768,6 +2768,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
27682768

27692769
case ISD::FMODF:
27702770
case ISD::FSINCOS:
2771+
case ISD::FSINCOSPI:
27712772
R = PromoteFloatRes_UnaryWithTwoFPResults(N);
27722773
break;
27732774
case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
@@ -3230,6 +3231,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
32303231

32313232
case ISD::FMODF:
32323233
case ISD::FSINCOS:
3234+
case ISD::FSINCOSPI:
32333235
R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
32343236
break;
32353237

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
456456
case ISD::FFREXP:
457457
case ISD::FMODF:
458458
case ISD::FSINCOS:
459+
case ISD::FSINCOSPI:
459460
case ISD::SADDSAT:
460461
case ISD::UADDSAT:
461462
case ISD::SSUBSAT:
@@ -1217,9 +1218,12 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
12171218
return;
12181219

12191220
break;
1220-
case ISD::FSINCOS: {
1221-
RTLIB::Libcall LC =
1222-
RTLIB::getFSINCOS(Node->getValueType(0).getVectorElementType());
1221+
case ISD::FSINCOS:
1222+
case ISD::FSINCOSPI: {
1223+
EVT VT = Node->getValueType(0).getVectorElementType();
1224+
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
1225+
? RTLIB::getFSINCOS(VT)
1226+
: RTLIB::getSINCOSPI(VT);
12231227
if (DAG.expandMultipleResultFPLibCall(LC, Node, Results))
12241228
return;
12251229
break;

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
136136
case ISD::FMODF:
137137
case ISD::FFREXP:
138138
case ISD::FSINCOS:
139+
case ISD::FSINCOSPI:
139140
R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
140141
break;
141142
case ISD::ADD:
@@ -1265,6 +1266,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
12651266
case ISD::FMODF:
12661267
case ISD::FFREXP:
12671268
case ISD::FSINCOS:
1269+
case ISD::FSINCOSPI:
12681270
SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
12691271
break;
12701272

@@ -4787,7 +4789,8 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
47874789
break;
47884790
case ISD::FMODF:
47894791
case ISD::FFREXP:
4790-
case ISD::FSINCOS: {
4792+
case ISD::FSINCOS:
4793+
case ISD::FSINCOSPI: {
47914794
if (!unrollExpandedOp())
47924795
Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
47934796
break;

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6979,6 +6979,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
69796979
return;
69806980
case Intrinsic::modf:
69816981
case Intrinsic::sincos:
6982+
case Intrinsic::sincospi:
69826983
case Intrinsic::frexp: {
69836984
unsigned Opcode;
69846985
switch (Intrinsic) {
@@ -6987,6 +6988,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
69876988
case Intrinsic::sincos:
69886989
Opcode = ISD::FSINCOS;
69896990
break;
6991+
case Intrinsic::sincospi:
6992+
Opcode = ISD::FSINCOSPI;
6993+
break;
69906994
case Intrinsic::modf:
69916995
Opcode = ISD::FMODF;
69926996
break;

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
219219
case ISD::FCOS: return "fcos";
220220
case ISD::STRICT_FCOS: return "strict_fcos";
221221
case ISD::FSINCOS: return "fsincos";
222+
case ISD::FSINCOSPI: return "fsincospi";
222223
case ISD::FMODF: return "fmodf";
223224
case ISD::FTAN: return "ftan";
224225
case ISD::STRICT_FTAN: return "strict_ftan";

‎llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,11 @@ RTLIB::Libcall RTLIB::getFSINCOS(EVT RetVT) {
407407
SINCOS_PPCF128);
408408
}
409409

410+
RTLIB::Libcall RTLIB::getSINCOSPI(EVT RetVT) {
411+
return getFPLibCall(RetVT, SINCOSPI_F32, SINCOSPI_F64, SINCOSPI_F80,
412+
SINCOSPI_F128, SINCOSPI_PPCF128);
413+
}
414+
410415
RTLIB::Libcall RTLIB::getMODF(EVT RetVT) {
411416
return getFPLibCall(RetVT, MODF_F32, MODF_F64, MODF_F80, MODF_F128,
412417
MODF_PPCF128);
@@ -781,7 +786,7 @@ void TargetLoweringBase::initActions() {
781786

782787
// These library functions default to expand.
783788
setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP,
784-
ISD::FSINCOS, ISD::FMODF},
789+
ISD::FSINCOS, ISD::FSINCOSPI, ISD::FMODF},
785790
VT, Expand);
786791

787792
// These operations default to expand for vector types.

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -735,19 +735,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
735735
setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
736736
}
737737

738-
for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
739-
ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
740-
ISD::FMODF, ISD::FACOS, ISD::FASIN,
741-
ISD::FATAN, ISD::FATAN2, ISD::FCOSH,
742-
ISD::FSINH, ISD::FTANH, ISD::FTAN,
743-
ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
744-
ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
745-
ISD::STRICT_FREM, ISD::STRICT_FPOW, ISD::STRICT_FPOWI,
746-
ISD::STRICT_FCOS, ISD::STRICT_FSIN, ISD::STRICT_FACOS,
747-
ISD::STRICT_FASIN, ISD::STRICT_FATAN, ISD::STRICT_FATAN2,
748-
ISD::STRICT_FCOSH, ISD::STRICT_FSINH, ISD::STRICT_FTANH,
749-
ISD::STRICT_FEXP, ISD::STRICT_FEXP2, ISD::STRICT_FLOG,
750-
ISD::STRICT_FLOG2, ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
738+
for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
739+
ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
740+
ISD::FSINCOSPI, ISD::FMODF, ISD::FACOS,
741+
ISD::FASIN, ISD::FATAN, ISD::FATAN2,
742+
ISD::FCOSH, ISD::FSINH, ISD::FTANH,
743+
ISD::FTAN, ISD::FEXP, ISD::FEXP2,
744+
ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
745+
ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
746+
ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
747+
ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
748+
ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
749+
ISD::STRICT_FTANH, ISD::STRICT_FEXP, ISD::STRICT_FEXP2,
750+
ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
751+
ISD::STRICT_FTAN}) {
751752
setOperationAction(Op, MVT::f16, Promote);
752753
setOperationAction(Op, MVT::v4f16, Expand);
753754
setOperationAction(Op, MVT::v8f16, Expand);
@@ -1208,7 +1209,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
12081209
ISD::FSIN, ISD::FCOS, ISD::FTAN,
12091210
ISD::FASIN, ISD::FACOS, ISD::FATAN,
12101211
ISD::FSINH, ISD::FCOSH, ISD::FTANH,
1211-
ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1212+
ISD::FPOW, ISD::FLOG, ISD::FLOG2,
12121213
ISD::FLOG10, ISD::FEXP, ISD::FEXP2,
12131214
ISD::FEXP10, ISD::FRINT, ISD::FROUND,
12141215
ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM,
@@ -1217,7 +1218,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
12171218
ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
12181219
ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FCEIL,
12191220
ISD::STRICT_FFLOOR, ISD::STRICT_FSQRT, ISD::STRICT_FRINT,
1220-
ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC,
1221+
ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC,
12211222
ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
12221223
ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM})
12231224
setOperationAction(Op, MVT::v1f64, Expand);

0 commit comments

Comments
 (0)