Skip to content

Commit 21b534d

Browse files
MacDueMacDue
committed
[LV] Teach the loop vectorizer llvm.sincos is trivially vectorizable
1 parent 421d3ec commit 21b534d

File tree

6 files changed

+193
-19
lines changed

6 files changed

+193
-19
lines changed

‎llvm/lib/Analysis/VectorUtils.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
7272
case Intrinsic::atan2:
7373
case Intrinsic::sin:
7474
case Intrinsic::cos:
75+
case Intrinsic::sincos:
7576
case Intrinsic::tan:
7677
case Intrinsic::sinh:
7778
case Intrinsic::cosh:
@@ -185,6 +186,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
185186
case Intrinsic::ucmp:
186187
case Intrinsic::scmp:
187188
return OpdIdx == -1 || OpdIdx == 0;
189+
case Intrinsic::sincos:
188190
case Intrinsic::is_fpclass:
189191
case Intrinsic::vp_is_fpclass:
190192
return OpdIdx == 0;

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2926,7 +2926,8 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
29262926
[&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
29272927

29282928
IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2929-
dyn_cast<IntrinsicInst>(CI));
2929+
dyn_cast<IntrinsicInst>(CI),
2930+
InstructionCost::getInvalid(), TLI);
29302931
return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
29312932
}
29322933

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1179,7 +1179,8 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
11791179
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
11801180
IntrinsicCostAttributes CostAttrs(
11811181
VectorIntrinsicID, RetTy, Arguments, ParamTys, FMF,
1182-
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()));
1182+
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()),
1183+
InstructionCost::getInvalid(), &Ctx.TLI);
11831184
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
11841185
}
11851186

‎llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|sincos|extractvalue|store)" --version 5
2+
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve < %s -S -o - -debug-only=loop-vectorize 2>%t.1 | FileCheck %s --check-prefix=CHECK
3+
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve -vector-library=ArmPL < %s -S -o - -debug-only=loop-vectorize 2>%t.2 | FileCheck %s --check-prefix=CHECK-ARMPL
4+
; RUN: cat %t.1 | FileCheck --check-prefix=CHECK-COST %s
5+
; RUN: cat %t.2 | FileCheck --check-prefix=CHECK-COST-ARMPL %s
6+
; REQUIRES: asserts
7+
8+
; CHECK-COST-LABEL: sincos_f32
9+
; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
10+
; CHECK-COST: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
11+
; CHECK-COST: Cost of 58 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
12+
; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
13+
; CHECK-COST: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
14+
; CHECK-COST: Cost of Invalid for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
15+
16+
; CHECK-COST-ARMPL-LABEL: sincos_f32
17+
; CHECK-COST-ARMPL: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
18+
; CHECK-COST-ARMPL: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
19+
; CHECK-COST-ARMPL: Cost of 12 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
20+
; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
21+
; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
22+
; CHECK-COST-ARMPL: Cost of 13 for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
23+
24+
define void @sincos_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
25+
; CHECK-LABEL: define void @sincos_f32(
26+
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
27+
; CHECK: [[ENTRY:.*:]]
28+
; CHECK: [[VECTOR_PH:.*:]]
29+
; CHECK: [[VECTOR_BODY:.*:]]
30+
; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
31+
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
32+
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
33+
; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
34+
; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
35+
; CHECK: [[MIDDLE_BLOCK:.*:]]
36+
; CHECK: [[SCALAR_PH:.*:]]
37+
; CHECK: [[FOR_BODY:.*:]]
38+
; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
39+
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
40+
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
41+
; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
42+
; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
43+
; CHECK: [[EXIT:.*:]]
44+
;
45+
; CHECK-ARMPL-LABEL: define void @sincos_f32(
46+
; CHECK-ARMPL-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
47+
; CHECK-ARMPL: [[ENTRY:.*:]]
48+
; CHECK-ARMPL: [[VECTOR_PH:.*:]]
49+
; CHECK-ARMPL: [[VECTOR_BODY:.*:]]
50+
; CHECK-ARMPL: [[TMP12:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincos.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD:%.*]])
51+
; CHECK-ARMPL: [[TMP13:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincos.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD1:%.*]])
52+
; CHECK-ARMPL: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP12]], 0
53+
; CHECK-ARMPL: [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP13]], 0
54+
; CHECK-ARMPL: [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP12]], 1
55+
; CHECK-ARMPL: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP13]], 1
56+
; CHECK-ARMPL: store <vscale x 4 x float> [[TMP14]], ptr [[TMP19:%.*]], align 4
57+
; CHECK-ARMPL: store <vscale x 4 x float> [[TMP15]], ptr [[TMP22:%.*]], align 4
58+
; CHECK-ARMPL: store <vscale x 4 x float> [[TMP16]], ptr [[TMP24:%.*]], align 4
59+
; CHECK-ARMPL: store <vscale x 4 x float> [[TMP17]], ptr [[TMP27:%.*]], align 4
60+
; CHECK-ARMPL: [[MIDDLE_BLOCK:.*:]]
61+
; CHECK-ARMPL: [[SCALAR_PH:.*:]]
62+
; CHECK-ARMPL: [[FOR_BODY:.*:]]
63+
; CHECK-ARMPL: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
64+
; CHECK-ARMPL: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
65+
; CHECK-ARMPL: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
66+
; CHECK-ARMPL: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
67+
; CHECK-ARMPL: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
68+
; CHECK-ARMPL: [[EXIT:.*:]]
69+
;
70+
entry:
71+
br label %for.body
72+
73+
for.body:
74+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
75+
%arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
76+
%in_val = load float, ptr %arrayidx, align 4
77+
%call = tail call { float, float } @llvm.sincos.f32(float %in_val)
78+
%extract_a = extractvalue { float, float } %call, 0
79+
%extract_b = extractvalue { float, float } %call, 1
80+
%arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
81+
store float %extract_a, ptr %arrayidx2, align 4
82+
%arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
83+
store float %extract_b, ptr %arrayidx4, align 4
84+
%iv.next = add nuw nsw i64 %iv, 1
85+
%exitcond.not = icmp eq i64 %iv.next, 1024
86+
br i1 %exitcond.not, label %exit, label %for.body
87+
88+
exit:
89+
ret void
90+
}
91+
92+
; CHECK-COST-LABEL: sincos_f64
93+
; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
94+
; CHECK-COST: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
95+
; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
96+
; CHECK-COST: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
97+
98+
; CHECK-COST-ARMPL-LABEL: sincos_f64
99+
; CHECK-COST-ARMPL: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
100+
; CHECK-COST-ARMPL: Cost of 12 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
101+
; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
102+
; CHECK-COST-ARMPL: Cost of 13 for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
103+
104+
define void @sincos_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
105+
; CHECK-LABEL: define void @sincos_f64(
106+
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
107+
; CHECK: [[ENTRY:.*:]]
108+
; CHECK: [[VECTOR_PH:.*:]]
109+
; CHECK: [[VECTOR_BODY:.*:]]
110+
; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
111+
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
112+
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
113+
; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
114+
; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
115+
; CHECK: [[MIDDLE_BLOCK:.*:]]
116+
; CHECK: [[SCALAR_PH:.*:]]
117+
; CHECK: [[FOR_BODY:.*:]]
118+
; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
119+
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
120+
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
121+
; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
122+
; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
123+
; CHECK: [[EXIT:.*:]]
124+
;
125+
; CHECK-ARMPL-LABEL: define void @sincos_f64(
126+
; CHECK-ARMPL-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
127+
; CHECK-ARMPL: [[ENTRY:.*:]]
128+
; CHECK-ARMPL: [[VECTOR_PH:.*:]]
129+
; CHECK-ARMPL: [[VECTOR_BODY:.*:]]
130+
; CHECK-ARMPL: [[TMP12:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.sincos.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD:%.*]])
131+
; CHECK-ARMPL: [[TMP13:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.sincos.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD1:%.*]])
132+
; CHECK-ARMPL: [[TMP14:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP12]], 0
133+
; CHECK-ARMPL: [[TMP15:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP13]], 0
134+
; CHECK-ARMPL: [[TMP16:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP12]], 1
135+
; CHECK-ARMPL: [[TMP17:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP13]], 1
136+
; CHECK-ARMPL: store <vscale x 2 x double> [[TMP14]], ptr [[TMP19:%.*]], align 8
137+
; CHECK-ARMPL: store <vscale x 2 x double> [[TMP15]], ptr [[TMP22:%.*]], align 8
138+
; CHECK-ARMPL: store <vscale x 2 x double> [[TMP16]], ptr [[TMP24:%.*]], align 8
139+
; CHECK-ARMPL: store <vscale x 2 x double> [[TMP17]], ptr [[TMP27:%.*]], align 8
140+
; CHECK-ARMPL: [[MIDDLE_BLOCK:.*:]]
141+
; CHECK-ARMPL: [[SCALAR_PH:.*:]]
142+
; CHECK-ARMPL: [[FOR_BODY:.*:]]
143+
; CHECK-ARMPL: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
144+
; CHECK-ARMPL: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
145+
; CHECK-ARMPL: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
146+
; CHECK-ARMPL: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
147+
; CHECK-ARMPL: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
148+
; CHECK-ARMPL: [[EXIT:.*:]]
149+
;
150+
entry:
151+
br label %for.body
152+
153+
for.body:
154+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
155+
%arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
156+
%in_val = load double, ptr %arrayidx, align 8
157+
%call = tail call { double, double } @llvm.sincos.f64(double %in_val)
158+
%extract_a = extractvalue { double, double } %call, 0
159+
%extract_b = extractvalue { double, double } %call, 1
160+
%arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
161+
store double %extract_a, ptr %arrayidx2, align 8
162+
%arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
163+
store double %extract_b, ptr %arrayidx4, align 8
164+
%iv.next = add nuw nsw i64 %iv, 1
165+
%exitcond.not = icmp eq i64 %iv.next, 1024
166+
br i1 %exitcond.not, label %exit, label %for.body
167+
168+
exit:
169+
ret void
170+
}

‎llvm/test/Transforms/Scalarizer/deinterleave2.ll

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt %s -passes="function(scalarizer)" -S | FileCheck %s
3+
4+
; Test to make sure that struct return intrinsics that are not `isTriviallyScalarizable` do not get scalarized.
5+
6+
define <2 x float> @test_(<4 x float> %Val) {
7+
; CHECK-LABEL: define <2 x float> @test_(
8+
; CHECK-SAME: <4 x float> [[VAL:%.*]]) {
9+
; CHECK-NEXT: [[R:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[VAL]])
10+
; CHECK-NEXT: [[EL:%.*]] = extractvalue { <2 x float>, <2 x float> } [[R]], 0
11+
; CHECK-NEXT: ret <2 x float> [[EL]]
12+
;
13+
%r = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %Val)
14+
%el = extractvalue { <2 x float>, <2 x float> } %r, 0
15+
ret <2 x float> %el
16+
}
17+

‎llvm/test/Transforms/Scalarizer/sincos.ll

Lines changed: 0 additions & 17 deletions
This file was deleted.

0 commit comments

Comments
 (0)