Skip to content

Commit bc3620d

Browse files
arsenmarsenm
authored
AMDGPU: Move libcall simplify into PeepholeEP (#88853)
We were running this immediately on the incoming IR, which is still littered with temporary allocas obscuring trivial values. This needs to run after initial SROA to handle sincos insertion.
1 parent dd84d23 commit bc3620d

File tree

4 files changed

+96
-17
lines changed

4 files changed

+96
-17
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -655,9 +655,6 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(
655655
PB.registerPipelineStartEPCallback(
656656
[](ModulePassManager &PM, OptimizationLevel Level) {
657657
FunctionPassManager FPM;
658-
FPM.addPass(AMDGPUUseNativeCallsPass());
659-
if (EnableLibCallSimplify && Level != OptimizationLevel::O0)
660-
FPM.addPass(AMDGPUSimplifyLibCallsPass());
661658
PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
662659
if (EnableHipStdPar)
663660
PM.addPass(HipStdParAcceleratorCodeSelectionPass());
@@ -681,6 +678,16 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(
681678
PM.addPass(AMDGPUAlwaysInlinePass());
682679
});
683680

681+
PB.registerPeepholeEPCallback(
682+
[](FunctionPassManager &FPM, OptimizationLevel Level) {
683+
if (Level == OptimizationLevel::O0)
684+
return;
685+
686+
FPM.addPass(AMDGPUUseNativeCallsPass());
687+
if (EnableLibCallSimplify)
688+
FPM.addPass(AMDGPUSimplifyLibCallsPass());
689+
});
690+
684691
PB.registerCGSCCOptimizerLateEPCallback(
685692
[this](CGSCCPassManager &PM, OptimizationLevel Level) {
686693
if (Level == OptimizationLevel::O0)

‎llvm/test/CodeGen/AMDGPU/amdgpu-libcall-sincos-pass-ordering.ll

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
2+
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 -amdgpu-prelink %s | FileCheck %s
3+
4+
; Make sure that sin+cos -> sincos simplification happens after
5+
; initial IR simplifications, otherwise we can't identify the common
6+
; argument value.
7+
8+
@.str = private unnamed_addr addrspace(4) constant [21 x i8] c"x: %f, y: %f, z: %f\0A\00", align 1
9+
10+
; Should have call to sincos declarations, not calls to the asm pseudo-libcalls
11+
define protected amdgpu_kernel void @swdev456865(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, float noundef %x) #0 {
12+
; CHECK-LABEL: define protected amdgpu_kernel void @swdev456865(
13+
; CHECK-SAME: ptr addrspace(1) nocapture writeonly [[OUT0:%.*]], ptr addrspace(1) nocapture writeonly [[OUT1:%.*]], ptr addrspace(1) nocapture writeonly [[OUT2:%.*]], float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
14+
; CHECK-NEXT: entry:
15+
; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
16+
; CHECK-NEXT: [[I_I:%.*]] = call float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]]) #[[ATTR1:[0-9]+]]
17+
; CHECK-NEXT: [[I_I2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4
18+
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[I_I]], [[I_I2]]
19+
; CHECK-NEXT: [[CONV:%.*]] = fpext float [[X]] to double
20+
; CHECK-NEXT: [[CONV5:%.*]] = fpext float [[ADD]] to double
21+
; CHECK-NEXT: store double [[CONV]], ptr addrspace(1) [[OUT0]], align 8
22+
; CHECK-NEXT: store double [[CONV5]], ptr addrspace(1) [[OUT1]], align 8
23+
; CHECK-NEXT: store double [[CONV5]], ptr addrspace(1) [[OUT2]], align 8
24+
; CHECK-NEXT: ret void
25+
;
26+
entry:
27+
%x.addr = alloca float, align 4, addrspace(5)
28+
%y = alloca float, align 4, addrspace(5)
29+
%z = alloca float, align 4, addrspace(5)
30+
store float %x, ptr addrspace(5) %x.addr, align 4
31+
call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %y)
32+
%i = load float, ptr addrspace(5) %x.addr, align 4
33+
%call = call float @_Z3sinf(float noundef %i) #3
34+
%i1 = load float, ptr addrspace(5) %x.addr, align 4
35+
%call1 = call float @_Z3cosf(float noundef %i1) #3
36+
%add = fadd float %call, %call1
37+
store float %add, ptr addrspace(5) %y, align 4
38+
call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %z)
39+
%i2 = load float, ptr addrspace(5) %x.addr, align 4
40+
%call2 = call float @_Z3cosf(float noundef %i2) #3
41+
%i3 = load float, ptr addrspace(5) %x.addr, align 4
42+
%call3 = call float @_Z3sinf(float noundef %i3) #3
43+
%add4 = fadd float %call2, %call3
44+
store float %add4, ptr addrspace(5) %z, align 4
45+
%i4 = load float, ptr addrspace(5) %x.addr, align 4
46+
%conv = fpext float %i4 to double
47+
%i5 = load float, ptr addrspace(5) %y, align 4
48+
%conv5 = fpext float %i5 to double
49+
%i6 = load float, ptr addrspace(5) %z, align 4
50+
%conv6 = fpext float %i6 to double
51+
store double %conv, ptr addrspace(1) %out0, align 8
52+
store double %conv5, ptr addrspace(1) %out1, align 8
53+
store double %conv6, ptr addrspace(1) %out2, align 8
54+
call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %z)
55+
call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %y)
56+
ret void
57+
}
58+
59+
declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture) #1
60+
declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture) #1
61+
62+
define internal float @_Z3cosf(float noundef %arg) #2 {
63+
bb:
64+
%i = tail call float asm "pseudo-libcall-cos %0, %1", "=v,v"(float noundef %arg) #2
65+
ret float %i
66+
}
67+
68+
define internal float @_Z3sinf(float noundef %arg) #2 {
69+
bb:
70+
%i = tail call float asm "pseudo-libcall-sin %0, %1", "=v,v"(float noundef %arg) #2
71+
ret float %i
72+
}
73+
74+
attributes #0 = { norecurse nounwind }
75+
attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
76+
attributes #2 = { mustprogress nofree norecurse nounwind willreturn memory(none) }
77+
attributes #3 = { nounwind willreturn memory(none) }

‎llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-sincos.ll

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -884,10 +884,9 @@ entry:
884884

885885
define float @sincos_f32_unused_result_cos(float %x) {
886886
; CHECK-LABEL: define float @sincos_f32_unused_result_cos
887-
; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR4]] {
887+
; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
888888
; CHECK-NEXT: entry:
889-
; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
890-
; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
889+
; CHECK-NEXT: [[TMP0:%.*]] = tail call contract float @_Z3sinf(float [[X]])
891890
; CHECK-NEXT: ret float [[TMP0]]
892891
;
893892
entry:
@@ -900,11 +899,9 @@ entry:
900899

901900
define float @sincos_f32_unused_result_sin(float %x) {
902901
; CHECK-LABEL: define float @sincos_f32_unused_result_sin
903-
; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR4]] {
902+
; CHECK-SAME: (float [[X:%.*]]) local_unnamed_addr #[[ATTR5]] {
904903
; CHECK-NEXT: entry:
905-
; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
906-
; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
907-
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4
904+
; CHECK-NEXT: [[TMP1:%.*]] = tail call contract float @_Z3cosf(float [[X]])
908905
; CHECK-NEXT: ret float [[TMP1]]
909906
;
910907
entry:
@@ -917,13 +914,11 @@ entry:
917914

918915
define void @sincos_f32_repeated_uses(float %x, ptr addrspace(1) %sin_out, ptr addrspace(1) %cos_out) {
919916
; CHECK-LABEL: define void @sincos_f32_repeated_uses
920-
; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) [[SIN_OUT:%.*]], ptr addrspace(1) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
917+
; CHECK-SAME: (float [[X:%.*]], ptr addrspace(1) [[SIN_OUT:%.*]], ptr addrspace(1) [[COS_OUT:%.*]]) local_unnamed_addr #[[ATTR6:[0-9]+]] {
921918
; CHECK-NEXT: entry:
922-
; CHECK-NEXT: [[__SINCOS_:%.*]] = alloca float, align 4, addrspace(5)
923919
; CHECK-NEXT: [[__SINCOS_3:%.*]] = alloca float, align 4, addrspace(5)
924920
; CHECK-NEXT: [[TMP0:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_3]])
925-
; CHECK-NEXT: [[TMP1:%.*]] = call contract float @_Z6sincosfPU3AS5f(float [[X]], ptr addrspace(5) [[__SINCOS_]])
926-
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_]], align 4
921+
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(5) [[__SINCOS_3]], align 4
927922
; CHECK-NEXT: store volatile float [[TMP0]], ptr addrspace(1) [[SIN_OUT]], align 4
928923
; CHECK-NEXT: store volatile float [[TMP0]], ptr addrspace(1) [[SIN_OUT]], align 4
929924
; CHECK-NEXT: store volatile float [[TMP2]], ptr addrspace(1) [[COS_OUT]], align 4

‎llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ entry:
278278

279279
; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_half
280280
; GCN-POSTLINK: call fast float @_Z3powff(float %tmp, float 5.000000e-01)
281-
; GCN-PRELINK: %__pow2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
281+
; GCN-PRELINK: %__pow2sqrt = tail call fast float @llvm.sqrt.f32(float %tmp)
282282
define amdgpu_kernel void @test_pow_half(ptr addrspace(1) nocapture %a) {
283283
entry:
284284
%arrayidx = getelementptr inbounds float, ptr addrspace(1) %a, i64 1
@@ -476,7 +476,7 @@ declare float @_Z5rootnfi(float, i32)
476476

477477
; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_2
478478
; GCN-POSTLINK: call fast float @_Z5rootnfi(float %tmp, i32 2)
479-
; GCN-PRELINK: %__rootn2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
479+
; GCN-PRELINK: %__rootn2sqrt = tail call fast float @llvm.sqrt.f32(float %tmp)
480480
define amdgpu_kernel void @test_rootn_2(ptr addrspace(1) nocapture %a) {
481481
entry:
482482
%tmp = load float, ptr addrspace(1) %a, align 4
@@ -838,5 +838,5 @@ entry:
838838
; GCN-PRELINK: declare float @_Z4cbrtf(float) local_unnamed_addr #[[$NOUNWIND_READONLY:[0-9]+]]
839839

840840
; GCN-PRELINK-DAG: attributes #[[$NOUNWIND]] = { nounwind }
841-
; GCN-PRELINK-DAG: attributes #[[$NOUNWIND_READONLY]] = { nofree nounwind memory(read) }
841+
; GCN-PRELINK-DAG: attributes #[[$NOUNWIND_READONLY]] = { nounwind memory(read) }
842842
attributes #0 = { nounwind }

0 commit comments

Comments
 (0)