Skip to content

Add PProf to admin pages and to gitea manager #22742

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 34 commits into from
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
5093d7d
Add PProf to admin pages
zeripath Feb 3, 2023
9cfcbb4
Update pprof.tmpl
zeripath Feb 4, 2023
f22aa21
Update locale_en-US.ini
zeripath Feb 4, 2023
b9989b1
Add FGProf handler
zeripath Feb 4, 2023
6451d01
wire in PProfFGProfile
zeripath Feb 4, 2023
351149b
Wire in FGProf
zeripath Feb 4, 2023
7941fbb
Update locale_en-US.ini
zeripath Feb 4, 2023
0ef07f6
Update pprof.tmpl
zeripath Feb 4, 2023
8d0d93e
Update pprof.go
zeripath Feb 4, 2023
768a960
Update pprof.go
zeripath Feb 4, 2023
82169ca
Update web.go
zeripath Feb 4, 2023
ba894a1
Update pprof.go
zeripath Feb 4, 2023
75d96e8
Update pprof.tmpl
zeripath Feb 4, 2023
381f12a
Update pprof.tmpl
zeripath Feb 4, 2023
2fc5d72
Update locale_en-US.ini
zeripath Feb 4, 2023
79756cd
placate lint
zeripath Feb 4, 2023
6247e72
Add pprof endpoints to manager too
zeripath Feb 4, 2023
d9eac18
add documentation
zeripath Feb 4, 2023
7decd93
Add output option
zeripath Feb 4, 2023
4d916b4
fix missing name
zeripath Feb 4, 2023
b9942cb
add trace and fix format
zeripath Feb 4, 2023
7c4be9c
fix format on stacktraces
zeripath Feb 4, 2023
6c2017d
fix trace
zeripath Feb 4, 2023
d37978e
Apply suggestions from code review
zeripath Feb 5, 2023
950c474
Merge remote-tracking branch 'origin/main' into add-pprof-to-admin-pages
zeripath Feb 5, 2023
2dbab58
as per delvh
zeripath Feb 5, 2023
2b523f1
include indent in WriteProcess
zeripath Feb 5, 2023
5925f47
as per delvh
zeripath Feb 5, 2023
91bc97a
remove space
zeripath Feb 5, 2023
61755a7
as per delvh
zeripath Feb 5, 2023
1400ab2
as per delvh
zeripath Feb 5, 2023
edebd86
add some comments
zeripath Feb 5, 2023
1ffe696
Merge remote-tracking branch 'origin/main' into add-pprof-to-admin-pages
zeripath Feb 19, 2023
92ac295
move trace help in to the ui form
zeripath Feb 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions modules/process/stacktraces_processlist.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package process

import (
"bytes"
"fmt"
"io"
)

// WriteProcesses writes out processes to a provided writer
func WriteProcesses(out io.Writer, processes []*Process, processCount int, goroutineCount int64, indent string, flat bool) error {
if goroutineCount > 0 {
if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil {
return err
}
}
if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil {
return err
}
if len(processes) > 0 {
if err := WriteProcess(out, processes[0], " ", flat); err != nil {
return err
}
}
if len(processes) > 1 {
for _, process := range processes[1:] {
if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil {
return err
}
if err := WriteProcess(out, process, " ", flat); err != nil {
return err
}
}
}
return nil
}

// WriteProcess writes out a process to a provided writer
func WriteProcess(out io.Writer, process *Process, indent string, flat bool) error {
sb := &bytes.Buffer{}
if flat {
if process.ParentPID != "" {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type)
} else {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type)
}
} else {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type)
}
indent += "| "

_, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description)
_, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start)

if len(process.Stacks) > 0 {
_, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent)
for _, stack := range process.Stacks {
indent := indent + " "
_, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description)
if stack.Count > 1 {
_, _ = fmt.Fprintf(sb, "* %d", stack.Count)
}
_, _ = fmt.Fprintf(sb, "\n")
indent += "| "
if len(stack.Labels) > 0 {
_, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value)

if len(stack.Labels) > 1 {
for _, label := range stack.Labels[1:] {
_, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value)
}
}
_, _ = fmt.Fprintf(sb, "\n")
}
_, _ = fmt.Fprintf(sb, "%sStack:\n", indent)
indent += " "
for _, entry := range stack.Entry {
_, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function)
_, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line)
}
}
}
if _, err := out.Write(sb.Bytes()); err != nil {
return err
}
sb.Reset()
if len(process.Children) > 0 {
if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil {
return err
}
for _, child := range process.Children {
if err := WriteProcess(out, child, indent+" ", flat); err != nil {
return err
}
}
}
return nil
}
15 changes: 15 additions & 0 deletions options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2947,6 +2947,7 @@ monitor.previous = Previous Time
monitor.execute_times = Executions
monitor.process = Running Processes
monitor.stacktrace = Stacktraces
monitor.stacktrace.download_stacktrace = Download Stacktrace
monitor.goroutines = %d Goroutines
monitor.desc = Description
monitor.start = Start Time
Expand All @@ -2956,6 +2957,20 @@ monitor.process.cancel = Cancel process
monitor.process.cancel_desc = Cancelling a process may cause data loss
monitor.process.cancel_notices = Cancel: <strong>%s</strong>?
monitor.process.children = Children

monitor.pprof = PProf Profiles
monitor.pprof.download = Download

monitor.pprof.cpuprofile = CPU Profile
monitor.pprof.cpuprofile.duration = Duration
monitor.pprof.cpuprofile.duration_placeholder = e.g. 30s
monitor.pprof.cpuprofile.duration_invalid = Invalid duration - duration must be a golang duration string

monitor.pprof.named_profiles = Named Profiles
monitor.pprof.named_profiles.name = Name
monitor.pprof.named_profiles.debug = Debug level
monitor.pprof.named_profiles.debug_placeholder = e.g. 0

monitor.queues = Queues
monitor.queue = Queue: %s
monitor.queue.name = Name
Expand Down
92 changes: 1 addition & 91 deletions routers/private/manager_process.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
package private

import (
"bytes"
"fmt"
"io"
"net/http"
"runtime"
"time"
Expand Down Expand Up @@ -60,7 +58,7 @@ func Processes(ctx *context.PrivateContext) {
ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8")
ctx.Resp.WriteHeader(http.StatusOK)

if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil {
if err := process_module.WriteProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil {
log.Error("Unable to write out process stacktrace: %v", err)
if !ctx.Written() {
ctx.JSON(http.StatusInternalServerError, private.Response{
Expand All @@ -70,91 +68,3 @@ func Processes(ctx *context.PrivateContext) {
return
}
}

func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error {
if goroutineCount > 0 {
if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil {
return err
}
}
if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil {
return err
}
if len(processes) > 0 {
if err := writeProcess(out, processes[0], " ", flat); err != nil {
return err
}
}
if len(processes) > 1 {
for _, process := range processes[1:] {
if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil {
return err
}
if err := writeProcess(out, process, " ", flat); err != nil {
return err
}
}
}
return nil
}

func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error {
sb := &bytes.Buffer{}
if flat {
if process.ParentPID != "" {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type)
} else {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type)
}
} else {
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type)
}
indent += "| "

_, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description)
_, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start)

if len(process.Stacks) > 0 {
_, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent)
for _, stack := range process.Stacks {
indent := indent + " "
_, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description)
if stack.Count > 1 {
_, _ = fmt.Fprintf(sb, "* %d", stack.Count)
}
_, _ = fmt.Fprintf(sb, "\n")
indent += "| "
if len(stack.Labels) > 0 {
_, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value)

if len(stack.Labels) > 1 {
for _, label := range stack.Labels[1:] {
_, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value)
}
}
_, _ = fmt.Fprintf(sb, "\n")
}
_, _ = fmt.Fprintf(sb, "%sStack:\n", indent)
indent += " "
for _, entry := range stack.Entry {
_, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function)
_, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line)
}
}
}
if _, err := out.Write(sb.Bytes()); err != nil {
return err
}
sb.Reset()
if len(process.Children) > 0 {
if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil {
return err
}
for _, child := range process.Children {
if err := writeProcess(out, child, indent+" ", flat); err != nil {
return err
}
}
}
return nil
}
16 changes: 15 additions & 1 deletion routers/web/admin/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"fmt"
"net/http"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"time"

activities_model "code.gitea.io/gitea/models/activities"
Expand Down Expand Up @@ -159,10 +161,14 @@ func Monitor(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.monitor")
ctx.Data["PageIsAdmin"] = true
ctx.Data["PageIsAdminMonitor"] = true
ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true)
processes, processCount := process.GetManager().Processes(false, true)
ctx.Data["Entries"] = cron.ListTasks()
ctx.Data["Queues"] = queue.GetManager().ManagedQueues()

ctx.Data["Processes"], ctx.Data["ProcessCount"] = processes, processCount

ctx.Data["Profiles"] = pprof.Profiles()

ctx.HTML(http.StatusOK, tplMonitor)
}

Expand All @@ -182,6 +188,14 @@ func GoroutineStacktrace(ctx *context.Context) {

ctx.Data["GoroutineCount"] = goroutineCount
ctx.Data["ProcessCount"] = processCount
sb := new(strings.Builder)

if err := process.WriteProcesses(sb, processStacks, processCount, goroutineCount, "", false); err != nil {
ctx.ServerError("WriteProcesses", err)
return
}

ctx.Data["StacktraceString"] = sb.String()

ctx.HTML(http.StatusOK, tplStacktrace)
}
Expand Down
113 changes: 113 additions & 0 deletions routers/web/admin/pprof.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package admin

import (
"fmt"
"runtime/pprof"
"strconv"
"time"

"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/setting"

"github.com/felixge/fgprof"
)

// PProfFGProfile returns the Full Go Profile from fgprof
func PProfFGProfile(ctx *context.Context) {
durationStr := ctx.FormString("duration")
duration := 30 * time.Second
if durationStr != "" {
var err error
duration, err = time.ParseDuration(durationStr)
if err != nil {
ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid"))
ctx.Redirect(setting.AppSubURL + "/admin/monitor")
return
}
}

format := ctx.FormString("format")
if format != fgprof.FormatFolded {
format = fgprof.FormatPprof
}

start := time.Now()

ctx.SetServeHeaders(&context.ServeHeaderOptions{
Filename: "fgprof-profile-" + strconv.FormatInt(start.Unix(), 10),
LastModified: start,
})

fn := fgprof.Start(ctx.Resp, format)

select {
case <-time.After(duration):
case <-ctx.Done():
}

err := fn()
if err != nil {
ctx.ServerError("fgprof.Write", err)
}
}


// PProfCPUProfile returns the PProf CPU Profile
func PProfCPUProfile(ctx *context.Context) {
durationStr := ctx.FormString("duration")
duration := 30 * time.Second
if durationStr != "" {
var err error
duration, err = time.ParseDuration(durationStr)
if err != nil {
ctx.Flash.Error(ctx.Tr("monitor.pprof.cpuprofile.duration_invalid"))
ctx.Redirect(setting.AppSubURL + "/admin/monitor")
return
}
}

start := time.Now()

ctx.SetServeHeaders(&context.ServeHeaderOptions{
Filename: "cpu-profile-" + strconv.FormatInt(start.Unix(), 10),
LastModified: start,
})

err := pprof.StartCPUProfile(ctx.Resp)
if err != nil {
ctx.ServerError("StartCPUProfile", err)
return
}

select {
case <-time.After(duration):
case <-ctx.Done():
}
pprof.StopCPUProfile()
}

// PProfNamedProfile returns the PProf Profile
func PProfNamedProfile(ctx *context.Context) {
name := ctx.FormString("name")
profile := pprof.Lookup(name)
if profile == nil {
ctx.ServerError(fmt.Sprintf("pprof.Lookup(%s)", name), fmt.Errorf("missing profile: %s", name))
return
}

debug := ctx.FormInt("debug")

start := time.Now()

ctx.SetServeHeaders(&context.ServeHeaderOptions{
Filename: name + "-profile-" + strconv.FormatInt(start.Unix(), 10),
LastModified: start,
})
if err := profile.WriteTo(ctx.Resp, debug); err != nil {
ctx.ServerError(fmt.Sprintf("PProfNamedProfile(%s).WriteTo", name), err)
return
}
}
Loading