Skip to content

Commit 1804e52

Browse files
committed
Reformat stacktraces to keep with processes
Signed-off-by: Andrew Thornton <[email protected]>
1 parent a08537a commit 1804e52

File tree

5 files changed

+262
-47
lines changed

5 files changed

+262
-47
lines changed

modules/process/manager.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,18 @@ var (
3030
DefaultContext = context.Background()
3131
)
3232

33+
// DescriptionPProfLabel is a label set on goroutines that have a process attached
34+
const DescriptionPProfLabel = "process-description"
35+
36+
// PIDPProfLabel is a label set on goroutines that have a process attached
37+
const PIDPProfLabel = "pid"
38+
39+
// PPIDPProfLabel is a label set on goroutines that have a process attached
40+
const PPIDPProfLabel = "ppid"
41+
42+
// ProcessTypePProfLabel is a label set on goroutines that have a process attached
43+
const ProcessTypePProfLabel = "process-type"
44+
3345
// IDType is a pid type
3446
type IDType string
3547

@@ -148,7 +160,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C
148160
pm.processes[pid] = process
149161
pm.mutex.Unlock()
150162

151-
pprofCtx := pprof.WithLabels(ctx, pprof.Labels("process-description", description, "ppid", string(parentPID), "pid", string(pid), "process-type", processType))
163+
pprofCtx := pprof.WithLabels(ctx, pprof.Labels(DescriptionPProfLabel, description, PPIDPProfLabel, string(parentPID), PIDPProfLabel, string(pid), ProcessTypePProfLabel, processType))
152164
pprof.SetGoroutineLabels(pprofCtx)
153165

154166
return pprofCtx, pid, finished
@@ -206,12 +218,12 @@ func (pm *Manager) Cancel(pid IDType) {
206218
}
207219

208220
// Processes gets the processes in a thread safe manner
209-
func (pm *Manager) Processes(onlyRoots bool) []*Process {
221+
func (pm *Manager) Processes(onlyRoots, noSystem bool, runInLock func()) []*Process {
210222
pm.mutex.Lock()
211223
processes := make([]*Process, 0, len(pm.processes))
212224
if onlyRoots {
213225
for _, process := range pm.processes {
214-
if process.Type == SystemProcessType {
226+
if noSystem && process.Type == SystemProcessType {
215227
continue
216228
}
217229
if parent, has := pm.processes[process.ParentPID]; !has || parent.Type == SystemProcessType {
@@ -220,9 +232,15 @@ func (pm *Manager) Processes(onlyRoots bool) []*Process {
220232
}
221233
} else {
222234
for _, process := range pm.processes {
235+
if noSystem && process.Type == SystemProcessType {
236+
continue
237+
}
223238
processes = append(processes, process)
224239
}
225240
}
241+
if runInLock != nil {
242+
runInLock()
243+
}
226244
pm.mutex.Unlock()
227245

228246
sort.Slice(processes, func(i, j int) bool {

options/locale/locale_en-US.ini

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2813,6 +2813,7 @@ monitor.previous = Previous Time
28132813
monitor.execute_times = Executions
28142814
monitor.process = Running Processes
28152815
monitor.stacktrace = Stacktraces
2816+
monitor.goroutines=%d Goroutines
28162817
monitor.desc = Description
28172818
monitor.start = Start Time
28182819
monitor.execute_time = Execution Time

routers/web/admin/admin.go

Lines changed: 171 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,11 @@ import (
1313
"os"
1414
"runtime"
1515
"runtime/pprof"
16+
"sort"
1617
"strconv"
1718
"strings"
1819
"time"
1920

20-
"github.com/google/pprof/profile"
21-
2221
"code.gitea.io/gitea/models"
2322
"code.gitea.io/gitea/modules/base"
2423
"code.gitea.io/gitea/modules/context"
@@ -36,6 +35,7 @@ import (
3635
"code.gitea.io/gitea/services/mailer"
3736

3837
"gitea.com/go-chi/session"
38+
"github.com/google/pprof/profile"
3939
)
4040

4141
const (
@@ -331,7 +331,7 @@ func Monitor(ctx *context.Context) {
331331
ctx.Data["Title"] = ctx.Tr("admin.monitor")
332332
ctx.Data["PageIsAdmin"] = true
333333
ctx.Data["PageIsAdminMonitor"] = true
334-
ctx.Data["Processes"] = process.GetManager().Processes(true)
334+
ctx.Data["Processes"] = process.GetManager().Processes(true, true, nil)
335335
ctx.Data["Entries"] = cron.ListTasks()
336336
ctx.Data["Queues"] = queue.GetManager().ManagedQueues()
337337

@@ -344,18 +344,176 @@ func GoroutineStacktrace(ctx *context.Context) {
344344
ctx.Data["PageIsAdmin"] = true
345345
ctx.Data["PageIsAdminMonitor"] = true
346346

347-
reader, writer := io.Pipe()
348-
defer reader.Close()
349-
go func() {
350-
err := pprof.Lookup("goroutine").WriteTo(writer, 0)
351-
_ = writer.CloseWithError(err)
352-
}()
353-
p, err := profile.Parse(reader)
354-
if err != nil {
355-
ctx.ServerError("GoroutineStacktrace", err)
347+
var stacks *profile.Profile
348+
processes := process.GetManager().Processes(false, false, func() {
349+
reader, writer := io.Pipe()
350+
defer reader.Close()
351+
go func() {
352+
err := pprof.Lookup("goroutine").WriteTo(writer, 0)
353+
_ = writer.CloseWithError(err)
354+
}()
355+
var err error
356+
stacks, err = profile.Parse(reader)
357+
if err != nil {
358+
ctx.ServerError("GoroutineStacktrace", err)
359+
return
360+
}
361+
})
362+
if ctx.Written() {
356363
return
357364
}
358-
ctx.Data["Profile"] = p
365+
366+
type StackEntry struct {
367+
Function string
368+
File string
369+
Line int
370+
}
371+
372+
type Label struct {
373+
Name string
374+
Value string
375+
}
376+
377+
type Stack struct {
378+
Count int64
379+
Description string
380+
Labels []*Label
381+
Entry []*StackEntry
382+
}
383+
384+
type ProcessStack struct {
385+
PID process.IDType
386+
ParentPID process.IDType
387+
Description string
388+
Start time.Time
389+
Type string
390+
391+
Children []*ProcessStack
392+
Stacks []*Stack
393+
}
394+
395+
// Now earlier we sorted by process time so we know that we should not have children before parents
396+
pidMap := map[process.IDType]*ProcessStack{}
397+
processStacks := make([]*ProcessStack, 0, len(processes))
398+
for _, process := range processes {
399+
pStack := &ProcessStack{
400+
PID: process.PID,
401+
ParentPID: process.ParentPID,
402+
Description: process.Description,
403+
Start: process.Start,
404+
Type: process.Type,
405+
}
406+
407+
pidMap[process.PID] = pStack
408+
if parent, ok := pidMap[process.ParentPID]; ok {
409+
parent.Children = append(parent.Children, pStack)
410+
} else {
411+
processStacks = append(processStacks, pStack)
412+
}
413+
}
414+
415+
goroutineCount := int64(0)
416+
417+
// Now walk through the "Sample" slice in the goroutines stack
418+
for _, sample := range stacks.Sample {
419+
stack := &Stack{}
420+
421+
// Add the labels
422+
for name, value := range sample.Label {
423+
if name == process.DescriptionPProfLabel || name == process.PIDPProfLabel || name == process.PPIDPProfLabel || name == process.ProcessTypePProfLabel {
424+
continue
425+
}
426+
if len(value) != 1 {
427+
// Unexpected...
428+
log.Error("Label: %s in goroutine stack with unexpected number of values: %v", name, value)
429+
continue
430+
}
431+
432+
stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]})
433+
}
434+
435+
stack.Count = sample.Value[0]
436+
goroutineCount += stack.Count
437+
438+
// Now get the processStack for this goroutine sample
439+
var processStack *ProcessStack
440+
if pidvalue, ok := sample.Label[process.PIDPProfLabel]; ok && len(pidvalue) == 1 {
441+
pid := process.IDType(pidvalue[0])
442+
processStack, ok = pidMap[pid]
443+
if !ok && pid != "" {
444+
ppid := process.IDType("")
445+
if value, ok := sample.Label[process.PPIDPProfLabel]; ok && len(value) == 1 {
446+
ppid = process.IDType(value[0])
447+
}
448+
description := "(missing process)"
449+
if value, ok := sample.Label[process.DescriptionPProfLabel]; ok && len(value) == 1 {
450+
description = value[0] + " " + description
451+
}
452+
ptype := process.SystemProcessType
453+
if value, ok := sample.Label[process.ProcessTypePProfLabel]; ok && len(value) == 1 {
454+
ptype = value[0]
455+
}
456+
processStack = &ProcessStack{
457+
PID: pid,
458+
ParentPID: ppid,
459+
Description: description,
460+
Type: ptype,
461+
}
462+
463+
pidMap[processStack.PID] = processStack
464+
if parent, ok := pidMap[processStack.ParentPID]; ok {
465+
parent.Children = append(parent.Children, processStack)
466+
}
467+
}
468+
}
469+
if processStack == nil {
470+
var ok bool
471+
processStack, ok = pidMap[""]
472+
if !ok {
473+
processStack = &ProcessStack{
474+
Description: "(unknown)",
475+
Type: "code",
476+
}
477+
pidMap[processStack.PID] = processStack
478+
processStacks = append(processStacks, processStack)
479+
}
480+
}
481+
482+
// Now walk through the locations...
483+
for _, location := range sample.Location {
484+
for _, line := range location.Line {
485+
entry := &StackEntry{
486+
Function: line.Function.Name,
487+
File: line.Function.Filename,
488+
Line: int(line.Line),
489+
}
490+
stack.Entry = append(stack.Entry, entry)
491+
}
492+
}
493+
stack.Description = "(others)"
494+
if len(stack.Entry) > 0 {
495+
stack.Description = stack.Entry[len(stack.Entry)-1].Function
496+
}
497+
498+
processStack.Stacks = append(processStack.Stacks, stack)
499+
}
500+
501+
// Now finally re-sort the processstacks so the newest processes are at the top
502+
after := func(processStacks []*ProcessStack) func(i, j int) bool {
503+
return func(i, j int) bool {
504+
left, right := processStacks[i], processStacks[j]
505+
return left.Start.After(right.Start)
506+
}
507+
}
508+
sort.Slice(processStacks, after(processStacks))
509+
for _, processStack := range processStacks {
510+
sort.Slice(processStack.Children, after(processStack.Children))
511+
}
512+
513+
ctx.Data["ProcessStacks"] = processStacks
514+
ctx.Data["Profile"] = stacks
515+
516+
ctx.Data["GoroutineCount"] = goroutineCount
359517

360518
ctx.HTML(http.StatusOK, tplStacktrace)
361519
}

templates/admin/stacktrace-row.tmpl

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<div class="item">
2+
<div class="df ac">
3+
<div class="icon ml-3 mr-3">
4+
{{if eq .Process.Type "request"}}
5+
{{svg "octicon-globe" 16 }}
6+
{{else if eq .Process.Type "system"}}
7+
{{svg "octicon-cpu" 16 }}
8+
{{else if eq .Process.Type "normal"}}
9+
{{svg "octicon-terminal" 16 }}
10+
{{else}}
11+
{{svg "octicon-code" 16 }}
12+
{{end}}
13+
</div>
14+
<div class="content f1">
15+
<div class="header">{{.Process.Description}}</div>
16+
<div class="description">{{if ne .Process.Type "code"}}<span title="{{DateFmtLong .Process.Start}}">{{TimeSince .Process.Start .root.i18n.Lang}}</span>{{end}}</div>
17+
</div>
18+
<div>
19+
{{if or (eq .Process.Type "request") (eq .Process.Type "normal") }}
20+
<a class="delete-button icon" href="" data-url="{{.root.Link}}/cancel/{{.Process.PID}}" data-id="{{.Process.PID}}" data-name="{{.Process.Description}}">{{svg "octicon-trash" 16 "text-red"}}</a>
21+
{{end}}
22+
</div>
23+
</div>
24+
{{if .Process.Stacks}}
25+
<div class="divided list ml-3">
26+
{{range .Process.Stacks}}
27+
<div class="item">
28+
<details>
29+
<summary>
30+
<div class="dif content">
31+
<div class="header ml-3">
32+
<span class="icon mr-3">{{svg "octicon-code" 16 }}</span>{{.Description}}{{if gt .Count 1}} * {{.Count}}{{end}}
33+
</div>
34+
<div class="description">
35+
{{range .Labels}}
36+
<div class="ui label">{{.Name}}<div class="detail">{{.Value}}</div></div>
37+
{{end}}
38+
</div>
39+
</div>
40+
</summary>
41+
<div class="list">
42+
{{range .Entry}}
43+
<div class="item df ac">
44+
<span class="icon mr-4">{{svg "octicon-dot-fill" 16 }}</span>
45+
<div class="content f1">
46+
<div class="header"><code>{{.Function}}</code></div>
47+
<div class="description"><code>{{.File}}:{{.Line}}</code></div>
48+
</div>
49+
</div>
50+
{{end}}
51+
</div>
52+
</details>
53+
</div>
54+
{{end}}
55+
</div>
56+
{{end}}
57+
58+
{{if .Process.Children}}
59+
<div class="divided list">
60+
{{range .Process.Children}}
61+
{{template "admin/process-row" dict "Process" . "root" $.root}}
62+
{{end}}
63+
</div>
64+
{{end}}
65+
66+
</div>

templates/admin/stacktrace.tmpl

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,15 @@
44
<div class="ui container">
55
{{template "base/alert" .}}
66
<h4 class="ui top attached header">
7-
{{.i18n.Tr "admin.monitor.stacktrace"}}
7+
{{.i18n.Tr "admin.monitor.stacktrace"}}: {{.i18n.Tr "admin.monitor.goroutines" .GoroutineCount}}
88
<div class="ui right">
99
<a class="ui blue tiny button" href="{{AppSubUrl}}/admin/monitor">{{.i18n.Tr "admin.monitor"}}</a>
1010
</div>
1111
</h4>
1212
<div class="ui attached segment">
1313
<div class="ui relaxed divided list">
14-
{{range .Profile.Sample}}
15-
<div class="item">
16-
<div class="df ac">
17-
<div class="content f1">
18-
<details><summary>
19-
<div class="dif header ml-3" style="max-width: 95%">{{if index .Label "process-description"}}<span class="icon mr-3">{{if eq (index (index .Label "process-type") 0) "request"}}{{svg "octicon-globe" 16 }}{{else if eq (index (index .Label "process-type") 0) "system"}}{{svg "octicon-cpu" 16 }}{{else}}{{svg "octicon-terminal" 16 }}{{end}}</span><div class="mr-3">{{index (index .Label "process-description") 0}}</div>{{else}}<span class="icon mr-3">{{svg "octicon-code" 16 }}</span><div class="mr-3"><code>{{(index (index .Location (Subtract (len .Location) 1)).Line 0).Function.Name}}</code></div>{{end}}{{if gt (index .Value 0) 1}}<div class="mr-3"> * {{index .Value 0}}</div>{{end}}</div>
20-
{{range $key, $value := .Label}}
21-
{{if ne $key "process-description" }}
22-
<div class="ui label">{{$key}}<div class="detail">{{index $value 0}}</div></div>
23-
{{end}}
24-
{{end}}
25-
</summary>
26-
27-
<div class="list">
28-
{{range .Location}}
29-
<div class="item df ac">
30-
<span class="icon mr-4">{{svg "octicon-dot-fill" 16 }}</span>
31-
<div class="content f1">
32-
<div class="header"><code>{{(index .Line 0).Function.Name}}</code></div>
33-
<div class="description"><code>{{(index .Line 0).Function.Filename}}:{{(index .Line 0).Line}}</code></div>
34-
</div>
35-
</div>
36-
{{end}}
37-
</div>
38-
</details>
39-
</div>
40-
<div>
41-
</div>
42-
</div>
43-
</div>
14+
{{range .ProcessStacks}}
15+
{{template "admin/stacktrace-row" dict "Process" . "root" $}}
4416
{{end}}
4517
</div>
4618
</div>

0 commit comments

Comments
 (0)