Add Goroutine stack inspector to admin/monitor (#19207)

Continues on from #19202.

Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. 

This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile.

The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines.

If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. 

The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound.

In this way we should be able to more easily examine goroutines that have been stuck.

A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line.

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2022-03-31 18:01:43 +01:00 committed by GitHub
parent 9c349a4277
commit c88547ce71
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1479 additions and 595 deletions

View file

@ -6,61 +6,34 @@ package process
import (
"context"
"sync"
"time"
)
// Process represents a working process inheriting from Gitea.
type Process struct {
var (
SystemProcessType = "system"
RequestProcessType = "request"
NormalProcessType = "normal"
NoneProcessType = "none"
)
// process represents a working process inheriting from Gitea.
type process struct {
PID IDType // Process ID, not system one.
ParentPID IDType
Description string
Start time.Time
Cancel context.CancelFunc
lock sync.Mutex
children []*Process
Type string
}
// Children gets the children of the process
// Note: this function will behave nicely even if p is nil
func (p *Process) Children() (children []*Process) {
if p == nil {
return
}
p.lock.Lock()
defer p.lock.Unlock()
children = make([]*Process, len(p.children))
copy(children, p.children)
return children
}
// AddChild adds a child process
// Note: this function will behave nicely even if p is nil
func (p *Process) AddChild(child *Process) {
if p == nil {
return
}
p.lock.Lock()
defer p.lock.Unlock()
p.children = append(p.children, child)
}
// RemoveChild removes a child process
// Note: this function will behave nicely even if p is nil
func (p *Process) RemoveChild(process *Process) {
if p == nil {
return
}
p.lock.Lock()
defer p.lock.Unlock()
for i, child := range p.children {
if child == process {
p.children = append(p.children[:i], p.children[i+1:]...)
return
}
// ToProcess converts a process to a externally usable Process
func (p *process) toProcess() *Process {
process := &Process{
PID: p.PID,
ParentPID: p.ParentPID,
Description: p.Description,
Start: p.Start,
Type: p.Type,
}
return process
}