Add Goroutine stack inspector to admin/monitor (#19207)

Continues on from #19202.

Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. 

This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile.

The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines.

If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. 

The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound.

In this way we should be able to more easily examine goroutines that have been stuck.

A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line.

Signed-off-by: Andrew Thornton <art27@cantab.net>

modules/log/event.go

@@ -5,9 +5,13 @@
 package log
 
 import (
+	"context"
 	"fmt"
+	"runtime/pprof"
 	"sync"
 	"time"
+
+	"code.gitea.io/gitea/modules/process"
 )
 
 // Event represents a logging event
@@ -34,6 +38,8 @@ type EventLogger interface {
 
 // ChannelledLog represents a cached channel to a LoggerProvider
 type ChannelledLog struct {
+	ctx            context.Context
+	finished       context.CancelFunc
 	name           string
 	provider       string
 	queue          chan *Event
@@ -44,8 +50,9 @@ type ChannelledLog struct {
 }
 
 // NewChannelledLog a new logger instance with given logger provider and config.
-func NewChannelledLog(name, provider, config string, bufferLength int64) (*ChannelledLog, error) {
+func NewChannelledLog(parent context.Context, name, provider, config string, bufferLength int64) (*ChannelledLog, error) {
 	if log, ok := providers[provider]; ok {
+
 		l := &ChannelledLog{
 			queue:  make(chan *Event, bufferLength),
 			flush:  make(chan bool),
@@ -58,6 +65,7 @@ func NewChannelledLog(name, provider, config string, bufferLength int64) (*Chann
 		}
 		l.name = name
 		l.provider = provider
+		l.ctx, _, l.finished = process.GetManager().AddTypedContext(parent, fmt.Sprintf("Logger: %s(%s)", l.name, l.provider), process.SystemProcessType, false)
 		go l.Start()
 		return l, nil
 	}
@@ -66,6 +74,8 @@ func NewChannelledLog(name, provider, config string, bufferLength int64) (*Chann
 
 // Start processing the ChannelledLog
 func (l *ChannelledLog) Start() {
+	pprof.SetGoroutineLabels(l.ctx)
+	defer l.finished()
 	for {
 		select {
 		case event, ok := <-l.queue:
@@ -140,6 +150,8 @@ func (l *ChannelledLog) GetName() string {
 
 // MultiChannelledLog represents a cached channel to a LoggerProvider
 type MultiChannelledLog struct {
+	ctx             context.Context
+	finished        context.CancelFunc
 	name            string
 	bufferLength    int64
 	queue           chan *Event
@@ -156,7 +168,11 @@ type MultiChannelledLog struct {
 
 // NewMultiChannelledLog a new logger instance with given logger provider and config.
 func NewMultiChannelledLog(name string, bufferLength int64) *MultiChannelledLog {
+	ctx, _, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Logger: %s", name), process.SystemProcessType, false)
+
 	m := &MultiChannelledLog{
+		ctx:             ctx,
+		finished:        finished,
 		name:            name,
 		queue:           make(chan *Event, bufferLength),
 		flush:           make(chan bool),
@@ -277,6 +293,9 @@ func (m *MultiChannelledLog) Start() {
 		m.rwmutex.Unlock()
 		return
 	}
+	pprof.SetGoroutineLabels(m.ctx)
+	defer m.finished()
+
 	m.started = true
 	m.rwmutex.Unlock()
 	paused := false

modules/log/multichannel.go

@@ -31,7 +31,7 @@ func newLogger(name string, buffer int64) *MultiChannelledLogger {
 
 // SetLogger sets new logger instance with given logger provider and config.
 func (l *MultiChannelledLogger) SetLogger(name, provider, config string) error {
-	eventLogger, err := NewChannelledLog(name, provider, config, l.bufferLength)
+	eventLogger, err := NewChannelledLog(l.ctx, name, provider, config, l.bufferLength)
 	if err != nil {
 		return fmt.Errorf("Failed to create sublogger (%s): %v", name, err)
 	}