web/forgejo
10
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Refactor Cron and merge dashboard tasks (#10745)

Browse source 
* Refactor Cron and merge dashboard tasks

* Merge Cron and Dashboard tasks
* Make every cron task report a system notice on completion
* Refactor the creation of these tasks
* Ensure that execution counts of tasks is correct
* Allow cron tasks to be started from the cron page

* golangci-lint fixes

* Enforce that only one task with the same name can be registered

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix name check

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per @guillep2k

* as per @lafriks

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add git.CommandContext variants

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
zeripath 2020-05-17 00:31:38 +01:00 • committed by GitHub
parent c18144086f
commit 9a2e47b23a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 851 additions and 453 deletions
Download patch file Download diff file Expand all files Collapse all files

179
modules/cron/cron.go
View file

@ -9,143 +9,86 @@ import (
"context"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/sync"
mirror_service "code.gitea.io/gitea/services/mirror"
"github.com/gogs/cron"
)
const (
mirrorUpdate = "mirror_update"
gitFsck = "git_fsck"
checkRepos = "check_repos"
archiveCleanup = "archive_cleanup"
syncExternalUsers = "sync_external_users"
deletedBranchesCleanup = "deleted_branches_cleanup"
updateMigrationPosterID = "update_migration_post_id"
)
var c = cron.New()
// Prevent duplicate running tasks.
var taskStatusTable = sync.NewStatusTable()
// Func defines a cron function body
type Func func()
// WithUnique wrap a cron func with an unique running check
func WithUnique(name string, body func(context.Context)) Func {
return func() {
if !taskStatusTable.StartIfNotRunning(name) {
return
}
defer taskStatusTable.Stop(name)
graceful.GetManager().RunWithShutdownContext(body)
}
}
// NewContext begins cron tasks
// Each cron task is run within the shutdown context as a running server
// AtShutdown the cron server is stopped
func NewContext() {
var (
entry *cron.Entry
err error
)
if setting.Cron.UpdateMirror.Enabled {
entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, mirror_service.Update))
if err != nil {
log.Fatal("Cron[Update mirrors]: %v", err)
}
if setting.Cron.UpdateMirror.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(mirrorUpdate, mirror_service.Update)()
}
}
if setting.Cron.RepoHealthCheck.Enabled {
entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, WithUnique(gitFsck, func(ctx context.Context) {
if err := repo_module.GitFsck(ctx); err != nil {
log.Error("GitFsck: %s", err)
}
}))
if err != nil {
log.Fatal("Cron[Repository health check]: %v", err)
}
if setting.Cron.RepoHealthCheck.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(gitFsck, func(ctx context.Context) {
if err := repo_module.GitFsck(ctx); err != nil {
log.Error("GitFsck: %s", err)
}
})()
}
}
if setting.Cron.CheckRepoStats.Enabled {
entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, WithUnique(checkRepos, models.CheckRepoStats))
if err != nil {
log.Fatal("Cron[Check repository statistics]: %v", err)
}
if setting.Cron.CheckRepoStats.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(checkRepos, models.CheckRepoStats)()
}
}
if setting.Cron.ArchiveCleanup.Enabled {
entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives))
if err != nil {
log.Fatal("Cron[Clean up old repository archives]: %v", err)
}
if setting.Cron.ArchiveCleanup.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives)()
}
}
if setting.Cron.SyncExternalUsers.Enabled {
entry, err = c.AddFunc("Synchronize external users", setting.Cron.SyncExternalUsers.Schedule, WithUnique(syncExternalUsers, models.SyncExternalUsers))
if err != nil {
log.Fatal("Cron[Synchronize external users]: %v", err)
}
if setting.Cron.SyncExternalUsers.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(syncExternalUsers, models.SyncExternalUsers)()
}
}
if setting.Cron.DeletedBranchesCleanup.Enabled {
entry, err = c.AddFunc("Remove old deleted branches", setting.Cron.DeletedBranchesCleanup.Schedule, WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches))
if err != nil {
log.Fatal("Cron[Remove old deleted branches]: %v", err)
}
if setting.Cron.DeletedBranchesCleanup.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
}
}
initBasicTasks()
initExtendedTasks()
entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID))
if err != nil {
log.Fatal("Cron[Update migrated repositories]: %v", err)
lock.Lock()
for _, task := range tasks {
if task.IsEnabled() && task.DoRunAtStart() {
go task.Run()
}
}
entry.Prev = time.Now()
entry.ExecTimes++
go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
c.Start()
graceful.GetManager().RunAtShutdown(context.Background(), c.Stop)
started = true
lock.Unlock()
graceful.GetManager().RunAtShutdown(context.Background(), func() {
c.Stop()
lock.Lock()
started = false
lock.Unlock()
})
}
// ListTasks returns all running cron tasks.
func ListTasks() []*cron.Entry {
return c.Entries()
// TaskTableRow represents a task row in the tasks table
type TaskTableRow struct {
Name string
Spec string
Next time.Time
Prev time.Time
ExecTimes int64
}
// TaskTable represents a table of tasks
type TaskTable []*TaskTableRow
// ListTasks returns all running cron tasks.
func ListTasks() TaskTable {
entries := c.Entries()
eMap := map[string]*cron.Entry{}
for _, e := range entries {
eMap[e.Description] = e
}
lock.Lock()
defer lock.Unlock()
tTable := make([]*TaskTableRow, 0, len(tasks))
for _, task := range tasks {
spec := "-"
var (
next time.Time
prev time.Time
)
if e, ok := eMap[task.Name]; ok {
spec = e.Spec
next = e.Next
prev = e.Prev
}
task.lock.Lock()
tTable = append(tTable, &TaskTableRow{
Name: task.Name,
Spec: spec,
Next: next,
Prev: prev,
ExecTimes: task.ExecTimes,
})
task.lock.Unlock()
}
return tTable
}