bjoernager/forgejo
bjoernager/forgejo
1
Fork 0
Code Activity

Merge pull request '[gitea] week 2024-32 cherry pick (gitea/main -> forgejo)' (#4801) from earl-warren/wcp/2024-32 into forgejo

Browse source 
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4801
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
This commit is contained in:
Earl Warren 2024-08-06 05:47:20 +00:00
commit 517637137c
46 changed files with 661 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

66
services/actions/cleanup.go
View file

@ -5,19 +5,30 @@ package actions
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/models/actions"
actions_model "code.gitea.io/gitea/models/actions"
actions_module "code.gitea.io/gitea/modules/actions"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"
)
// Cleanup removes expired actions logs, data and artifacts
func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
// TODO: clean up expired actions logs
func Cleanup(ctx context.Context) error {
// clean up expired artifacts
return CleanupArtifacts(taskCtx)
if err := CleanupArtifacts(ctx); err != nil {
return fmt.Errorf("cleanup artifacts: %w", err)
}
// clean up old logs
if err := CleanupLogs(ctx); err != nil {
return fmt.Errorf("cleanup logs: %w", err)
}
return nil
}
// CleanupArtifacts removes expired add need-deleted artifacts and set records expired status
@ -29,13 +40,13 @@ func CleanupArtifacts(taskCtx context.Context) error {
}
func cleanExpiredArtifacts(taskCtx context.Context) error {
artifacts, err := actions.ListNeedExpiredArtifacts(taskCtx)
artifacts, err := actions_model.ListNeedExpiredArtifacts(taskCtx)
if err != nil {
return err
}
log.Info("Found %d expired artifacts", len(artifacts))
for _, artifact := range artifacts {
if err := actions.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
if err := actions_model.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
continue
}
@ -53,13 +64,13 @@ const deleteArtifactBatchSize = 100
func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
for {
artifacts, err := actions.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
artifacts, err := actions_model.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
if err != nil {
return err
}
log.Info("Found %d artifacts pending deletion", len(artifacts))
for _, artifact := range artifacts {
if err := actions.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
if err := actions_model.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
log.Error("Cannot set artifact %d deleted: %v", artifact.ID, err)
continue
}
@ -76,3 +87,40 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
}
return nil
}
const deleteLogBatchSize = 100
// CleanupLogs removes logs which are older than the configured retention time
func CleanupLogs(ctx context.Context) error {
olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.LogRetentionDays) * 24 * time.Hour)
count := 0
for {
tasks, err := actions_model.FindOldTasksToExpire(ctx, olderThan, deleteLogBatchSize)
if err != nil {
return fmt.Errorf("find old tasks: %w", err)
}
for _, task := range tasks {
if err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename); err != nil {
log.Error("Failed to remove log %s (in storage %v) of task %v: %v", task.LogFilename, task.LogInStorage, task.ID, err)
// do not return error here, continue to next task
continue
}
task.LogIndexes = nil // clear log indexes since it's a heavy field
task.LogExpired = true
if err := actions_model.UpdateTask(ctx, task, "log_indexes", "log_expired"); err != nil {
log.Error("Failed to update task %v: %v", task.ID, err)
// do not return error here, continue to next task
continue
}
count++
log.Trace("Removed log %s of task %v", task.LogFilename, task.ID)
}
if len(tasks) < deleteLogBatchSize {
break
}
}
log.Info("Removed %d logs", count)
return nil
}

11
services/cron/tasks_actions.go
View file

@ -19,6 +19,7 @@ func initActionsTasks() {
registerStopEndlessTasks()
registerCancelAbandonedJobs()
registerScheduleTasks()
registerActionsCleanup()
}
func registerStopZombieTasks() {
@ -63,3 +64,13 @@ func registerScheduleTasks() {
return actions_service.StartScheduleTasks(ctx)
})
}
func registerActionsCleanup() {
RegisterTaskFatal("cleanup_actions", &BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "@midnight",
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
return actions_service.Cleanup(ctx)
})
}

18
services/cron/tasks_basic.go
View file

@ -13,7 +13,6 @@ import (
"code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/actions"
"code.gitea.io/gitea/services/auth"
"code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
@ -157,20 +156,6 @@ func registerCleanupPackages() {
})
}
func registerActionsCleanup() {
RegisterTaskFatal("cleanup_actions", &OlderThanConfig{
BaseConfig: BaseConfig{
Enabled: true,
RunAtStart: true,
Schedule: "@midnight",
},
OlderThan: 24 * time.Hour,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
realConfig := config.(*OlderThanConfig)
return actions.Cleanup(ctx, realConfig.OlderThan)
})
}
func initBasicTasks() {
if setting.Mirror.Enabled {
registerUpdateMirrorTask()
@ -187,7 +172,4 @@ func initBasicTasks() {
if setting.Packages.Enabled {
registerCleanupPackages()
}
if setting.Actions.Enabled {
registerActionsCleanup()
}
}

1
services/repository/migrate.go
View file

@ -172,6 +172,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
lfsClient := lfs.NewClient(endpoint, httpTransport)
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil {
log.Error("Failed to store missing LFS objects for repository: %v", err)
return repo, fmt.Errorf("StoreMissingLfsObjectsInRepository: %w", err)
}
}
}