From 3a86167b45a9054641daf361182606555f1e47d8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 16 Jun 2023 13:56:27 +0000 Subject: [PATCH 1/6] Allow package cleanup from admin page. --- routers/web/admin/packages.go | 12 ++++++++++++ routers/web/web.go | 1 + services/cron/tasks_basic.go | 2 +- services/packages/cleanup/cleanup.go | 26 ++++++++++++++++++++++---- templates/admin/packages/list.tmpl | 6 ++++++ 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 4d5987e780b6f..fd0f1fa2431dc 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -6,6 +6,7 @@ package admin import ( "net/http" "net/url" + "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -14,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" ) const ( @@ -101,3 +103,13 @@ func DeletePackageVersion(ctx *context.Context) { "redirect": setting.AppSubURL + "/admin/packages?page=" + url.QueryEscape(ctx.FormString("page")) + "&q=" + url.QueryEscape(ctx.FormString("q")) + "&type=" + url.QueryEscape(ctx.FormString("type")), }) } + +func CleanupExpiredData(ctx *context.Context) { + if err := packages_cleanup_service.CleanupExpiredData(ctx, time.Duration(0)); err != nil { + ctx.ServerError("CleanupExpiredData", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.cleanup.success")) + ctx.Redirect(setting.AppSubURL + "/admin/packages") +} diff --git a/routers/web/web.go b/routers/web/web.go index 8683ef221dd97..2e9e60fd12db6 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -593,6 +593,7 @@ func registerRoutes(m *web.Route) { m.Group("/packages", func() { m.Get("", admin.Packages) m.Post("/delete", admin.DeletePackageVersion) + m.Post("/cleanup", admin.CleanupExpiredData) }, packagesEnabled) m.Group("/hooks", func() { diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 2e6560ec0c9dd..2a213ae51524c 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -152,7 +152,7 @@ func registerCleanupPackages() { OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { realConfig := config.(*OlderThanConfig) - return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan) + return packages_cleanup_service.CleanupTask(ctx, realConfig.OlderThan) }) } diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 43fbc1ad9b113..77bcfb194232f 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -20,9 +20,17 @@ import ( debian_service "code.gitea.io/gitea/services/packages/debian" ) -// Cleanup removes expired package data -func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(taskCtx) +// Task method to execute cleanup rules and cleanup expired package data +func CleanupTask(ctx context.Context, olderThan time.Duration) error { + if err := ExecuteCleanupRules(ctx); err != nil { + return err + } + + return CleanupExpiredData(ctx, olderThan) +} + +func ExecuteCleanupRules(outerCtx context.Context) error { + ctx, committer, err := db.TxContext(outerCtx) if err != nil { return err } @@ -30,7 +38,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { select { - case <-taskCtx.Done(): + case <-outerCtx.Done(): return db.ErrCancelledf("While processing package cleanup rules") default: } @@ -122,6 +130,16 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { return err } + return committer.Commit() +} + +func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(outerCtx) + if err != nil { + return err + } + defer committer.Close() + if err := container_service.Cleanup(ctx, olderThan); err != nil { return err } diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index f320878b80de9..03675d62725d7 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -4,6 +4,12 @@ {{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .TotalCount}}, {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}, {{.locale.Tr "admin.packages.unreferenced_size" (FileSize .TotalUnreferencedBlobSize)}}) +
+
+ {{.CsrfTokenHtml}} + +
+
From d99dbb845b25cdaa26e1c09878e9355e622947f3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 16 Jun 2023 14:09:21 +0000 Subject: [PATCH 2/6] Add translation. --- options/locale/locale_en-US.ini | 1 + templates/admin/packages/list.tmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 25456d0493426..77cbaa272c8ec 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2794,6 +2794,7 @@ repos.size = Size packages.package_manage_panel = Package Management packages.total_size = Total Size: %s packages.unreferenced_size = Unreferenced Size: %s +packages.cleanup = Cleanup expired data packages.owner = Owner packages.creator = Creator packages.name = Name diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 03675d62725d7..dd779b7bc8f9f 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -7,7 +7,7 @@
{{.CsrfTokenHtml}} - +
From 9597f60f1c10a4c90074e22c296489cf36ecc266 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 16 Jun 2023 16:39:41 +0200 Subject: [PATCH 3/6] Update templates/admin/packages/list.tmpl Co-authored-by: silverwind --- templates/admin/packages/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index dd779b7bc8f9f..d0cc2034a7241 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -5,7 +5,7 @@ {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}, {{.locale.Tr "admin.packages.unreferenced_size" (FileSize .TotalUnreferencedBlobSize)}})
-
+ {{.CsrfTokenHtml}}
From 930bb3336e79cd8aff5438d442821eeb9f35eec5 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 16 Jun 2023 15:39:44 +0000 Subject: [PATCH 4/6] Fix test. --- tests/integration/api_packages_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 84733f683b1c9..3054eeeea35cd 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -277,7 +277,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, "cleanup-test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.CleanupTask(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -412,7 +412,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.CleanupTask(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions { From 876aca7bc40a70ca7bec18a2c39d951e9354b327 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Aug 2023 20:36:39 +0200 Subject: [PATCH 5/6] Update options/locale/locale_en-US.ini Co-authored-by: silverwind --- options/locale/locale_en-US.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 77cbaa272c8ec..38da60315dd7c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2794,7 +2794,7 @@ repos.size = Size packages.package_manage_panel = Package Management packages.total_size = Total Size: %s packages.unreferenced_size = Unreferenced Size: %s -packages.cleanup = Cleanup expired data +packages.cleanup = Clean up expired data packages.owner = Owner packages.creator = Creator packages.name = Name From 04301aaae01f32316be68ea89e0b92a1dc03439b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 6 Aug 2023 17:58:07 +0200 Subject: [PATCH 6/6] Update templates/admin/packages/list.tmpl Co-authored-by: silverwind --- templates/admin/packages/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index d0cc2034a7241..7343cc545a190 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -7,7 +7,7 @@
{{.CsrfTokenHtml}} - +