Prune hook_task table

custom/conf/app.example.ini

@@ -926,6 +926,15 @@ UPDATE_EXISTING = true
 ; Interval as a duration between each synchronization. (default every 24h)
 SCHEDULE = @every 24h
 
+; Prune hook_task table
+[cron.prune_hook_task_table]
+; Whether to enable the job
+ENABLED = true
+; Whether to always run at start up time (if ENABLED)
+RUN_AT_START = false
+; Time interval for job to run
+SCHEDULE = @every 24h
+
 [git]
 ; The path of git executable. If empty, Gitea searches through the PATH environment.
 PATH =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -594,6 +594,12 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false`
 
 - `SCHEDULE`: **@every 24h** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts.
 
+### Cron - Prune hook_task Table (`cron.prune_hook_task_table`)
+
+- `ENABLED`: **true**: Enable service. In the repository table, there are two columns to additionally override the behavior per repository: is_hook_task_purge_enabled - enable or disable purging of hook_task data by repository and number_webhook_deliveries_to_keep - the process will leave the most recently delivered webhooks, this value controls how many are kept. These settings can also be modified per repository in the Gitea UI. By default, the process is enabled for every repository and 10 webhooks are kept.
+- `RUN_AT_START`: **false**: Run prune hook_task at start time (if ENABLED).
+- `SCHEDULE`: **@every 24h**: Cron syntax for pruning hook_task table.
+
 ## Git (`git`)
 
 - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment.

models/migrations/migrations.go

@@ -235,6 +235,8 @@ var migrations = []Migration{
 	NewMigration("Add Created and Updated to Milestone table", addCreatedAndUpdatedToMilestones),
 	// v150 -> v151
 	NewMigration("add primary key to repo_topic", addPrimaryKeyToRepoTopic),
+	// v151 -> v152
+	NewMigration("Add IsHookTaskPurgeEnabled and NumberWebhookDeliveriesToKeep columns to Repository table", addHookTaskPurge),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v151.go

@@ -0,0 +1,27 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func addHookTaskPurge(x *xorm.Engine) error {
+	type Repository struct {
+		ID                            int64 `xorm:"pk autoincr"`
+		IsHookTaskPurgeEnabled        bool  `xorm:"NOT NULL DEFAULT true"`
+		NumberWebhookDeliveriesToKeep int64 `xorm:"NOT NULL DEFAULT 10"`
+	}
+
+	if err := x.Sync2(new(Repository)); err != nil {
+		return err
+	}
+
+	_, err := x.Exec("UPDATE repository SET is_hook_task_purge_enabled = ?, number_webhook_deliveries_to_keep = ?",
+		setting.Repository.DefaultIsHookTaskPurgeEnabled, setting.Repository.DefaultNumberWebhookDeliveriesToKeep)
+	return err
+}

models/repo.go

@@ -196,6 +196,8 @@ type Repository struct {
 	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
 	CloseIssuesViaCommitInAnyBranch bool               `xorm:"NOT NULL DEFAULT false"`
+	IsHookTaskPurgeEnabled          bool               `xorm:"NOT NULL DEFAULT true"`
+	NumberWebhookDeliveriesToKeep   int64              `xorm:"NOT NULL DEFAULT 10"`
 	Topics                          []string           `xorm:"TEXT JSON"`
 
 	// Avatar: ID(10-20)-md5(32) - must fit into 64 symbols

models/webhook.go

@@ -798,3 +798,31 @@ func FindRepoUndeliveredHookTasks(repoID int64) ([]*HookTask, error) {
 	}
 	return tasks, nil
 }
+
+// DeleteDeliveredHookTasks deletes delivered hook tasks of one repository, leaving the most recent delivered based on the parameter
+func DeleteDeliveredHookTasks(repoID int64, numberDeliveriesToKeep int64) error {
+	var deliveryDates = make([]int64, 0, 10)
+	err := x.Table("hook_task").
+		Where("hook_task.repo_id = ? AND hook_task.is_delivered = ?", repoID, true).
+		Cols("hook_task.delivered").
+		Join("INNER", "repository", "hook_task.repo_id = repository.id").
+		And("repository.is_hook_task_purge_enabled = ?", true).
+		OrderBy("hook_task.delivered desc").
+		Limit(1, int(numberDeliveriesToKeep)).
+		Find(&deliveryDates)
+	if err != nil {
+		return err
+	}
+
+	if len(deliveryDates) > 0 {
+		deletes, err := x.
+			Where("repo_id = ? and is_delivered = ? and delivered <= ?", repoID, true, deliveryDates[0]).
+			Delete(new(HookTask))
+		if err != nil {
+			return err
+		}
+		log.Trace("From repo %d deleted in total %d from hook_task", repoID, deletes)
+	}
+
+	return nil
+}

models/webhook_test.go

@@ -7,6 +7,7 @@ package models
 import (
 	"encoding/json"
 	"testing"
+	"time"
 
 	api "code.gitea.io/gitea/modules/structs"
 
@@ -245,3 +246,58 @@ func TestUpdateHookTask(t *testing.T) {
 	assert.NoError(t, UpdateHookTask(hook))
 	AssertExistsAndLoadBean(t, hook)
 }
+
+func TestDeleteDeliveredHookTasks_DeletesDelivered(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+	hookTask := &HookTask{
+		RepoID:      3,
+		HookID:      3,
+		Type:        GITEA,
+		URL:         "http://www.example.com/unit_test",
+		Payloader:   &api.PushPayload{},
+		IsDelivered: true,
+	}
+	AssertNotExistsBean(t, hookTask)
+	assert.NoError(t, CreateHookTask(hookTask))
+	AssertExistsAndLoadBean(t, hookTask)
+
+	assert.NoError(t, DeleteDeliveredHookTasks(3, 0))
+	AssertNotExistsBean(t, hookTask)
+}
+
+func TestDeleteDeliveredHookTasks_LeavesUndelivered(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+	hookTask := &HookTask{
+		RepoID:      2,
+		HookID:      4,
+		Type:        GITEA,
+		URL:         "http://www.example.com/unit_test",
+		Payloader:   &api.PushPayload{},
+		IsDelivered: false,
+	}
+	AssertNotExistsBean(t, hookTask)
+	assert.NoError(t, CreateHookTask(hookTask))
+	AssertExistsAndLoadBean(t, hookTask)
+
+	assert.NoError(t, DeleteDeliveredHookTasks(3, 0))
+	AssertExistsAndLoadBean(t, hookTask)
+}
+
+func TestDeleteDeliveredHookTasks_LeavesMostRecentTask(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+	hookTask := &HookTask{
+		RepoID:      2,
+		HookID:      4,
+		Type:        GITEA,
+		URL:         "http://www.example.com/unit_test",
+		Payloader:   &api.PushPayload{},
+		IsDelivered: true,
+		Delivered:   time.Now().UnixNano(),
+	}
+	AssertNotExistsBean(t, hookTask)
+	assert.NoError(t, CreateHookTask(hookTask))
+	AssertExistsAndLoadBean(t, hookTask)
+
+	assert.NoError(t, DeleteDeliveredHookTasks(3, 1))
+	AssertExistsAndLoadBean(t, hookTask)
+}

modules/auth/repo_form.go

@@ -145,6 +145,8 @@ type RepoSettingForm struct {
 	// Admin settings
 	EnableHealthCheck                     bool
 	EnableCloseIssuesViaCommitInAnyBranch bool
+	IsHookTaskPurgeEnabled                bool
+	NumberWebhookDeliveriesToKeep         int64
 }
 
 // Validate validates the fields

modules/cron/tasks_basic.go

@@ -108,6 +108,16 @@ func registerUpdateMigrationPosterID() {
 	})
 }
 
+func registerPrunHookTaskTable() {
+	RegisterTaskFatal("prune_hook_task_table", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@every 24h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return repository_service.PruneHookTaskTable(ctx)
+	})
+}
+
 func initBasicTasks() {
 	registerUpdateMirrorTask()
 	registerRepoHealthCheck()
@@ -116,4 +126,5 @@ func initBasicTasks() {
 	registerSyncExternalUsers()
 	registerDeletedBranchesCleanup()
 	registerUpdateMigrationPosterID()
+	registerPrunHookTaskTable()
 }

modules/repository/create.go

@@ -39,6 +39,8 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m
 		IsPrivate:                       opts.IsPrivate,
 		IsFsckEnabled:                   !opts.IsMirror,
 		CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch,
+		IsHookTaskPurgeEnabled:          setting.Repository.DefaultIsHookTaskPurgeEnabled,
+		NumberWebhookDeliveriesToKeep:   setting.Repository.DefaultNumberWebhookDeliveriesToKeep,
 		Status:                          opts.Status,
 		IsEmpty:                         !opts.AutoInit,
 	}

modules/repository/generate.go

@@ -233,16 +233,18 @@ func GenerateGitContent(ctx models.DBContext, templateRepo, generateRepo *models
 // GenerateRepository generates a repository from a template
 func GenerateRepository(ctx models.DBContext, doer, owner *models.User, templateRepo *models.Repository, opts models.GenerateRepoOptions) (_ *models.Repository, err error) {
 	generateRepo := &models.Repository{
-		OwnerID:       owner.ID,
-		Owner:         owner,
-		OwnerName:     owner.Name,
-		Name:          opts.Name,
-		LowerName:     strings.ToLower(opts.Name),
-		Description:   opts.Description,
-		IsPrivate:     opts.Private,
-		IsEmpty:       !opts.GitContent || templateRepo.IsEmpty,
-		IsFsckEnabled: templateRepo.IsFsckEnabled,
-		TemplateID:    templateRepo.ID,
+		OwnerID:                       owner.ID,
+		Owner:                         owner,
+		OwnerName:                     owner.Name,
+		Name:                          opts.Name,
+		LowerName:                     strings.ToLower(opts.Name),
+		Description:                   opts.Description,
+		IsPrivate:                     opts.Private,
+		IsEmpty:                       !opts.GitContent || templateRepo.IsEmpty,
+		IsFsckEnabled:                 templateRepo.IsFsckEnabled,
+		IsHookTaskPurgeEnabled:        templateRepo.IsHookTaskPurgeEnabled,
+		NumberWebhookDeliveriesToKeep: templateRepo.NumberWebhookDeliveriesToKeep,
+		TemplateID:                    templateRepo.ID,
 	}
 
 	if err = models.CreateRepository(ctx, doer, owner, generateRepo); err != nil {

modules/repository/prune_hook_task.go

@@ -0,0 +1,49 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repository
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+
+	"xorm.io/builder"
+)
+
+// PruneHookTaskTable deletes rows from hook_task as needed.
+func PruneHookTaskTable(ctx context.Context) error {
+	log.Trace("Doing: PruneHookTaskTable")
+
+	if err := models.Iterate(
+		models.DefaultDBContext(),
+		new(models.Repository),
+		builder.Expr("id>0 AND is_hook_task_purge_enabled=?", true),
+		func(idx int, bean interface{}) error {
+			select {
+			case <-ctx.Done():
+				return fmt.Errorf("Aborted due to shutdown")
+			default:
+			}
+			repo := bean.(*models.Repository)
+			repoPath := repo.RepoPath()
+			log.Trace("Running prune hook_task table on repository %s", repoPath)
+			if err := models.DeleteDeliveredHookTasks(repo.ID, repo.NumberWebhookDeliveriesToKeep); err != nil {
+				desc := fmt.Sprintf("Failed to prune hook_task on repository (%s): %v", repoPath, err)
+				log.Warn(desc)
+				if err = models.CreateRepositoryNotice(desc); err != nil {
+					log.Error("CreateRepositoryNotice: %v", err)
+				}
+			}
+			return nil
+		},
+	); err != nil {
+		return err
+	}
+
+	log.Trace("Finished: PruneHookTaskTable")
+	return nil
+}

modules/setting/repository.go

@@ -37,6 +37,8 @@ var (
 		AccessControlAllowOrigin                string
 		UseCompatSSHURI                         bool
 		DefaultCloseIssuesViaCommitsInAnyBranch bool
+		DefaultIsHookTaskPurgeEnabled           bool
+		DefaultNumberWebhookDeliveriesToKeep    int64
 		EnablePushCreateUser                    bool
 		EnablePushCreateOrg                     bool
 		DisabledRepoUnits                       []string
@@ -139,6 +141,8 @@ var (
 		AccessControlAllowOrigin:                "",
 		UseCompatSSHURI:                         false,
 		DefaultCloseIssuesViaCommitsInAnyBranch: false,
+		DefaultIsHookTaskPurgeEnabled:           true,
+		DefaultNumberWebhookDeliveriesToKeep:    10,
 		EnablePushCreateUser:                    false,
 		EnablePushCreateOrg:                     false,
 		DisabledRepoUnits:                       []string{},

options/locale/locale_en-US.ini

@@ -1446,6 +1446,8 @@ settings.projects_desc = Enable Repository Projects
 settings.admin_settings = Administrator Settings
 settings.admin_enable_health_check = Enable Repository Health Checks (git fsck)
 settings.admin_enable_close_issues_via_commit_in_any_branch = Close an issue via a commit made in a non default branch
+settings.admin_enable_hook_task_purge_1 = Purge webhook delivery notifications
+settings.admin_enable_hook_task_purge_2 = Webhook deliveries to keep
 settings.danger_zone = Danger Zone
 settings.new_owner_has_same_repo = The new owner already has a repository with same name. Please choose another name.
 settings.convert = Convert to Regular Repository
@@ -1968,6 +1970,7 @@ dashboard.resync_all_sshkeys.desc = (Not needed for the built-in SSH server.)
 dashboard.resync_all_hooks = Resynchronize pre-receive, update and post-receive hooks of all repositories.
 dashboard.reinit_missing_repos = Reinitialize all missing Git repositories for which records exist
 dashboard.sync_external_users = Synchronize external user data
+dashboard.prune_hook_task_table = Prune hook_task table.
 dashboard.server_uptime = Server Uptime
 dashboard.current_goroutine = Current Goroutines
 dashboard.current_memory_usage = Current Memory Usage

routers/repo/setting.go

@@ -332,6 +332,14 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
 			repo.CloseIssuesViaCommitInAnyBranch = form.EnableCloseIssuesViaCommitInAnyBranch
 		}
 
+		if repo.IsHookTaskPurgeEnabled != form.IsHookTaskPurgeEnabled {
+			repo.IsHookTaskPurgeEnabled = form.IsHookTaskPurgeEnabled
+		}
+
+		if form.IsHookTaskPurgeEnabled && repo.NumberWebhookDeliveriesToKeep != form.NumberWebhookDeliveriesToKeep {
+			repo.NumberWebhookDeliveriesToKeep = form.NumberWebhookDeliveriesToKeep
+		}
+
 		if err := models.UpdateRepository(repo, false); err != nil {
 			ctx.ServerError("UpdateRepository", err)
 			return

templates/repo/settings/options.tmpl

@@ -354,11 +354,22 @@
 						<label>{{.i18n.Tr "repo.settings.admin_enable_health_check"}}</label>
 					</div>
 				</div>
-				<div class="ui checkbox">
-					<input name="enable_close_issues_via_commit_in_any_branch" type="checkbox" {{ if .Repository.CloseIssuesViaCommitInAnyBranch }}checked{{end}}>
-					<label>{{.i18n.Tr "repo.settings.admin_enable_close_issues_via_commit_in_any_branch"}}</label>
+				<div class="field">
+					<div class="ui checkbox">
+						<input name="enable_close_issues_via_commit_in_any_branch" type="checkbox" {{ if .Repository.CloseIssuesViaCommitInAnyBranch }}checked{{end}}>
+						<label>{{.i18n.Tr "repo.settings.admin_enable_close_issues_via_commit_in_any_branch"}}</label>
+					</div>
+				</div>
+				<div class="field">
+					<div class="ui checkbox">
+						<input class="enable-webhooks-to-keep" id="is_hook_task_purge_enabled" name="is_hook_task_purge_enabled" data-target="#number_webhook_deliveries_to_keep_box" type="checkbox" {{ if .Repository.IsHookTaskPurgeEnabled }}checked{{end}}>
+						<label for="is_hook_task_purge_enabled">{{.i18n.Tr "repo.settings.admin_enable_hook_task_purge_1"}}</label>
+					</div>
+				</div>
+				<div class="field {{if not .Repository.IsHookTaskPurgeEnabled}}disabled{{end}}" id="number_webhook_deliveries_to_keep_box">
+					<label for="number_webhook_deliveries_to_keep">{{.i18n.Tr "repo.settings.admin_enable_hook_task_purge_2"}}</label>
+					<input id="number_webhook_deliveries_to_keep" name="number_webhook_deliveries_to_keep" type="number" value="{{ .Repository.NumberWebhookDeliveriesToKeep }}">
 				</div>
-
 				<div class="ui divider"></div>
 				<div class="field">
 					<button class="ui green button">{{$.i18n.Tr "repo.settings.update_settings"}}</button>

web_src/js/index.js

@@ -743,6 +743,17 @@ async function initRepository() {
         if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled');
       }
     });
+
+    // Enable number of webhooks to keep.
+    $('.enable-webhooks-to-keep').change(function () {
+      if (this.checked) {
+        $($(this).data('target')).removeClass('disabled');
+        if (!$(this).data('context')) $($(this).data('context')).addClass('disabled');
+      } else {
+        $($(this).data('target')).addClass('disabled');
+        if (!$(this).data('context')) $($(this).data('context')).removeClass('disabled');
+      }
+    });
   }
 
   // Labels