support auto test exec history info database gc

chengjoey · Jul 19, 2024 · 0b665c1 · 0b665c1
1 parent 0666607
commit 0b665c1
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 0 deletions.
diff --git a/cmd/erda-server/bootstrap.yaml b/cmd/erda-server/bootstrap.yaml
@@ -415,6 +415,8 @@ etcd-election@project-management-report:
   root_path: "/project-management-report-election"
 etcd-election@efficiency-measure:
   root_path: "/efficiency-measure-election"
+etcd-election@autotest-testplan-provider:
+  root_path: "/autotest-testplan-election"
 cassandra:
   _enable: ${CASSANDRA_ENABLE:false}
   host: "${CASSANDRA_ADDR:localhost:9042}"

diff --git a/internal/apps/dop/providers/autotest/testplan/db/testplan.go b/internal/apps/dop/providers/autotest/testplan/db/testplan.go
@@ -15,6 +15,8 @@
 package db
 
 import (
+	"time"
+
 	"github.com/erda-project/erda/apistructs"
 	"github.com/erda-project/erda/internal/apps/dop/dao"
 )
@@ -37,6 +39,11 @@ func (db *TestPlanDB) CreateAutoTestExecHistory(execHistory *AutoTestExecHistory
 	return db.Create(execHistory).Error
 }
 
+// DeleteAutoTestExecHistory .
+func (db *TestPlanDB) DeleteAutoTestExecHistory(endTimeCreated time.Time) error {
+	return db.Where("created_at < ?", endTimeCreated).Delete(&AutoTestExecHistory{}).Error
+}
+
 // BatchCreateAutoTestExecHistory .
 func (db *TestPlanDB) BatchCreateAutoTestExecHistory(list []AutoTestExecHistory) error {
 	return db.BulkInsert(list)

diff --git a/internal/apps/dop/providers/autotest/testplan/provider.go b/internal/apps/dop/providers/autotest/testplan/provider.go
@@ -16,22 +16,27 @@ package testplan
 
 import (
 	"context"
+	"time"
 
 	"github.com/jinzhu/gorm"
 
 	"github.com/erda-project/erda-infra/base/logs"
 	"github.com/erda-project/erda-infra/base/servicehub"
 	"github.com/erda-project/erda-infra/pkg/transport"
+	election "github.com/erda-project/erda-infra/providers/etcd-election"
 	"github.com/erda-project/erda-infra/providers/httpserver"
 	"github.com/erda-project/erda-proto-go/core/dop/autotest/testplan/pb"
 	"github.com/erda-project/erda/bundle"
 	"github.com/erda-project/erda/internal/apps/dop/dao"
 	"github.com/erda-project/erda/internal/apps/dop/providers/autotest/testplan/db"
 	"github.com/erda-project/erda/internal/core/org"
+	"github.com/erda-project/erda/internal/tools/pipeline/providers/reconciler/rutil"
 	"github.com/erda-project/erda/pkg/database/dbengine"
 )
 
 type config struct {
+	ExecHistoryDBGCDuration time.Duration `file:"exec_history_dbgc_duration" env:"EXEC_HISTORY_DBGC_DURATION" default:"12h"`
+	ExecHistoryRetainHour   time.Duration `file:"exec_history_retain_hour" env:"EXEC_HISTORY_RETAIN_HOUR" default:"720h"`
 }
 
 // +provider
@@ -45,6 +50,7 @@ type provider struct {
 	TestPlanService *TestPlanService
 	Org             org.Interface
 	RouterManager   httpserver.RouterManager
+	Election        election.Interface `autowired:"etcd-election@autotest-testplan-provider"`
 }
 
 func (p *provider) Init(ctx servicehub.Context) error {
@@ -66,6 +72,7 @@ func (p *provider) Init(ctx servicehub.Context) error {
 	if p.Register != nil {
 		pb.RegisterTestPlanServiceImp(p.Register, p.TestPlanService)
 	}
+	p.Election.OnLeader(p.ExecHistoryGC)
 	return nil
 }
 
@@ -77,6 +84,14 @@ func (p *provider) Run(ctx context.Context) error {
 	return nil
 }
 
+func (p *provider) ExecHistoryGC(ctx context.Context) {
+	p.Log.Infof("start exec history gc")
+	rutil.ContinueWorking(ctx, p.Log, func(ctx context.Context) rutil.WaitDuration {
+		p.TestPlanService.doExecHistoryGC()
+		return rutil.ContinueWorkingWithDefaultInterval
+	}, rutil.WithContinueWorkingDefaultRetryInterval(p.Cfg.ExecHistoryDBGCDuration))
+}
+
 func (p *provider) Provide(ctx servicehub.DependencyContext, args ...interface{}) interface{} {
 	switch {
 	case ctx.Service() == "erda.core.dop.autotest.testplan.TestPlanService" || ctx.Type() == pb.TestPlanServiceServerType() || ctx.Type() == pb.TestPlanServiceHandlerType():

diff --git a/internal/apps/dop/providers/autotest/testplan/testplan.service.go b/internal/apps/dop/providers/autotest/testplan/testplan.service.go
@@ -451,6 +451,15 @@ func (s *TestPlanService) getSceneIDsIncludeRef(setRefMap map[uint64]uint64, set
 	return
 }
 
+func (s *TestPlanService) doExecHistoryGC() error {
+	endTimeCreated := time.Now().Add(-s.p.Cfg.ExecHistoryRetainHour)
+	if err := s.db.DeleteAutoTestExecHistory(endTimeCreated); err != nil {
+		logrus.Errorf("failed to delete exec history, err: %v", err)
+		return err
+	}
+	return nil
+}
+
 func calcRate(num, totalNum int64) float64 {
 	if totalNum == 0 {
 		return 0