diff --git a/internal/db/models/metric_item_dao.go b/internal/db/models/metric_item_dao.go index cb606fe0..65a1284e 100644 --- a/internal/db/models/metric_item_dao.go +++ b/internal/db/models/metric_item_dao.go @@ -78,6 +78,12 @@ func (this *MetricItemDAO) DisableMetricItem(tx *dbs.Tx, itemId int64) error { if err != nil { return err } + + err = SharedMetricSumStatDAO.DeleteItemStats(tx, itemId) + if err != nil { + return err + } + return nil } diff --git a/internal/db/models/metric_stat_dao.go b/internal/db/models/metric_stat_dao.go index 6df0c43f..b5d299a9 100644 --- a/internal/db/models/metric_stat_dao.go +++ b/internal/db/models/metric_stat_dao.go @@ -2,6 +2,7 @@ package models import ( "encoding/json" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" @@ -22,7 +23,7 @@ func init() { var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour) go func() { for range ticker.C { - err := SharedMetricStatDAO.Clean(nil, 30) // 只保留30天 + err := SharedMetricStatDAO.Clean(nil) if err != nil { logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error()) } @@ -461,12 +462,38 @@ func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId } // Clean 清理数据 -func (this *MetricStatDAO) Clean(tx *dbs.Tx, days int64) error { - _, err := this.Query(tx). - Lt("createdDay", timeutil.FormatTime("Ymd", time.Now().Unix()-days*86400)). - Delete() - if err != nil { - return err +func (this *MetricStatDAO) Clean(tx *dbs.Tx) error { + for _, category := range serverconfigs.FindAllMetricItemCategoryCodes() { + var offset int64 = 0 + var size int64 = 100 + for { + items, err := SharedMetricItemDAO.ListEnabledItems(tx, category, offset, size) + if err != nil { + return err + } + for _, item := range items { + var config = &serverconfigs.MetricItemConfig{ + Id: int64(item.Id), + Period: int(item.Period), + PeriodUnit: item.PeriodUnit, + } + var expiresDay = config.ServerExpiresDay() + _, err := this.Query(tx). + Attr("itemId", item.Id). + Lte("createdDay", expiresDay). + Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作 + Delete() + if err != nil { + return err + } + } + + if len(items) == 0 { + break + } + + offset += size + } } return nil } diff --git a/internal/db/models/metric_stat_dao_test.go b/internal/db/models/metric_stat_dao_test.go index 2b3d3c83..c736431b 100644 --- a/internal/db/models/metric_stat_dao_test.go +++ b/internal/db/models/metric_stat_dao_test.go @@ -3,6 +3,7 @@ package models import ( _ "github.com/go-sql-driver/mysql" _ "github.com/iwind/TeaGo/bootstrap" + "github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" @@ -10,14 +11,24 @@ import ( ) func TestNewMetricStatDAO_InsertMany(t *testing.T) { - for i := 0; i <= 10_000_000; i++ { + for i := 0; i <= 1; i++ { err := NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0) if err != nil { t.Fatal(err) } - if i % 10000 == 0 { + if i%10000 == 0 { t.Log(i) } } t.Log("done") } + +func TestMetricStatDAO_Clean(t *testing.T) { + dbs.NotifyReady() + + err := NewMetricStatDAO().Clean(nil) + if err != nil { + t.Fatal(err) + } + t.Log("ok") +} diff --git a/internal/db/models/metric_sum_stat_dao.go b/internal/db/models/metric_sum_stat_dao.go index a012c1f2..5526b737 100644 --- a/internal/db/models/metric_sum_stat_dao.go +++ b/internal/db/models/metric_sum_stat_dao.go @@ -1,6 +1,7 @@ package models import ( + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" @@ -19,7 +20,7 @@ func init() { var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour) go func() { for range ticker.C { - err := SharedMetricSumStatDAO.Clean(nil, 30) // 只保留30天 + err := SharedMetricSumStatDAO.Clean(nil) if err != nil { logs.Println("SharedMetricSumStatDAO: clean expired data failed: " + err.Error()) } @@ -104,6 +105,7 @@ func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int6 // FindServerSum 查找某个服务的统计数据 func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { one, err := this.Query(tx). + UseIndex("server_item_time"). Attr("serverId", serverId). Attr("time", time). Attr("itemId", itemId). @@ -122,6 +124,7 @@ func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time str // FindClusterSum 查找集群上的统计数据 func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { one, err := this.Query(tx). + UseIndex("cluster_item_time"). Attr("clusterId", clusterId). Attr("time", time). Attr("itemId", itemId). @@ -140,6 +143,7 @@ func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time s // FindNodeSum 查找节点上的统计数据 func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { one, err := this.Query(tx). + UseIndex("node_item_time"). Attr("nodeId", nodeId). Attr("time", time). Attr("itemId", itemId). @@ -155,14 +159,48 @@ func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil } -// Clean 清理数据 -func (this *MetricSumStatDAO) Clean(tx *dbs.Tx, days int64) error { +// DeleteItemStats 删除某个指标相关的统计数据 +func (this *MetricSumStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error { _, err := this.Query(tx). - Where("(createdDay IS NULL OR createdDay<:day)"). - Param("day", timeutil.FormatTime("Ymd", time.Now().Unix()-days*86400)). + Attr("itemId", itemId). Delete() - if err != nil { - return err + return err +} + +// Clean 清理数据 +func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error { + for _, category := range serverconfigs.FindAllMetricItemCategoryCodes() { + var offset int64 = 0 + var size int64 = 100 + for { + items, err := SharedMetricItemDAO.ListEnabledItems(tx, category, offset, size) + if err != nil { + return err + } + for _, item := range items { + var config = &serverconfigs.MetricItemConfig{ + Id: int64(item.Id), + Period: int(item.Period), + PeriodUnit: item.PeriodUnit, + } + var expiresDay = config.ServerExpiresDay() + _, err := this.Query(tx). + Attr("itemId", item.Id). + Where("(createdDay IS NULL OR createdDay<:day)"). + Param("day", expiresDay). + Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作 + Delete() + if err != nil { + return err + } + } + + if len(items) == 0 { + break + } + + offset += size + } } return nil } diff --git a/internal/db/models/metric_sum_stat_dao_test.go b/internal/db/models/metric_sum_stat_dao_test.go index b4ad5400..ddbd2a90 100644 --- a/internal/db/models/metric_sum_stat_dao_test.go +++ b/internal/db/models/metric_sum_stat_dao_test.go @@ -3,11 +3,14 @@ package models import ( _ "github.com/go-sql-driver/mysql" _ "github.com/iwind/TeaGo/bootstrap" + "github.com/iwind/TeaGo/dbs" "testing" ) func TestMetricSumStatDAO_Clean(t *testing.T) { - err := NewMetricSumStatDAO().Clean(nil, 20) + dbs.NotifyReady() + + err := NewMetricSumStatDAO().Clean(nil) if err != nil { t.Fatal(err) }