TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2026-03-06 22:35:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加部分数据清理周期设置

Browse Source
This commit is contained in:
刘祥超
2023-07-01 17:54:40 +08:00
parent ed2b831e5a
commit 0ef7e6ccd8
13 changed files with 219 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/db/models/stats/node_cluster_traffic_daily_stat_dao.go
View File

@@ -1,6 +1,7 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedNodeClusterTrafficDailyStatDAO.Clean(nil, 30) // 只保留N天
err := SharedNodeClusterTrafficDailyStatDAO.CleanDefaultDays(nil, 30) // 只保留N天
if err != nil {
remotelogs.Error("NodeClusterTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
}
@@ -128,11 +129,27 @@ func (this *NodeClusterTrafficDailyStatDAO) SumDailyStat(tx *dbs.Tx, clusterId i
return one.(*NodeClusterTrafficDailyStat), nil
}
// Clean 清理历史数据
func (this *NodeClusterTrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
// CleanDays 清理历史数据
func (this *NodeClusterTrafficDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}
func (this *NodeClusterTrafficDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
if err != nil {
return err
}
if databaseConfig != nil && databaseConfig.NodeClusterTrafficDailyStat.Clean.Days > 0 {
defaultDays = databaseConfig.NodeClusterTrafficDailyStat.Clean.Days
}
if defaultDays <= 0 {
defaultDays = 30
}
return this.CleanDays(tx, defaultDays)
}

23
internal/db/models/stats/node_traffic_hourly_stat_dao.go
View File

@@ -1,6 +1,7 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedNodeTrafficHourlyStatDAO.Clean(nil, 15) // 只保留N天
err := SharedNodeTrafficHourlyStatDAO.CleanDefaultDays(nil, 15) // 只保留N天
if err != nil {
remotelogs.Error("NodeTrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
}
@@ -197,11 +198,27 @@ func (this *NodeTrafficHourlyStatDAO) FindHourlyStatsWithNodeId(tx *dbs.Tx, role
return result, nil
}
// Clean 清理历史数据
func (this *NodeTrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
// CleanDays 清理历史数据
func (this *NodeTrafficHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("hour", hour).
Delete()
return err
}
func (this *NodeTrafficHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
if err != nil {
return err
}
if databaseConfig != nil && databaseConfig.NodeTrafficHourlyStat.Clean.Days > 0 {
defaultDays = databaseConfig.NodeTrafficHourlyStat.Clean.Days
}
if defaultDays <= 0 {
defaultDays = 15
}
return this.CleanDays(tx, defaultDays)
}

23
internal/db/models/stats/server_domain_hourly_stat_dao.go
View File

@@ -1,6 +1,7 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -25,7 +26,7 @@ func init() {
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedServerDomainHourlyStatDAO.Clean(nil, 7) // 只保留 N 天
err := SharedServerDomainHourlyStatDAO.CleanDefaultDays(nil, 7) // 只保留 N 天
if err != nil {
remotelogs.Error("ServerDomainHourlyStatDAO", "clean expired data failed: "+err.Error())
}
@@ -366,8 +367,8 @@ func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithServerId(tx *dbs.Tx
return
}
// Clean 清理历史数据
func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
// CleanDays 清理历史数据
func (this *ServerDomainHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
for _, table := range this.FindAllPartitionTables() {
_, err := this.Query(tx).
@@ -380,3 +381,19 @@ func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
}
return nil
}
func (this *ServerDomainHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
if err != nil {
return err
}
if databaseConfig != nil && databaseConfig.ServerDomainHourlyStat.Clean.Days > 0 {
defaultDays = databaseConfig.ServerDomainHourlyStat.Clean.Days
}
if defaultDays <= 0 {
defaultDays = 7
}
return this.CleanDays(tx, defaultDays)
}

23
internal/db/models/stats/traffic_daily_stat_dao.go
View File

@@ -1,6 +1,7 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedTrafficDailyStatDAO.Clean(nil, 30) // 只保留N天
err := SharedTrafficDailyStatDAO.CleanDefaultDays(nil, 30) // 只保留N天
if err != nil {
remotelogs.Error("TrafficDailyStatDAO", "clean expired data failed: "+err.Error())
}
@@ -124,11 +125,27 @@ func (this *TrafficDailyStatDAO) FindDailyStat(tx *dbs.Tx, day string) (*Traffic
return one.(*TrafficDailyStat), nil
}
// Clean 清理历史数据
func (this *TrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
// CleanDays 清理历史数据
func (this *TrafficDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("day", day).
Delete()
return err
}
func (this *TrafficDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
if err != nil {
return err
}
if databaseConfig != nil && databaseConfig.TrafficDailyStat.Clean.Days > 0 {
defaultDays = databaseConfig.TrafficDailyStat.Clean.Days
}
if defaultDays <= 0 {
defaultDays = 30
}
return this.CleanDays(tx, defaultDays)
}

23
internal/db/models/stats/traffic_hourly_stat_dao.go
View File

@@ -1,6 +1,7 @@
package stats
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
goman.New(func() {
for range ticker.C {
err := SharedTrafficHourlyStatDAO.Clean(nil, 15) // 只保留N天
err := SharedTrafficHourlyStatDAO.CleanDefaultDays(nil, 15) // 只保留N天
if err != nil {
remotelogs.Error("TrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
}
@@ -137,11 +138,27 @@ func (this *TrafficHourlyStatDAO) SumHourlyStats(tx *dbs.Tx, hourFrom string, ho
return one.(*TrafficHourlyStat), nil
}
// Clean 清理历史数据
func (this *TrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
// CleanDays 清理历史数据
func (this *TrafficHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Lt("hour", hour).
Delete()
return err
}
func (this *TrafficHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
if err != nil {
return err
}
if databaseConfig != nil && databaseConfig.TrafficHourlyStat.Clean.Days > 0 {
defaultDays = databaseConfig.TrafficHourlyStat.Clean.Days
}
if defaultDays <= 0 {
defaultDays = 15
}
return this.CleanDays(tx, defaultDays)
}