增加部分数据清理周期设置

internal/db/models/http_cache_task_dao.go

@@ -33,7 +33,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedHTTPCacheTaskDAO.Clean(nil, 30) // 只保留N天
+				err := SharedHTTPCacheTaskDAO.CleanDefaultDays(nil, 30) // 只保留N天
 				if err != nil {
 					remotelogs.Error("HTTPCacheTaskDAO", "clean expired data failed: "+err.Error())
 				}
@@ -228,8 +228,8 @@ func (this *HTTPCacheTaskDAO) CheckUserTask(tx *dbs.Tx, userId int64, taskId int
 	return nil
 }
 
-// Clean 清理以往的任务
+// CleanDays 清理N天以前的任务
-func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *HTTPCacheTaskDAO) CleanDays(tx *dbs.Tx, days int) error {
 	if days <= 0 {
 		days = 30
 	}
@@ -248,6 +248,23 @@ func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
 	return err
 }
 
+// CleanDefaultDays 清除任务
+func (this *HTTPCacheTaskDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.HTTPCacheTask.Clean.Days > 0 {
+		defaultDays = databaseConfig.HTTPCacheTask.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 30
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}
+
 // NotifyChange 发送通知
 func (this *HTTPCacheTaskDAO) NotifyChange(tx *dbs.Tx, taskId int64) error {
 	// TODO

internal/db/models/node_traffic_daily_stat_dao.go

@@ -22,7 +22,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedNodeTrafficDailyStatDAO.Clean(nil, 30) // 只保留N天
+				err := SharedNodeTrafficDailyStatDAO.CleanDefaultDays(nil, 32) // 只保留N天
 				if err != nil {
 					remotelogs.Error("NodeTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -134,11 +134,27 @@ func (this *NodeTrafficDailyStatDAO) SumDailyStat(tx *dbs.Tx, role string, nodeI
 	return one.(*NodeTrafficDailyStat), nil
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *NodeTrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *NodeTrafficDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("day", day).
 		Delete()
 	return err
 }
+
+func (this *NodeTrafficDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.NodeTrafficDailyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.NodeTrafficDailyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 32
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/server_bandwidth_stat_dao.go

@@ -34,7 +34,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedServerBandwidthStatDAO.Clean(nil)
+				err := SharedServerBandwidthStatDAO.CleanDefaultDays(nil, 100)
 				if err != nil {
 					remotelogs.Error("SharedServerBandwidthStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -745,9 +745,9 @@ func (this *ServerBandwidthStatDAO) SumDailyStat(tx *dbs.Tx, serverId int64, reg
 	return
 }
 
-// Clean 清理过期数据
+// CleanDays 清理过期数据
-func (this *ServerBandwidthStatDAO) Clean(tx *dbs.Tx) error {
+func (this *ServerBandwidthStatDAO) CleanDays(tx *dbs.Tx, days int) error {
-	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days)) // 保留大约3个月的数据
 	return this.runBatch(func(table string, locker *sync.Mutex) error {
 		_, err := this.Query(tx).
 			Table(table).
@@ -757,6 +757,22 @@ func (this *ServerBandwidthStatDAO) Clean(tx *dbs.Tx) error {
 	})
 }
 
+func (this *ServerBandwidthStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.ServerBandwidthStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.ServerBandwidthStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 100
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}
+
 // 批量执行
 func (this *ServerBandwidthStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
 	var locker = &sync.Mutex{}

internal/db/models/server_daily_stat_dao.go

@@ -28,7 +28,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedServerDailyStatDAO.Clean(nil, 60) // 只保留 N 天，时间需要长一些，因为需要用来生成账单
+				err := SharedServerDailyStatDAO.CleanDefaultDays(nil, 60) // 只保留 N 天，时间需要长一些，因为需要用来生成账单
 				if err != nil {
 					logs.Println("ServerDailyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -673,8 +673,8 @@ func (this *ServerDailyStatDAO) compatFindHourlyStats(tx *dbs.Tx, serverId int64
 			result = append(result, stat)
 		} else {
 			result = append(result, &ServerDailyStat{
-				Hour: hour,
+				Hour:     hour,
-				Day: hour[:8],
+				Day:      hour[:8],
 				TimeFrom: hour[8:] + "00",
 			})
 		}
@@ -742,11 +742,27 @@ func (this *ServerDailyStatDAO) UpdateStatFee(tx *dbs.Tx, statId int64, fee floa
 		UpdateQuickly()
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *ServerDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *ServerDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("day", day).
 		Delete()
 	return err
 }
+
+func (this *ServerDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.ServerDailyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.ServerDailyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 60
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/stats/node_cluster_traffic_daily_stat_dao.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedNodeClusterTrafficDailyStatDAO.Clean(nil, 30) // 只保留N天
+				err := SharedNodeClusterTrafficDailyStatDAO.CleanDefaultDays(nil, 30) // 只保留N天
 				if err != nil {
 					remotelogs.Error("NodeClusterTrafficDailyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -128,11 +129,27 @@ func (this *NodeClusterTrafficDailyStatDAO) SumDailyStat(tx *dbs.Tx, clusterId i
 	return one.(*NodeClusterTrafficDailyStat), nil
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *NodeClusterTrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *NodeClusterTrafficDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("day", day).
 		Delete()
 	return err
 }
+
+func (this *NodeClusterTrafficDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.NodeClusterTrafficDailyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.NodeClusterTrafficDailyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 30
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/stats/node_traffic_hourly_stat_dao.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedNodeTrafficHourlyStatDAO.Clean(nil, 15) // 只保留N天
+				err := SharedNodeTrafficHourlyStatDAO.CleanDefaultDays(nil, 15) // 只保留N天
 				if err != nil {
 					remotelogs.Error("NodeTrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -197,11 +198,27 @@ func (this *NodeTrafficHourlyStatDAO) FindHourlyStatsWithNodeId(tx *dbs.Tx, role
 	return result, nil
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *NodeTrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *NodeTrafficHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("hour", hour).
 		Delete()
 	return err
 }
+
+func (this *NodeTrafficHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.NodeTrafficHourlyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.NodeTrafficHourlyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 15
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/stats/server_domain_hourly_stat_dao.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -25,7 +26,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedServerDomainHourlyStatDAO.Clean(nil, 7) // 只保留 N 天
+				err := SharedServerDomainHourlyStatDAO.CleanDefaultDays(nil, 7) // 只保留 N 天
 				if err != nil {
 					remotelogs.Error("ServerDomainHourlyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -366,8 +367,8 @@ func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithServerId(tx *dbs.Tx
 	return
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *ServerDomainHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
 	for _, table := range this.FindAllPartitionTables() {
 		_, err := this.Query(tx).
@@ -380,3 +381,19 @@ func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
 	}
 	return nil
 }
+
+func (this *ServerDomainHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.ServerDomainHourlyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.ServerDomainHourlyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 7
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/stats/traffic_daily_stat_dao.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedTrafficDailyStatDAO.Clean(nil, 30) // 只保留N天
+				err := SharedTrafficDailyStatDAO.CleanDefaultDays(nil, 30) // 只保留N天
 				if err != nil {
 					remotelogs.Error("TrafficDailyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -124,11 +125,27 @@ func (this *TrafficDailyStatDAO) FindDailyStat(tx *dbs.Tx, day string) (*Traffic
 	return one.(*TrafficDailyStat), nil
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *TrafficDailyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *TrafficDailyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("day", day).
 		Delete()
 	return err
 }
+
+func (this *TrafficDailyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.TrafficDailyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.TrafficDailyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 30
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/stats/traffic_hourly_stat_dao.go

@@ -1,6 +1,7 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,7 +23,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedTrafficHourlyStatDAO.Clean(nil, 15) // 只保留N天
+				err := SharedTrafficHourlyStatDAO.CleanDefaultDays(nil, 15) // 只保留N天
 				if err != nil {
 					remotelogs.Error("TrafficHourlyStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -137,11 +138,27 @@ func (this *TrafficHourlyStatDAO) SumHourlyStats(tx *dbs.Tx, hourFrom string, ho
 	return one.(*TrafficHourlyStat), nil
 }
 
-// Clean 清理历史数据
+// CleanDays 清理历史数据
-func (this *TrafficHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
+func (this *TrafficHourlyStatDAO) CleanDays(tx *dbs.Tx, days int) error {
 	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
 	_, err := this.Query(tx).
 		Lt("hour", hour).
 		Delete()
 	return err
 }
+
+func (this *TrafficHourlyStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := models.SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.TrafficHourlyStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.TrafficHourlyStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 15
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}

internal/db/models/sys_setting_dao.go

@@ -262,3 +262,20 @@ func (this *SysSettingDAO) ReadUserRegisterConfig(tx *dbs.Tx) (*userconfigs.User
 	}
 	return config, nil
 }
+
+func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs.DatabaseConfig, err error) {
+	valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeDatabaseConfigSetting)
+	if err != nil {
+		return nil, err
+	}
+	if len(valueJSON) == 0 {
+		return systemconfigs.NewDatabaseConfig(), nil
+	}
+
+	config = systemconfigs.NewDatabaseConfig()
+	err = json.Unmarshal(valueJSON, config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}

internal/db/models/user_bandwidth_stat_dao.go

@@ -36,7 +36,7 @@ func init() {
 		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		goman.New(func() {
 			for range ticker.C {
-				err := SharedUserBandwidthStatDAO.Clean(nil)
+				err := SharedUserBandwidthStatDAO.CleanDefaultDays(nil, 100)
 				if err != nil {
 					remotelogs.Error("SharedUserBandwidthStatDAO", "clean expired data failed: "+err.Error())
 				}
@@ -460,9 +460,9 @@ func (this *UserBandwidthStatDAO) SumDailyStat(tx *dbs.Tx, userId int64, regionI
 	return
 }
 
-// Clean 清理过期数据
+// CleanDays 清理过期数据
-func (this *UserBandwidthStatDAO) Clean(tx *dbs.Tx) error {
+func (this *UserBandwidthStatDAO) CleanDays(tx *dbs.Tx, days int) error {
-	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -100)) // 保留大约3个月的数据
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days)) // 保留大约3个月的数据
 	return this.runBatch(func(table string, locker *sync.Mutex) error {
 		_, err := this.Query(tx).
 			Table(table).
@@ -472,6 +472,22 @@ func (this *UserBandwidthStatDAO) Clean(tx *dbs.Tx) error {
 	})
 }
 
+func (this *UserBandwidthStatDAO) CleanDefaultDays(tx *dbs.Tx, defaultDays int) error {
+	databaseConfig, err := SharedSysSettingDAO.ReadDatabaseConfig(tx)
+	if err != nil {
+		return err
+	}
+
+	if databaseConfig != nil && databaseConfig.UserBandwidthStat.Clean.Days > 0 {
+		defaultDays = databaseConfig.UserBandwidthStat.Clean.Days
+	}
+	if defaultDays <= 0 {
+		defaultDays = 100
+	}
+
+	return this.CleanDays(tx, defaultDays)
+}
+
 // 批量执行
 func (this *UserBandwidthStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
 	var locker = &sync.Mutex{}

internal/setup/sql_upgrade.go

@@ -536,7 +536,7 @@ func upgradeV0_4_1(db *dbs.DB) error {
 	}
 
 	// 执行域名统计清理
-	err = stats.NewServerDomainHourlyStatDAO().Clean(nil, 7)
+	err = stats.NewServerDomainHourlyStatDAO().CleanDays(nil, 7)
 	if err != nil {
 		return err
 	}

internal/tasks/server_access_log_cleaner.go

@@ -52,7 +52,7 @@ func (this *ServerAccessLogCleaner) Loop() error {
 	if len(configJSON) == 0 {
 		return nil
 	}
-	var config = &systemconfigs.DatabaseConfig{}
+	var config = systemconfigs.NewDatabaseConfig()
 	err = json.Unmarshal(configJSON, config)
 	if err != nil {
 		return err