自动清理过期指标统计数据

internal/db/models/metric_stat_dao.go

@@ -6,12 +6,31 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"strconv"
+	"time"
 )
 
 type MetricStatDAO dbs.DAO
 
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
+		go func() {
+			for range ticker.C {
+				err := SharedMetricStatDAO.Clean(nil, 120) // 只保留120天
+				if err != nil {
+					logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error())
+				}
+			}
+		}()
+	})
+}
+
 func NewMetricStatDAO() *MetricStatDAO {
 	return dbs.NewDAO(&MetricStatDAO{
 		DAOObject: dbs.DAOObject{
@@ -47,15 +66,16 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
 	return this.Query(tx).
 		Param("value", value).
 		InsertOrUpdateQuickly(maps.Map{
-			"hash":      hash,
+			"hash":       hash,
-			"clusterId": clusterId,
+			"clusterId":  clusterId,
-			"nodeId":    nodeId,
+			"nodeId":     nodeId,
-			"serverId":  serverId,
+			"serverId":   serverId,
-			"itemId":    itemId,
+			"itemId":     itemId,
-			"value":     value,
+			"value":      value,
-			"time":      time,
+			"time":       time,
-			"version":   version,
+			"version":    version,
-			"keys":      keysString,
+			"keys":       keysString,
+			"createdDay": timeutil.Format("Ymd"),
 		}, maps.Map{
 			"value": value,
 		})
@@ -322,3 +342,14 @@ func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId
 	lists.Reverse(result)
 	return
 }
+
+// Clean 清理数据
+func (this *MetricStatDAO) Clean(tx *dbs.Tx, days int64) error {
+	_, err := this.Query(tx).
+		Lt("createdDay", timeutil.FormatTime("Ymd", time.Now().Unix()-days*86400)).
+		Delete()
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/db/models/metric_stat_model.go

@@ -2,29 +2,31 @@ package models
 
 // MetricStat 指标统计数据
 type MetricStat struct {
-	Id        uint64  `field:"id"`        // ID
+	Id         uint64  `field:"id"`         // ID
-	Hash      string  `field:"hash"`      // Hash值
+	Hash       string  `field:"hash"`       // Hash值
-	ClusterId uint32  `field:"clusterId"` // 集群ID
+	ClusterId  uint32  `field:"clusterId"`  // 集群ID
-	NodeId    uint32  `field:"nodeId"`    // 节点ID
+	NodeId     uint32  `field:"nodeId"`     // 节点ID
-	ServerId  uint32  `field:"serverId"`  // 服务ID
+	ServerId   uint32  `field:"serverId"`   // 服务ID
-	ItemId    uint64  `field:"itemId"`    // 指标
+	ItemId     uint64  `field:"itemId"`     // 指标
-	Keys      string  `field:"keys"`      // 键值
+	Keys       string  `field:"keys"`       // 键值
-	Value     float64 `field:"value"`     // 数值
+	Value      float64 `field:"value"`      // 数值
-	Time      string  `field:"time"`      // 分钟值YYYYMMDDHHII
+	Time       string  `field:"time"`       // 分钟值YYYYMMDDHHII
-	Version   uint32  `field:"version"`   // 版本号
+	Version    uint32  `field:"version"`    // 版本号
+	CreatedDay string  `field:"createdDay"` // YYYYMMDD
 }
 
 type MetricStatOperator struct {
-	Id        interface{} // ID
+	Id         interface{} // ID
-	Hash      interface{} // Hash值
+	Hash       interface{} // Hash值
-	ClusterId interface{} // 集群ID
+	ClusterId  interface{} // 集群ID
-	NodeId    interface{} // 节点ID
+	NodeId     interface{} // 节点ID
-	ServerId  interface{} // 服务ID
+	ServerId   interface{} // 服务ID
-	ItemId    interface{} // 指标
+	ItemId     interface{} // 指标
-	Keys      interface{} // 键值
+	Keys       interface{} // 键值
-	Value     interface{} // 数值
+	Value      interface{} // 数值
-	Time      interface{} // 分钟值YYYYMMDDHHII
+	Time       interface{} // 分钟值YYYYMMDDHHII
-	Version   interface{} // 版本号
+	Version    interface{} // 版本号
+	CreatedDay interface{} // YYYYMMDD
 }
 
 func NewMetricStatOperator() *MetricStatOperator {

internal/db/models/nameservers/ns_record_hourly_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type NSRecordHourlyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedNSRecordHourlyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/server_daily_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"regexp"
 	"time"
@@ -18,7 +19,7 @@ type ServerDailyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedServerDailyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/node_cluster_traffic_daily_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type NodeClusterTrafficDailyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedNodeClusterTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/node_traffic_daily_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type NodeTrafficDailyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedNodeTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/node_traffic_hourly_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type NodeTrafficHourlyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedNodeTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/server_domain_hourly_stat_dao.go

@@ -7,6 +7,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -16,7 +17,7 @@ type ServerDomainHourlyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedServerDomainHourlyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/server_http_firewall_daily_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type ServerHTTPFirewallDailyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedServerHTTPFirewallDailyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/server_http_firewall_hourly_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type ServerHTTPFirewallHourlyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedServerHTTPFirewallHourlyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/traffic_daily_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type TrafficDailyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

internal/db/models/stats/traffic_hourly_stat_dao.go

@@ -8,6 +8,7 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -17,7 +18,7 @@ type TrafficHourlyStatDAO dbs.DAO
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
-		var ticker = time.NewTicker(24 * time.Hour)
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
 		go func() {
 			for range ticker.C {
 				err := SharedTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天