TRKJ/EdgeAPI
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeAPI.git synced 2025-11-03 23:20:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

自动清理过期指标统计数据

Browse Source
This commit is contained in:
GoEdgeLab
2021-07-19 21:01:26 +08:00
parent e262bdb498
commit 1dbfb11dd8
12 changed files with 82 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/db/models/metric_stat_dao.go
View File

@@ -6,12 +6,31 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"strconv"
"time"
)
type MetricStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedMetricStatDAO.Clean(nil, 120) // 只保留120天
if err != nil {
logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error())
}
}
}()
})
}
func NewMetricStatDAO() *MetricStatDAO {
return dbs.NewDAO(&MetricStatDAO{
DAOObject: dbs.DAOObject{
@@ -47,15 +66,16 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
return this.Query(tx).
Param("value", value).
InsertOrUpdateQuickly(maps.Map{
"hash": hash,
"clusterId": clusterId,
"nodeId": nodeId,
"serverId": serverId,
"itemId": itemId,
"value": value,
"time": time,
"version": version,
"keys": keysString,
"hash": hash,
"clusterId": clusterId,
"nodeId": nodeId,
"serverId": serverId,
"itemId": itemId,
"value": value,
"time": time,
"version": version,
"keys": keysString,
"createdDay": timeutil.Format("Ymd"),
}, maps.Map{
"value": value,
})
@@ -322,3 +342,14 @@ func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId
lists.Reverse(result)
return
}
// Clean 清理数据
func (this *MetricStatDAO) Clean(tx *dbs.Tx, days int64) error {
_, err := this.Query(tx).
Lt("createdDay", timeutil.FormatTime("Ymd", time.Now().Unix()-days*86400)).
Delete()
if err != nil {
return err
}
return nil
}

42
internal/db/models/metric_stat_model.go
View File

@@ -2,29 +2,31 @@ package models
// MetricStat 指标统计数据
type MetricStat struct {
Id uint64 `field:"id"` // ID
Hash string `field:"hash"` // Hash值
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
ItemId uint64 `field:"itemId"` // 指标
Keys string `field:"keys"` // 键值
Value float64 `field:"value"` // 数值
Time string `field:"time"` // 分钟值YYYYMMDDHHII
Version uint32 `field:"version"` // 版本号
Id uint64 `field:"id"` // ID
Hash string `field:"hash"` // Hash值
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
ItemId uint64 `field:"itemId"` // 指标
Keys string `field:"keys"` // 键值
Value float64 `field:"value"` // 数值
Time string `field:"time"` // 分钟值YYYYMMDDHHII
Version uint32 `field:"version"` // 版本号
CreatedDay string `field:"createdDay"` // YYYYMMDD
}
type MetricStatOperator struct {
Id interface{} // ID
Hash interface{} // Hash值
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID
ItemId interface{} // 指标
Keys interface{} // 键值
Value interface{} // 数值
Time interface{} // 分钟值YYYYMMDDHHII
Version interface{} // 版本号
Id interface{} // ID
Hash interface{} // Hash值
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID
ItemId interface{} // 指标
Keys interface{} // 键值
Value interface{} // 数值
Time interface{} // 分钟值YYYYMMDDHHII
Version interface{} // 版本号
CreatedDay interface{} // YYYYMMDD
}
func NewMetricStatOperator() *MetricStatOperator {

3
internal/db/models/nameservers/ns_record_hourly_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type NSRecordHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedNSRecordHourlyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/server_daily_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"regexp"
"time"
@@ -18,7 +19,7 @@ type ServerDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedServerDailyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/node_cluster_traffic_daily_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type NodeClusterTrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeClusterTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/node_traffic_daily_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type NodeTrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/node_traffic_hourly_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type NodeTrafficHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedNodeTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/server_domain_hourly_stat_dao.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -16,7 +17,7 @@ type ServerDomainHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedServerDomainHourlyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/server_http_firewall_daily_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type ServerHTTPFirewallDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedServerHTTPFirewallDailyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/server_http_firewall_hourly_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type ServerHTTPFirewallHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedServerHTTPFirewallHourlyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/traffic_daily_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type TrafficDailyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedTrafficDailyStatDAO.Clean(nil, 60) // 只保留60天

3
internal/db/models/stats/traffic_hourly_stat_dao.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -17,7 +18,7 @@ type TrafficHourlyStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(24 * time.Hour)
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedTrafficHourlyStatDAO.Clean(nil, 60) // 只保留60天