package models import ( "sync" "time" "github.com/TeaOSLab/EdgeAPI/internal/goman" "github.com/TeaOSLab/EdgeAPI/internal/remotelogs" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" _ "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/dbs" "github.com/iwind/TeaGo/maps" "github.com/iwind/TeaGo/rands" "github.com/iwind/TeaGo/types" timeutil "github.com/iwind/TeaGo/utils/time" ) type MetricSumStatDAO dbs.DAO const MetricSumStatTablePartials = 20 // 表格Partial数量 func init() { dbs.OnReadyDone(func() { // 清理数据任务 var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour) goman.New(func() { for range ticker.C { err := SharedMetricSumStatDAO.Clean(nil) if err != nil { remotelogs.Error("SharedMetricSumStatDAO", "clean expired data failed: "+err.Error()) } } }) }) } func NewMetricSumStatDAO() *MetricSumStatDAO { return dbs.NewDAO(&MetricSumStatDAO{ DAOObject: dbs.DAOObject{ DB: Tea.Env, Table: "edgeMetricSumStats", Model: new(MetricSumStat), PkName: "id", }, }).(*MetricSumStatDAO) } var SharedMetricSumStatDAO *MetricSumStatDAO func init() { dbs.OnReady(func() { SharedMetricSumStatDAO = NewMetricSumStatDAO() }) } // UpdateSum 更新统计数据 func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, time string, itemId int64, version int32, count int64, total float32) error { err := this.Query(tx). Table(this.partialTable(serverId)). InsertOrUpdateQuickly(maps.Map{ "clusterId": clusterId, "nodeId": nodeId, "serverId": serverId, "itemId": itemId, "version": version, "time": time, "count": count, "total": total, "createdDay": timeutil.Format("Ymd"), }, maps.Map{ "count": count, "total": total, }) if this.canIgnore(err) { return nil } return err } // FindNodeServerSum 查找某个服务在某个节点上的统计数据 func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { one, err := this.Query(tx). Table(this.partialTable(serverId)). Attr("nodeId", nodeId). Attr("serverId", serverId). Attr("time", time). Attr("itemId", itemId). Attr("version", version). Find() if err != nil { return 0, 0, err } if one == nil { return } count = int64(one.(*MetricSumStat).Count) total = float32(one.(*MetricSumStat).Total) return } // FindSumAtTime 查找某个时间的统计数据 func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int64, version int32) (count int64, total float32, err error) { err = this.runBatch(func(table string, locker *sync.Mutex) error { one, err := this.Query(tx). Table(table). Attr("time", time). Attr("itemId", itemId). Attr("version", version). Result("SUM(count) AS `count`, SUM(total) AS total"). Find() if err != nil { return err } if one == nil { return nil } locker.Lock() count += int64(one.(*MetricSumStat).Count) total += float32(one.(*MetricSumStat).Total) locker.Unlock() return nil }) return } // FindServerSum 查找某个服务的统计数据 func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { one, err := this.Query(tx). Table(this.partialTable(serverId)). UseIndex("server_item_time"). Attr("serverId", serverId). Attr("time", time). Attr("itemId", itemId). Attr("version", version). Result("SUM(count) AS `count`, SUM(total) AS total"). Find() if err != nil { return 0, 0, err } if one == nil { return } return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil } // FindClusterSum 查找集群上的统计数据 func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { err = this.runBatch(func(table string, locker *sync.Mutex) error { one, err := this.Query(tx). Table(table). UseIndex("cluster_item_time"). Attr("clusterId", clusterId). Attr("time", time). Attr("itemId", itemId). Attr("version", version). Result("SUM(count) AS `count`, SUM(total) AS total"). Find() if err != nil { return err } if one == nil { return nil } locker.Lock() count += int64(one.(*MetricSumStat).Count) total += float32(one.(*MetricSumStat).Total) locker.Unlock() return nil }) return } // FindNodeSum 查找节点上的统计数据 func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) { err = this.runBatch(func(table string, locker *sync.Mutex) error { one, err := this.Query(tx). Table(table). UseIndex("node_item_time"). Attr("nodeId", nodeId). Attr("time", time). Attr("itemId", itemId). Attr("version", version). Result("SUM(count) AS `count`, SUM(total) AS total"). Find() if err != nil { return err } if one == nil { return nil } locker.Lock() count += int64(one.(*MetricSumStat).Count) total += float32(one.(*MetricSumStat).Total) locker.Unlock() return nil }) return } // DeleteItemStats 删除某个指标相关的统计数据 func (this *MetricSumStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error { return this.runBatch(func(table string, locker *sync.Mutex) error { _, err := this.Query(tx). Table(table). Attr("itemId", itemId). Delete() return err }) } // Clean 清理数据 func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error { for _, category := range serverconfigs.FindAllMetricItemCategoryCodes() { var offset int64 = 0 var size int64 = 100 for { items, err := SharedMetricItemDAO.ListEnabledItems(tx, category, offset, size) if err != nil { return err } for _, item := range items { var config = &serverconfigs.MetricItemConfig{ Id: int64(item.Id), Period: int(item.Period), PeriodUnit: item.PeriodUnit, ExpiresPeriod: int(item.ExpiresPeriod), } var expiresDay = config.ServerExpiresDay() err = this.runBatch(func(table string, locker *sync.Mutex) error { _, err := this.Query(tx). Table(table). Attr("itemId", item.Id). Where("(createdDay IS NULL OR createdDay<:day)"). Param("day", expiresDay). Limit(10_000). // 一次性不要删除太多,防止阻塞其他操作 Delete() return err }) if err != nil { return err } } if len(items) == 0 { break } offset += size } } return nil } // 获取分区表 func (this *MetricSumStatDAO) partialTable(serverId int64) string { return this.Table + "_" + types.String(serverId%int64(MetricSumStatTablePartials)) } // 批量执行 func (this *MetricSumStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error { var locker = &sync.Mutex{} var wg = sync.WaitGroup{} wg.Add(MetricSumStatTablePartials) var resultErr error for i := 0; i < MetricSumStatTablePartials; i++ { var table = this.partialTable(int64(i)) go func(table string) { defer wg.Done() err := f(table, locker) if err != nil { resultErr = err } }(table) } wg.Wait() return resultErr } // 检查错误是否可以忽略 func (this *MetricSumStatDAO) canIgnore(err error) bool { if err == nil { return true } // 忽略 Error 1213: Deadlock found 错误 return CheckSQLErrCode(err, 1213) }