指标统计数据分表

2025-12-24 22:16:36 +08:00 · 2022-03-30 15:35:42 +08:00
parent 4b3a9cedfa
commit 1c1e82ee38
5 changed files with 136 additions and 60 deletions
--- a/internal/db/models/metric_sum_stat_dao.go
+++ b/internal/db/models/metric_sum_stat_dao.go
@@ -9,12 +9,16 @@ import (
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
 	"time"
 )

 type MetricSumStatDAO dbs.DAO

+const MetricSumStatTablePartials = 20 // 表格Partial数量
+
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
@@ -52,6 +56,7 @@ func init() {
 // UpdateSum 更新统计数据
 func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, time string, itemId int64, version int32, count int64, total float32) error {
 	return this.Query(tx).
+		Table(this.partialTable(serverId)).
 		InsertOrUpdateQuickly(maps.Map{
 			"clusterId":  clusterId,
 			"nodeId":     nodeId,
@@ -71,6 +76,7 @@ func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int6
 // FindNodeServerSum 查找某个服务在某个节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Attr("nodeId", nodeId).
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -83,29 +89,42 @@ func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, server
 	if one == nil {
 		return
 	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+
+	count = int64(one.(*MetricSumStat).Count)
+	total = float32(one.(*MetricSumStat).Total)
+
+	return
 }

 // FindSumAtTime 查找某个时间的统计数据
 func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+	return
 }

 // FindServerSum 查找某个服务的统计数据
 func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		UseIndex("server_item_time").
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -124,48 +143,69 @@ func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time str

 // FindClusterSum 查找集群上的统计数据
 func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		UseIndex("cluster_item_time").
-		Attr("clusterId", clusterId).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			UseIndex("cluster_item_time").
+			Attr("clusterId", clusterId).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }

 // FindNodeSum 查找节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		UseIndex("node_item_time").
-		Attr("nodeId", nodeId).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			UseIndex("node_item_time").
+			Attr("nodeId", nodeId).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }

 // DeleteItemStats 删除某个指标相关的统计数据
 func (this *MetricSumStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
-	_, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Delete()
-	return err
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Delete()
+		return err
+	})
 }

 // Clean 清理数据
@@ -186,13 +226,17 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 					ExpiresPeriod: int(item.ExpiresPeriod),
 				}
 				var expiresDay = config.ServerExpiresDay()
-				_, err := this.Query(tx).
-					Attr("itemId", item.Id).
-					Where("(createdDay IS NULL OR createdDay<:day)").
-					Param("day", expiresDay).
-					UseIndex("createdDay").
-					Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作
-					Delete()
+				err = this.runBatch(func(table string, locker *sync.Mutex) error {
+					_, err := this.Query(tx).
+						Table(table).
+						Attr("itemId", item.Id).
+						Where("(createdDay IS NULL OR createdDay<:day)").
+						Param("day", expiresDay).
+						UseIndex("createdDay").
+						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
+						Delete()
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -207,3 +251,29 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 	}
 	return nil
 }
+
+// 获取分区表
+func (this *MetricSumStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(MetricSumStatTablePartials))
+}
+
+// 批量执行
+func (this *MetricSumStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(MetricSumStatTablePartials)
+	var resultErr error
+	for i := 0; i < MetricSumStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}