对域名统计进行分表处理

2025-11-20 13:00:24 +08:00 · 2021-09-08 17:32:08 +08:00
parent b908050e5c
commit eb37493af2
4 changed files with 290 additions and 49 deletions
--- a/internal/db/models/stats/server_domain_hourly_stat_dao.go
+++ b/internal/db/models/stats/server_domain_hourly_stat_dao.go
@@ -8,7 +8,11 @@ import (
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
+	"strings"
+	"sync"
 	"time"
 )

@@ -48,12 +52,40 @@ func init() {
 	})
 }

+// PartitionTable 获取分区表格名称
+func (this *ServerDomainHourlyStatDAO) PartitionTable(domain string) string {
+	if len(domain) == 0 {
+		return this.Table + "_0"
+	}
+	if (domain[0] >= '0' && domain[0] <= '9') || (domain[0] >= 'a' && domain[0] <= 'z') || (domain[0] >= 'A' && domain[0] <= 'Z') {
+		return this.Table + "_" + strings.ToLower(string(domain[0]))
+	}
+
+	return this.Table + "_0"
+}
+
+// FindAllPartitionTables 获取所有表格名称
+func (this *ServerDomainHourlyStatDAO) FindAllPartitionTables() []string {
+	var tables = []string{}
+	for i := '0'; i <= '9'; i++ {
+		tables = append(tables, this.Table+"_"+string(i))
+	}
+	for i := 'a'; i <= 'z'; i++ {
+		tables = append(tables, this.Table+"_"+string(i))
+	}
+	return tables
+}
+
 // IncreaseHourlyStat 增加统计数据
 func (this *ServerDomainHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, domain string, hour string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttackRequests int64, attackBytes int64) error {
 	if len(hour) != 10 {
 		return errors.New("invalid hour '" + hour + "'")
 	}
+	if len(domain) == 0 {
+		return nil
+	}
 	err := this.Query(tx).
+		Table(this.PartitionTable(domain)).
 		Param("bytes", bytes).
 		Param("cachedBytes", cachedBytes).
 		Param("countRequests", countRequests).
@@ -87,69 +119,209 @@ func (this *ServerDomainHourlyStatDAO) IncreaseHourlyStat(tx *dbs.Tx, clusterId
 }

 // FindTopDomainStats 取得一定时间内的域名排行数据
-func (this *ServerDomainHourlyStatDAO) FindTopDomainStats(tx *dbs.Tx, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, err error) {
-	// TODO 节点如果已经被删除，则忽略
-	_, err = this.Query(tx).
-		Between("hour", hourFrom, hourTo).
-		Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
-		Group("domain").
-		Desc("countRequests").
-		Limit(size).
-		Slice(&result).
-		FindAll()
+func (this *ServerDomainHourlyStatDAO) FindTopDomainStats(tx *dbs.Tx, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, resultErr error) {
+	var tables = this.FindAllPartitionTables()
+	var wg = sync.WaitGroup{}
+	wg.Add(len(tables))
+	var locker = sync.Mutex{}
+
+	for _, table := range tables {
+		go func(table string) {
+			defer wg.Done()
+
+			var topResults = []*ServerDomainHourlyStat{}
+
+			// TODO 节点如果已经被删除，则忽略
+			_, err := this.Query(tx).
+				Table(table).
+				Between("hour", hourFrom, hourTo).
+				Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
+				Group("domain").
+				Desc("countRequests").
+				Limit(size).
+				Slice(&topResults).
+				FindAll()
+			if err != nil {
+				resultErr = err
+				return
+			}
+
+			if len(topResults) > 0 {
+				locker.Lock()
+				result = append(result, topResults...)
+				locker.Unlock()
+			}
+		}(table)
+	}
+	wg.Wait()
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].CountRequests > result[j].CountRequests
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	return
 }

 // FindTopDomainStatsWithClusterId 取得集群上的一定时间内的域名排行数据
-func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithClusterId(tx *dbs.Tx, clusterId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, err error) {
-	// TODO 节点如果已经被删除，则忽略
-	_, err = this.Query(tx).
-		Attr("clusterId", clusterId).
-		Between("hour", hourFrom, hourTo).
-		Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
-		Group("domain").
-		Desc("countRequests").
-		Limit(size).
-		Slice(&result).
-		FindAll()
+func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithClusterId(tx *dbs.Tx, clusterId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, resultErr error) {
+	var tables = this.FindAllPartitionTables()
+	var wg = sync.WaitGroup{}
+	wg.Add(len(tables))
+	var locker = sync.Mutex{}
+
+	for _, table := range tables {
+		go func(table string) {
+			defer wg.Done()
+
+			var topResults = []*ServerDomainHourlyStat{}
+
+			// TODO 节点如果已经被删除，则忽略
+			_, err := this.Query(tx).
+				Table(table).
+				Attr("clusterId", clusterId).
+				Between("hour", hourFrom, hourTo).
+				Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
+				Group("domain").
+				Desc("countRequests").
+				Limit(size).
+				Slice(&topResults).
+				FindAll()
+			if err != nil {
+				resultErr = err
+				return
+			}
+
+			if len(topResults) > 0 {
+				locker.Lock()
+				result = append(result, topResults...)
+				locker.Unlock()
+			}
+		}(table)
+	}
+	wg.Wait()
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].CountRequests > result[j].CountRequests
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	return
 }

 // FindTopDomainStatsWithNodeId 取得节点上的一定时间内的域名排行数据
-func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithNodeId(tx *dbs.Tx, nodeId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, err error) {
-	// TODO 节点如果已经被删除，则忽略
-	_, err = this.Query(tx).
-		Attr("nodeId", nodeId).
-		Between("hour", hourFrom, hourTo).
-		Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
-		Group("domain").
-		Desc("countRequests").
-		Limit(size).
-		Slice(&result).
-		FindAll()
+func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithNodeId(tx *dbs.Tx, nodeId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, resultErr error) {
+	var tables = this.FindAllPartitionTables()
+	var wg = sync.WaitGroup{}
+	wg.Add(len(tables))
+	var locker = sync.Mutex{}
+
+	for _, table := range tables {
+		go func(table string) {
+			defer wg.Done()
+
+			var topResults = []*ServerDomainHourlyStat{}
+
+			// TODO 节点如果已经被删除，则忽略
+			_, err := this.Query(tx).
+				Table(table).
+				Attr("nodeId", nodeId).
+				Between("hour", hourFrom, hourTo).
+				Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
+				Group("domain").
+				Desc("countRequests").
+				Limit(size).
+				Slice(&topResults).
+				FindAll()
+			if err != nil {
+				resultErr = err
+				return
+			}
+
+			if len(topResults) > 0 {
+				locker.Lock()
+				result = append(result, topResults...)
+				locker.Unlock()
+			}
+		}(table)
+	}
+	wg.Wait()
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].CountRequests > result[j].CountRequests
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	return
 }

 // FindTopDomainStatsWithServerId 取得某个服务的一定时间内的域名排行数据
-func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithServerId(tx *dbs.Tx, serverId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, err error) {
-	// TODO 节点如果已经被删除，则忽略
-	_, err = this.Query(tx).
-		Attr("serverId", serverId).
-		Between("hour", hourFrom, hourTo).
-		Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
-		Group("domain").
-		Desc("countRequests").
-		Limit(size).
-		Slice(&result).
-		FindAll()
+func (this *ServerDomainHourlyStatDAO) FindTopDomainStatsWithServerId(tx *dbs.Tx, serverId int64, hourFrom string, hourTo string, size int64) (result []*ServerDomainHourlyStat, resultErr error) {
+	var tables = this.FindAllPartitionTables()
+	var wg = sync.WaitGroup{}
+	wg.Add(len(tables))
+	var locker = sync.Mutex{}
+
+	for _, table := range tables {
+		go func(table string) {
+			defer wg.Done()
+
+			var topResults = []*ServerDomainHourlyStat{}
+
+			// TODO 节点如果已经被删除，则忽略
+			_, err := this.Query(tx).
+				Table(table).
+				Attr("serverId", serverId).
+				Between("hour", hourFrom, hourTo).
+				Result("domain, MIN(serverId) AS serverId, SUM(bytes) AS bytes, SUM(cachedBytes) AS cachedBytes, SUM(countRequests) AS countRequests, SUM(countCachedRequests) AS countCachedRequests, SUM(countAttackRequests) AS countAttackRequests, SUM(attackBytes) AS attackBytes").
+				Group("domain").
+				Desc("countRequests").
+				Limit(size).
+				Slice(&topResults).
+				FindAll()
+			if err != nil {
+				resultErr = err
+				return
+			}
+
+			if len(topResults) > 0 {
+				locker.Lock()
+				result = append(result, topResults...)
+				locker.Unlock()
+			}
+		}(table)
+	}
+	wg.Wait()
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].CountRequests > result[j].CountRequests
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	return
 }

 // Clean 清理历史数据
 func (this *ServerDomainHourlyStatDAO) Clean(tx *dbs.Tx, days int) error {
 	var hour = timeutil.Format("Ymd00", time.Now().AddDate(0, 0, -days))
-	_, err := this.Query(tx).
-		Lt("hour", hour).
-		Delete()
-	return err
+	for _, table := range this.FindAllPartitionTables() {
+		_, err := this.Query(tx).
+			Table(table).
+			Lt("hour", hour).
+			Delete()
+		return err
+	}
+	return nil
 }